This is an archive of the discontinued Mercurial Phabricator instance.

Differential D12262

automation: drop support for Python 2.7 in Linux environment
ClosedPublic

Authored by indygreg on Mar 2 2022, 10:43 AM.

Details

Reviewers
Alphare
Group Reviewers
hg-reviewers
Commits
rHG5c8148cd7f13: automation: drop support for Python 2.7 in Linux environment
Summary

We stop installing Python 2.7 via pyenv. We stop installing the system
Python 2 packages. We delete support for running tests on Python 2.7.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

Diff 32446

contrib/automation/hgautomation/aws.py

# 8 VCPUs for compiling Python. # 8 VCPUs for compiling Python.
'InstanceType': 't3.2xlarge', 'InstanceType': 't3.2xlarge',
'KeyName': '%sautomation' % prefix, 'KeyName': '%sautomation' % prefix,
'MaxCount': 1, 'MaxCount': 1,
'MinCount': 1, 'MinCount': 1,
'SecurityGroupIds': [c.security_groups['linux-dev-1'].id], 'SecurityGroupIds': [c.security_groups['linux-dev-1'].id],
} }
requirements2_path = (
pathlib.Path(__file__).parent.parent / 'linux-requirements-py2.txt'
)
requirements3_path = ( requirements3_path = (
pathlib.Path(__file__).parent.parent / 'linux-requirements-py3.txt' pathlib.Path(__file__).parent.parent / 'linux-requirements-py3.txt'
) )
requirements35_path = ( requirements35_path = (
pathlib.Path(__file__).parent.parent / 'linux-requirements-py3.5.txt' pathlib.Path(__file__).parent.parent / 'linux-requirements-py3.5.txt'
) )
with requirements2_path.open('r', encoding='utf-8') as fh:
requirements2 = fh.read()
with requirements3_path.open('r', encoding='utf-8') as fh: with requirements3_path.open('r', encoding='utf-8') as fh:
requirements3 = fh.read() requirements3 = fh.read()
with requirements35_path.open('r', encoding='utf-8') as fh: with requirements35_path.open('r', encoding='utf-8') as fh:
requirements35 = fh.read() requirements35 = fh.read()
# Compute a deterministic fingerprint to determine whether image needs to # Compute a deterministic fingerprint to determine whether image needs to
# be regenerated. # be regenerated.
fingerprint = resolve_fingerprint( fingerprint = resolve_fingerprint(
{ {
'instance_config': config, 'instance_config': config,
'bootstrap_script': BOOTSTRAP_DEBIAN, 'bootstrap_script': BOOTSTRAP_DEBIAN,
'requirements_py2': requirements2,
'requirements_py3': requirements3, 'requirements_py3': requirements3,
'requirements_py35': requirements35, 'requirements_py35': requirements35,
} }
) )
existing_image = find_and_reconcile_image(ec2resource, name, fingerprint) existing_image = find_and_reconcile_image(ec2resource, name, fingerprint)
if existing_image: if existing_image:
print('connecting to SSH server') print('connecting to SSH server')
sftp = client.open_sftp() sftp = client.open_sftp()
print('uploading bootstrap files') print('uploading bootstrap files')
with sftp.open('%s/bootstrap' % home, 'wb') as fh: with sftp.open('%s/bootstrap' % home, 'wb') as fh:
fh.write(BOOTSTRAP_DEBIAN) fh.write(BOOTSTRAP_DEBIAN)
fh.chmod(0o0700) fh.chmod(0o0700)
with sftp.open('%s/requirements-py2.txt' % home, 'wb') as fh:
fh.write(requirements2)
fh.chmod(0o0700)
with sftp.open('%s/requirements-py3.txt' % home, 'wb') as fh: with sftp.open('%s/requirements-py3.txt' % home, 'wb') as fh:
fh.write(requirements3) fh.write(requirements3)
fh.chmod(0o0700) fh.chmod(0o0700)
with sftp.open('%s/requirements-py3.5.txt' % home, 'wb') as fh: with sftp.open('%s/requirements-py3.5.txt' % home, 'wb') as fh:
fh.write(requirements35) fh.write(requirements35)
fh.chmod(0o0700) fh.chmod(0o0700)

contrib/automation/hgautomation/cli.py

'--instance-type', '--instance-type',
help='EC2 instance type to use', help='EC2 instance type to use',
default='c5.9xlarge', default='c5.9xlarge',
) )
sp.add_argument( sp.add_argument(
'--python-version', '--python-version',
help='Python version to use', help='Python version to use',
choices={ choices={
'system2',
'system3', 'system3',
'2.7',
'3.5', '3.5',
'3.6', '3.6',
'3.7', '3.7',
'3.8', '3.8',
'pypy', 'pypy',
'pypy3.5', 'pypy3.5',
'pypy3.6', 'pypy3.6',
}, },

contrib/automation/hgautomation/linux.py

DISTROS = { DISTROS = {
'debian9', 'debian9',
'debian10', 'debian10',
'ubuntu18.04', 'ubuntu18.04',
'ubuntu19.04', 'ubuntu19.04',
} }
INSTALL_PYTHONS = r''' INSTALL_PYTHONS = r'''
PYENV2_VERSIONS="2.7.17 pypy2.7-7.2.0"
PYENV3_VERSIONS="3.5.10 3.6.13 3.7.10 3.8.10 3.9.5 pypy3.5-7.0.0 pypy3.6-7.3.3 pypy3.7-7.3.3" PYENV3_VERSIONS="3.5.10 3.6.13 3.7.10 3.8.10 3.9.5 pypy3.5-7.0.0 pypy3.6-7.3.3 pypy3.7-7.3.3"
git clone https://github.com/pyenv/pyenv.git /hgdev/pyenv git clone https://github.com/pyenv/pyenv.git /hgdev/pyenv
pushd /hgdev/pyenv pushd /hgdev/pyenv
git checkout 328fd42c3a2fbf14ae46dae2021a087fe27ba7e2 git checkout 328fd42c3a2fbf14ae46dae2021a087fe27ba7e2
popd popd
export PYENV_ROOT="/hgdev/pyenv" export PYENV_ROOT="/hgdev/pyenv"
export PATH="$PYENV_ROOT/bin:$PATH" export PATH="$PYENV_ROOT/bin:$PATH"
# pip 19.2.3. # pip 19.2.3.
PIP_SHA256=57e3643ff19f018f8a00dfaa6b7e4620e3c1a7a2171fd218425366ec006b3bfe PIP_SHA256=57e3643ff19f018f8a00dfaa6b7e4620e3c1a7a2171fd218425366ec006b3bfe
wget -O get-pip.py --progress dot:mega https://github.com/pypa/get-pip/raw/309a56c5fd94bd1134053a541cb4657a4e47e09d/get-pip.py wget -O get-pip.py --progress dot:mega https://github.com/pypa/get-pip/raw/309a56c5fd94bd1134053a541cb4657a4e47e09d/get-pip.py
echo "${PIP_SHA256} get-pip.py" | sha256sum --check - echo "${PIP_SHA256} get-pip.py" | sha256sum --check -
VIRTUALENV_SHA256=f78d81b62d3147396ac33fc9d77579ddc42cc2a98dd9ea38886f616b33bc7fb2 VIRTUALENV_SHA256=f78d81b62d3147396ac33fc9d77579ddc42cc2a98dd9ea38886f616b33bc7fb2
VIRTUALENV_TARBALL=virtualenv-16.7.5.tar.gz VIRTUALENV_TARBALL=virtualenv-16.7.5.tar.gz
wget -O ${VIRTUALENV_TARBALL} --progress dot:mega https://files.pythonhosted.org/packages/66/f0/6867af06d2e2f511e4e1d7094ff663acdebc4f15d4a0cb0fed1007395124/${VIRTUALENV_TARBALL} wget -O ${VIRTUALENV_TARBALL} --progress dot:mega https://files.pythonhosted.org/packages/66/f0/6867af06d2e2f511e4e1d7094ff663acdebc4f15d4a0cb0fed1007395124/${VIRTUALENV_TARBALL}
echo "${VIRTUALENV_SHA256} ${VIRTUALENV_TARBALL}" | sha256sum --check - echo "${VIRTUALENV_SHA256} ${VIRTUALENV_TARBALL}" | sha256sum --check -
for v in ${PYENV2_VERSIONS}; do
pyenv install -v ${v}
${PYENV_ROOT}/versions/${v}/bin/python get-pip.py
${PYENV_ROOT}/versions/${v}/bin/pip install ${VIRTUALENV_TARBALL}
${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/requirements-py2.txt
done
for v in ${PYENV3_VERSIONS}; do for v in ${PYENV3_VERSIONS}; do
pyenv install -v ${v} pyenv install -v ${v}
${PYENV_ROOT}/versions/${v}/bin/python get-pip.py ${PYENV_ROOT}/versions/${v}/bin/python get-pip.py
case ${v} in case ${v} in
3.5.*) 3.5.*)
REQUIREMENTS=requirements-py3.5.txt REQUIREMENTS=requirements-py3.5.txt
;; ;;
pypy3.5*) pypy3.5*)
REQUIREMENTS=requirements-py3.5.txt REQUIREMENTS=requirements-py3.5.txt
;; ;;
*) *)
REQUIREMENTS=requirements-py3.txt REQUIREMENTS=requirements-py3.txt
;; ;;
esac esac
${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/${REQUIREMENTS} ${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/${REQUIREMENTS}
done done
pyenv global ${PYENV2_VERSIONS} ${PYENV3_VERSIONS} system pyenv global ${PYENV3_VERSIONS} system
'''.lstrip().replace( '''.lstrip().replace(
'\r\n', '\n' '\r\n', '\n'
) )
INSTALL_PYOXIDIZER = r''' INSTALL_PYOXIDIZER = r'''
PYOXIDIZER_VERSION=0.16.0 PYOXIDIZER_VERSION=0.16.0
PYOXIDIZER_SHA256=8875471c270312fbb934007fd30f65f1904cc0f5da6188d61c90ed2129b9f9c1 PYOXIDIZER_SHA256=8875471c270312fbb934007fd30f65f1904cc0f5da6188d61c90ed2129b9f9c1
PYOXIDIZER_URL=https://github.com/indygreg/PyOxidizer/releases/download/pyoxidizer%2F${PYOXIDIZER_VERSION}/pyoxidizer-${PYOXIDIZER_VERSION}-linux_x86_64.zip PYOXIDIZER_URL=https://github.com/indygreg/PyOxidizer/releases/download/pyoxidizer%2F${PYOXIDIZER_VERSION}/pyoxidizer-${PYOXIDIZER_VERSION}-linux_x86_64.zip
libncurses5-dev \ libncurses5-dev \
libnss3-dev \ libnss3-dev \
libreadline-dev \ libreadline-dev \
libsqlite3-dev \ libsqlite3-dev \
libssl-dev \ libssl-dev \
netbase \ netbase \
ntfs-3g \ ntfs-3g \
nvme-cli \ nvme-cli \
pyflakes \
pyflakes3 \ pyflakes3 \
pylint \
pylint3 \ pylint3 \
python-all-dev \
python-dev \
python-docutils \
python-fuzzywuzzy \
python-pygments \
python-subversion \
python-vcr \
python3-boto3 \ python3-boto3 \
python3-dev \ python3-dev \
python3-docutils \ python3-docutils \
python3-fuzzywuzzy \ python3-fuzzywuzzy \
python3-pygments \ python3-pygments \
python3-vcr \ python3-vcr \
python3-venv \ python3-venv \
rsync \ rsync \
def run_tests(ssh_client, python_version, test_flags=None): def run_tests(ssh_client, python_version, test_flags=None):
"""Run tests on a remote Linux machine via an SSH client.""" """Run tests on a remote Linux machine via an SSH client."""
test_flags = test_flags or [] test_flags = test_flags or []
print('running tests') print('running tests')
if python_version == 'system2': if python_version == 'system3':
python = '/usr/bin/python2'
elif python_version == 'system3':
python = '/usr/bin/python3' python = '/usr/bin/python3'
elif python_version.startswith('pypy'): elif python_version.startswith('pypy'):
python = '/hgdev/pyenv/shims/%s' % python_version python = '/hgdev/pyenv/shims/%s' % python_version
else: else:
python = '/hgdev/pyenv/shims/python%s' % python_version python = '/hgdev/pyenv/shims/python%s' % python_version
test_flags = ' '.join(shlex.quote(a) for a in test_flags) test_flags = ' '.join(shlex.quote(a) for a in test_flags)

contrib/automation/linux-requirements-py2.txt

  • This file was deleted.
#
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --generate-hashes --output-file=contrib/automation/linux-requirements-py2.txt contrib/automation/linux-requirements.txt.in
#
astroid==1.6.6 \
--hash=sha256:87de48a92e29cedf7210ffa853d11441e7ad94cb47bacd91b023499b51cbc756 \
--hash=sha256:d25869fc7f44f1d9fb7d24fd7ea0639656f5355fc3089cd1f3d18c6ec6b124c7 \
# via pylint
backports.functools-lru-cache==1.6.1 \
--hash=sha256:0bada4c2f8a43d533e4ecb7a12214d9420e66eb206d54bf2d682581ca4b80848 \
--hash=sha256:8fde5f188da2d593bd5bc0be98d9abc46c95bb8a9dde93429570192ee6cc2d4a \
# via astroid, isort, pylint
bzr==2.7.0 ; python_version <= "2.7" and platform_python_implementation == "CPython" \
--hash=sha256:c9f6bbe0a50201dadc5fddadd94ba50174193c6cf6e39e16f6dd0ad98a1df338 \
# via -r contrib/automation/linux-requirements.txt.in
configparser==4.0.2 \
--hash=sha256:254c1d9c79f60c45dfde850850883d5aaa7f19a23f13561243a050d5a7c3fe4c \
--hash=sha256:c7d282687a5308319bf3d2e7706e575c635b0a470342641c93bea0ea3b5331df \
# via pylint
contextlib2==0.6.0.post1 \
--hash=sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e \
--hash=sha256:3355078a159fbb44ee60ea80abd0d87b80b78c248643b49aa6d94673b413609b \
# via vcrpy
docutils==0.16 \
--hash=sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af \
--hash=sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc \
# via -r contrib/automation/linux-requirements.txt.in
enum34==1.1.10 \
--hash=sha256:a98a201d6de3f2ab3db284e70a33b0f896fbf35f8086594e8c9e74b909058d53 \
--hash=sha256:c3858660960c984d6ab0ebad691265180da2b43f07e061c0f8dca9ef3cffd328 \
--hash=sha256:cce6a7477ed816bd2542d03d53db9f0db935dd013b70f336a95c73979289f248 \
# via astroid
funcsigs==1.0.2 \
--hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \
--hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 \
# via mock
futures==3.3.0 \
--hash=sha256:49b3f5b064b6e3afc3316421a3f25f66c137ae88f068abbf72830170033c5e16 \
--hash=sha256:7e033af76a5e35f58e56da7a91e687706faf4e7bdfb2cbc3f2cca6b9bcda9794 \
# via isort
fuzzywuzzy==0.18.0 \
--hash=sha256:45016e92264780e58972dca1b3d939ac864b78437422beecebb3095f8efd00e8 \
--hash=sha256:928244b28db720d1e0ee7587acf660ea49d7e4c632569cad4f1cd7e68a5f0993 \
# via -r contrib/automation/linux-requirements.txt.in
isort==4.3.21 \
--hash=sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1 \
--hash=sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd \
# via pylint
lazy-object-proxy==1.5.1 \
--hash=sha256:00b78a97a79d0dfefa584d44dd1aba9668d3de7ec82335ba0ff51d53ef107143 \
--hash=sha256:042b54fd71c2092e6d10e5e66fa60f65c5954f8145e809f5d9f394c9b13d32ee \
--hash=sha256:11f87dc06eb5f376cc6d5f0c19a1b4dca202035622777c4ce8e5b72c87b035d6 \
--hash=sha256:19ae6f6511a02008ef3554e158c41bb2a8e5c8455935b98d6da076d9f152fd7c \
--hash=sha256:22c1935c6f8e3d6ea2e169eb03928adbdb8a2251d2890f8689368d65e70aa176 \
--hash=sha256:30ef2068f4f94660144515380ef04b93d15add2214eab8be4cd46ebc900d681c \
--hash=sha256:33da47ba3a581860ddd3d38c950a5fe950ca389f7123edd0d6ab0bc473499fe7 \
--hash=sha256:3e8698dc384857413580012f4ca322d89e63ef20fc3d4635a5b606d6d4b61f6a \
--hash=sha256:4fdd7113fc5143c72dacf415079eec42fcbe69cc9d3d291b4ca742e3a9455807 \
--hash=sha256:63b6d9a5077d54db271fcc6772440f7380ec3fa559d0e2497dbfae2f47c2c814 \
--hash=sha256:8133b63b05f12751cddd8e3e7f02ba39dc7cfa7d2ba99d80d7436f0ba26d6b75 \
--hash=sha256:89b8e5780e49753e2b4cd5aab45d3df092ddcbba3de2c4d4492a029588fe1758 \
--hash=sha256:8d82e27cbbea6edb8821751806f39f5dcfd7b46a5e23d27b98d6d8c8ec751df8 \
--hash=sha256:92cedd6e26712505adb1c17fab64651a498cc0102a80ba562ff4a2451088f57a \
--hash=sha256:9723364577b79ad9958a68851fe2acb94da6fd25170c595516a8289e6a129043 \
--hash=sha256:c484020ad26973a14a7cb1e1d2e0bfe97cf6803273ae9bd154e0213cc74bad49 \
--hash=sha256:c697bd1b333b3e6abdff04ef9f5fb4b1936633d9cc4e28d90606705c9083254c \
--hash=sha256:d0f7e14ff3424639d33e6bc449e77e4b345e52c21bbd6f6004a1d219196e2664 \
--hash=sha256:db2df3eff7ed3e6813638686f1bb5934d1a0662d9d3b4196b5164a86be3a1e8f \
--hash=sha256:edbcb4c5efabd93ede05b272296a5a78a67e9b6e82ba7f51a07b8103db06ce01 \
--hash=sha256:ef355fb3802e0fc5a71dadb65a3c317bfc9bdf567d357f8e0b1900b432ffe486 \
--hash=sha256:fe2f61fed5817bf8db01d9a72309ed5990c478a077e9585b58740c26774bce39 \
# via astroid
mccabe==0.6.1 \
--hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
--hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f \
# via pylint
mock==3.0.5 \
--hash=sha256:83657d894c90d5681d62155c82bda9c1187827525880eda8ff5df4ec813437c3 \
--hash=sha256:d157e52d4e5b938c550f39eb2fd15610db062441a9c2747d3dbfa9298211d0f8 \
# via vcrpy
pyflakes==2.2.0 \
--hash=sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92 \
--hash=sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8 \
# via -r contrib/automation/linux-requirements.txt.in
pygments==2.5.2 \
--hash=sha256:2a3fe295e54a20164a9df49c75fa58526d3be48e14aceba6d6b1e8ac0bfd6f1b \
--hash=sha256:98c8aa5a9f778fcd1026a17361ddaf7330d1b7c62ae97c3bb0ae73e0b9b6b0fe \
# via -r contrib/automation/linux-requirements.txt.in
pylint==1.9.5 \
--hash=sha256:367e3d49813d349a905390ac27989eff82ab84958731c5ef0bef867452cfdc42 \
--hash=sha256:97a42df23d436c70132971d1dcb9efad2fe5c0c6add55b90161e773caf729300 \
# via -r contrib/automation/linux-requirements.txt.in
python-levenshtein==0.12.0 \
--hash=sha256:033a11de5e3d19ea25c9302d11224e1a1898fe5abd23c61c7c360c25195e3eb1 \
# via -r contrib/automation/linux-requirements.txt.in
pyyaml==5.3.1 \
--hash=sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97 \
--hash=sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76 \
--hash=sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2 \
--hash=sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648 \
--hash=sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf \
--hash=sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f \
--hash=sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2 \
--hash=sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee \
--hash=sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d \
--hash=sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c \
--hash=sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a \
# via vcrpy
singledispatch==3.4.0.3 \
--hash=sha256:5b06af87df13818d14f08a028e42f566640aef80805c3b50c5056b086e3c2b9c \
--hash=sha256:833b46966687b3de7f438c761ac475213e53b306740f1abfaa86e1d1aae56aa8 \
# via astroid, pylint
six==1.15.0 \
--hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
--hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
# via astroid, mock, pylint, singledispatch, vcrpy
vcrpy==3.0.0 \
--hash=sha256:21168d5ae14263a833d4b71acfd8278d8841114f24be1b4ab4a5719d0c7f07bc \
--hash=sha256:a2e6b653a627f9f3d6ded4d68587e470b91e4c1444e7dae939510dfeacb65276 \
# via -r contrib/automation/linux-requirements.txt.in
wrapt==1.12.1 \
--hash=sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7 \
# via astroid, vcrpy
# WARNING: The following packages were not pinned, but pip requires them to be
# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.
# setuptools
contrib/automation/hgautomation/linux.py