( )⚙ D12262 automation: drop support for Python 2.7 in Linux environment

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D12262

automation: drop support for Python 2.7 in Linux environment
ClosedPublic

Authored by indygreg on Mar 2 2022, 10:43 AM.

Details

Reviewers
Alphare
Group Reviewers
hg-reviewers
Commits
rHG5c8148cd7f13: automation: drop support for Python 2.7 in Linux environment
Summary

We stop installing Python 2.7 via pyenv. We stop installing the system
Python 2 packages. We delete support for running tests on Python 2.7.

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Revision Contents
Changeset List

Diff 32353

contrib/automation/hgautomation/aws.py

# 8 VCPUs for compiling Python. # 8 VCPUs for compiling Python.
'InstanceType': 't3.2xlarge', 'InstanceType': 't3.2xlarge',
'KeyName': '%sautomation' % prefix, 'KeyName': '%sautomation' % prefix,
'MaxCount': 1, 'MaxCount': 1,
'MinCount': 1, 'MinCount': 1,
'SecurityGroupIds': [c.security_groups['linux-dev-1'].id], 'SecurityGroupIds': [c.security_groups['linux-dev-1'].id],
} }
requirements2_path = (
pathlib.Path(__file__).parent.parent / 'linux-requirements-py2.txt'
)
requirements3_path = ( requirements3_path = (
pathlib.Path(__file__).parent.parent / 'linux-requirements-py3.txt' pathlib.Path(__file__).parent.parent / 'linux-requirements-py3.txt'
) )
requirements35_path = ( requirements35_path = (
pathlib.Path(__file__).parent.parent / 'linux-requirements-py3.5.txt' pathlib.Path(__file__).parent.parent / 'linux-requirements-py3.5.txt'
) )
with requirements2_path.open('r', encoding='utf-8') as fh:
requirements2 = fh.read()
with requirements3_path.open('r', encoding='utf-8') as fh: with requirements3_path.open('r', encoding='utf-8') as fh:
requirements3 = fh.read() requirements3 = fh.read()
with requirements35_path.open('r', encoding='utf-8') as fh: with requirements35_path.open('r', encoding='utf-8') as fh:
requirements35 = fh.read() requirements35 = fh.read()
# Compute a deterministic fingerprint to determine whether image needs to # Compute a deterministic fingerprint to determine whether image needs to
# be regenerated. # be regenerated.
fingerprint = resolve_fingerprint( fingerprint = resolve_fingerprint(
{ {
'instance_config': config, 'instance_config': config,
'bootstrap_script': BOOTSTRAP_DEBIAN, 'bootstrap_script': BOOTSTRAP_DEBIAN,
'requirements_py2': requirements2,
'requirements_py3': requirements3, 'requirements_py3': requirements3,
'requirements_py35': requirements35, 'requirements_py35': requirements35,
} }
) )
existing_image = find_and_reconcile_image(ec2resource, name, fingerprint) existing_image = find_and_reconcile_image(ec2resource, name, fingerprint)
if existing_image: if existing_image:
print('connecting to SSH server') print('connecting to SSH server')
sftp = client.open_sftp() sftp = client.open_sftp()
print('uploading bootstrap files') print('uploading bootstrap files')
with sftp.open('%s/bootstrap' % home, 'wb') as fh: with sftp.open('%s/bootstrap' % home, 'wb') as fh:
fh.write(BOOTSTRAP_DEBIAN) fh.write(BOOTSTRAP_DEBIAN)
fh.chmod(0o0700) fh.chmod(0o0700)
with sftp.open('%s/requirements-py2.txt' % home, 'wb') as fh:
fh.write(requirements2)
fh.chmod(0o0700)
with sftp.open('%s/requirements-py3.txt' % home, 'wb') as fh: with sftp.open('%s/requirements-py3.txt' % home, 'wb') as fh:
fh.write(requirements3) fh.write(requirements3)
fh.chmod(0o0700) fh.chmod(0o0700)
with sftp.open('%s/requirements-py3.5.txt' % home, 'wb') as fh: with sftp.open('%s/requirements-py3.5.txt' % home, 'wb') as fh:
fh.write(requirements35) fh.write(requirements35)
fh.chmod(0o0700) fh.chmod(0o0700)

contrib/automation/hgautomation/cli.py

'--instance-type', '--instance-type',
help='EC2 instance type to use', help='EC2 instance type to use',
default='c5.9xlarge', default='c5.9xlarge',
) )
sp.add_argument( sp.add_argument(
'--python-version', '--python-version',
help='Python version to use', help='Python version to use',
choices={ choices={
'system2',
'system3', 'system3',
'2.7',
'3.5', '3.5',
'3.6', '3.6',
'3.7', '3.7',
'3.8', '3.8',
'pypy', 'pypy',
'pypy3.5', 'pypy3.5',
'pypy3.6', 'pypy3.6',
}, },

contrib/automation/hgautomation/linux.py

DISTROS = { DISTROS = {
'debian9', 'debian9',
'debian10', 'debian10',
'ubuntu18.04', 'ubuntu18.04',
'ubuntu19.04', 'ubuntu19.04',
} }
INSTALL_PYTHONS = r''' INSTALL_PYTHONS = r'''
PYENV2_VERSIONS="2.7.17 pypy2.7-7.2.0"
PYENV3_VERSIONS="3.5.10 3.6.13 3.7.10 3.8.10 3.9.5 pypy3.5-7.0.0 pypy3.6-7.3.3 pypy3.7-7.3.3" PYENV3_VERSIONS="3.5.10 3.6.13 3.7.10 3.8.10 3.9.5 pypy3.5-7.0.0 pypy3.6-7.3.3 pypy3.7-7.3.3"
git clone https://github.com/pyenv/pyenv.git /hgdev/pyenv git clone https://github.com/pyenv/pyenv.git /hgdev/pyenv
pushd /hgdev/pyenv pushd /hgdev/pyenv
git checkout 328fd42c3a2fbf14ae46dae2021a087fe27ba7e2 git checkout 328fd42c3a2fbf14ae46dae2021a087fe27ba7e2
popd popd
export PYENV_ROOT="/hgdev/pyenv" export PYENV_ROOT="/hgdev/pyenv"
export PATH="$PYENV_ROOT/bin:$PATH" export PATH="$PYENV_ROOT/bin:$PATH"
# pip 19.2.3. # pip 19.2.3.
PIP_SHA256=57e3643ff19f018f8a00dfaa6b7e4620e3c1a7a2171fd218425366ec006b3bfe PIP_SHA256=57e3643ff19f018f8a00dfaa6b7e4620e3c1a7a2171fd218425366ec006b3bfe
wget -O get-pip.py --progress dot:mega https://github.com/pypa/get-pip/raw/309a56c5fd94bd1134053a541cb4657a4e47e09d/get-pip.py wget -O get-pip.py --progress dot:mega https://github.com/pypa/get-pip/raw/309a56c5fd94bd1134053a541cb4657a4e47e09d/get-pip.py
echo "${PIP_SHA256} get-pip.py" | sha256sum --check - echo "${PIP_SHA256} get-pip.py" | sha256sum --check -
VIRTUALENV_SHA256=f78d81b62d3147396ac33fc9d77579ddc42cc2a98dd9ea38886f616b33bc7fb2 VIRTUALENV_SHA256=f78d81b62d3147396ac33fc9d77579ddc42cc2a98dd9ea38886f616b33bc7fb2
VIRTUALENV_TARBALL=virtualenv-16.7.5.tar.gz VIRTUALENV_TARBALL=virtualenv-16.7.5.tar.gz
wget -O ${VIRTUALENV_TARBALL} --progress dot:mega https://files.pythonhosted.org/packages/66/f0/6867af06d2e2f511e4e1d7094ff663acdebc4f15d4a0cb0fed1007395124/${VIRTUALENV_TARBALL} wget -O ${VIRTUALENV_TARBALL} --progress dot:mega https://files.pythonhosted.org/packages/66/f0/6867af06d2e2f511e4e1d7094ff663acdebc4f15d4a0cb0fed1007395124/${VIRTUALENV_TARBALL}
echo "${VIRTUALENV_SHA256} ${VIRTUALENV_TARBALL}" | sha256sum --check - echo "${VIRTUALENV_SHA256} ${VIRTUALENV_TARBALL}" | sha256sum --check -
for v in ${PYENV2_VERSIONS}; do
pyenv install -v ${v}
${PYENV_ROOT}/versions/${v}/bin/python get-pip.py
${PYENV_ROOT}/versions/${v}/bin/pip install ${VIRTUALENV_TARBALL}
${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/requirements-py2.txt
done
for v in ${PYENV3_VERSIONS}; do for v in ${PYENV3_VERSIONS}; do
pyenv install -v ${v} pyenv install -v ${v}
${PYENV_ROOT}/versions/${v}/bin/python get-pip.py ${PYENV_ROOT}/versions/${v}/bin/python get-pip.py
case ${v} in case ${v} in
3.5.*) 3.5.*)
REQUIREMENTS=requirements-py3.5.txt REQUIREMENTS=requirements-py3.5.txt
;; ;;
pypy3.5*) pypy3.5*)
REQUIREMENTS=requirements-py3.5.txt REQUIREMENTS=requirements-py3.5.txt
;; ;;
*) *)
REQUIREMENTS=requirements-py3.txt REQUIREMENTS=requirements-py3.txt
;; ;;
esac esac
${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/${REQUIREMENTS} ${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/${REQUIREMENTS}
done done
pyenv global ${PYENV2_VERSIONS} ${PYENV3_VERSIONS} system pyenv global ${PYENV3_VERSIONS} system
'''.lstrip().replace( '''.lstrip().replace(
'\r\n', '\n' '\r\n', '\n'
) )
INSTALL_PYOXIDIZER = r''' INSTALL_PYOXIDIZER = r'''
PYOXIDIZER_VERSION=0.16.0 PYOXIDIZER_VERSION=0.16.0
PYOXIDIZER_SHA256=8875471c270312fbb934007fd30f65f1904cc0f5da6188d61c90ed2129b9f9c1 PYOXIDIZER_SHA256=8875471c270312fbb934007fd30f65f1904cc0f5da6188d61c90ed2129b9f9c1
PYOXIDIZER_URL=https://github.com/indygreg/PyOxidizer/releases/download/pyoxidizer%2F${PYOXIDIZER_VERSION}/pyoxidizer-${PYOXIDIZER_VERSION}-linux_x86_64.zip PYOXIDIZER_URL=https://github.com/indygreg/PyOxidizer/releases/download/pyoxidizer%2F${PYOXIDIZER_VERSION}/pyoxidizer-${PYOXIDIZER_VERSION}-linux_x86_64.zip
libncurses5-dev \ libncurses5-dev \
libnss3-dev \ libnss3-dev \
libreadline-dev \ libreadline-dev \
libsqlite3-dev \ libsqlite3-dev \
libssl-dev \ libssl-dev \
netbase \ netbase \
ntfs-3g \ ntfs-3g \
nvme-cli \ nvme-cli \
pyflakes \
pyflakes3 \ pyflakes3 \
pylint \
pylint3 \ pylint3 \
python-all-dev \
python-dev \
python-docutils \
python-fuzzywuzzy \
python-pygments \
python-subversion \
python-vcr \
python3-boto3 \ python3-boto3 \
python3-dev \ python3-dev \
python3-docutils \ python3-docutils \
python3-fuzzywuzzy \ python3-fuzzywuzzy \
python3-pygments \ python3-pygments \
python3-vcr \ python3-vcr \
python3-venv \ python3-venv \
rsync \ rsync \
def run_tests(ssh_client, python_version, test_flags=None): def run_tests(ssh_client, python_version, test_flags=None):
"""Run tests on a remote Linux machine via an SSH client.""" """Run tests on a remote Linux machine via an SSH client."""
test_flags = test_flags or [] test_flags = test_flags or []
print('running tests') print('running tests')
if python_version == 'system2': if python_version == 'system3':
python = '/usr/bin/python2'
elif python_version == 'system3':
python = '/usr/bin/python3' python = '/usr/bin/python3'
elif python_version.startswith('pypy'): elif python_version.startswith('pypy'):
python = '/hgdev/pyenv/shims/%s' % python_version python = '/hgdev/pyenv/shims/%s' % python_version
else: else:
python = '/hgdev/pyenv/shims/python%s' % python_version python = '/hgdev/pyenv/shims/python%s' % python_version
test_flags = ' '.join(shlex.quote(a) for a in test_flags) test_flags = ' '.join(shlex.quote(a) for a in test_flags)

contrib/automation/linux-requirements-py2.txt

  • This file was deleted.
#
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --generate-hashes --output-file=contrib/automation/linux-requirements-py2.txt contrib/automation/linux-requirements.txt.in
#
astroid==1.6.6 \
--hash=sha256:87de48a92e29cedf7210ffa853d11441e7ad94cb47bacd91b023499b51cbc756 \
--hash=sha256:d25869fc7f44f1d9fb7d24fd7ea0639656f5355fc3089cd1f3d18c6ec6b124c7 \
# via pylint
backports.functools-lru-cache==1.6.1 \
--hash=sha256:0bada4c2f8a43d533e4ecb7a12214d9420e66eb206d54bf2d682581ca4b80848 \
--hash=sha256:8fde5f188da2d593bd5bc0be98d9abc46c95bb8a9dde93429570192ee6cc2d4a \
# via astroid, isort, pylint
bzr==2.7.0 ; python_version <= "2.7" and platform_python_implementation == "CPython" \
--hash=sha256:c9f6bbe0a50201dadc5fddadd94ba50174193c6cf6e39e16f6dd0ad98a1df338 \
# via -r contrib/automation/linux-requirements.txt.in
configparser==4.0.2 \
--hash=sha256:254c1d9c79f60c45dfde850850883d5aaa7f19a23f13561243a050d5a7c3fe4c \
--hash=sha256:c7d282687a5308319bf3d2e7706e575c635b0a470342641c93bea0ea3b5331df \
# via pylint
contextlib2==0.6.0.post1 \
--hash=sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e \
--hash=sha256:3355078a159fbb44ee60ea80abd0d87b80b78c248643b49aa6d94673b413609b \
# via vcrpy
docutils==0.16 \
--hash=sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af \
--hash=sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc \
# via -r contrib/automation/linux-requirements.txt.in
enum34==1.1.10 \
--hash=sha256:a98a201d6de3f2ab3db284e70a33b0f896fbf35f8086594e8c9e74b909058d53 \
--hash=sha256:c3858660960c984d6ab0ebad691265180da2b43f07e061c0f8dca9ef3cffd328 \
--hash=sha256:cce6a7477ed816bd2542d03d53db9f0db935dd013b70f336a95c73979289f248 \
# via astroid
funcsigs==1.0.2 \
--hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \
--hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 \
# via mock
futures==3.3.0 \
--hash=sha256:49b3f5b064b6e3afc3316421a3f25f66c137ae88f068abbf72830170033c5e16 \
--hash=sha256:7e033af76a5e35f58e56da7a91e687706faf4e7bdfb2cbc3f2cca6b9bcda9794 \
# via isort
fuzzywuzzy==0.18.0 \
--hash=sha256:45016e92264780e58972dca1b3d939ac864b78437422beecebb3095f8efd00e8 \
--hash=sha256:928244b28db720d1e0ee7587acf660ea49d7e4c632569cad4f1cd7e68a5f0993 \
# via -r contrib/automation/linux-requirements.txt.in
isort==4.3.21 \
--hash=sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1 \
--hash=sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd \
# via pylint
lazy-object-proxy==1.5.1 \
--hash=sha256:00b78a97a79d0dfefa584d44dd1aba9668d3de7ec82335ba0ff51d53ef107143 \
--hash=sha256:042b54fd71c2092e6d10e5e66fa60f65c5954f8145e809f5d9f394c9b13d32ee \
--hash=sha256:11f87dc06eb5f376cc6d5f0c19a1b4dca202035622777c4ce8e5b72c87b035d6 \
--hash=sha256:19ae6f6511a02008ef3554e158c41bb2a8e5c8455935b98d6da076d9f152fd7c \
--hash=sha256:22c1935c6f8e3d6ea2e169eb03928adbdb8a2251d2890f8689368d65e70aa176 \
--hash=sha256:30ef2068f4f94660144515380ef04b93d15add2214eab8be4cd46ebc900d681c \
--hash=sha256:33da47ba3a581860ddd3d38c950a5fe950ca389f7123edd0d6ab0bc473499fe7 \
--hash=sha256:3e8698dc384857413580012f4ca322d89e63ef20fc3d4635a5b606d6d4b61f6a \
--hash=sha256:4fdd7113fc5143c72dacf415079eec42fcbe69cc9d3d291b4ca742e3a9455807 \
--hash=sha256:63b6d9a5077d54db271fcc6772440f7380ec3fa559d0e2497dbfae2f47c2c814 \
--hash=sha256:8133b63b05f12751cddd8e3e7f02ba39dc7cfa7d2ba99d80d7436f0ba26d6b75 \
--hash=sha256:89b8e5780e49753e2b4cd5aab45d3df092ddcbba3de2c4d4492a029588fe1758 \
--hash=sha256:8d82e27cbbea6edb8821751806f39f5dcfd7b46a5e23d27b98d6d8c8ec751df8 \
--hash=sha256:92cedd6e26712505adb1c17fab64651a498cc0102a80ba562ff4a2451088f57a \
--hash=sha256:9723364577b79ad9958a68851fe2acb94da6fd25170c595516a8289e6a129043 \
--hash=sha256:c484020ad26973a14a7cb1e1d2e0bfe97cf6803273ae9bd154e0213cc74bad49 \
--hash=sha256:c697bd1b333b3e6abdff04ef9f5fb4b1936633d9cc4e28d90606705c9083254c \
--hash=sha256:d0f7e14ff3424639d33e6bc449e77e4b345e52c21bbd6f6004a1d219196e2664 \
--hash=sha256:db2df3eff7ed3e6813638686f1bb5934d1a0662d9d3b4196b5164a86be3a1e8f \
--hash=sha256:edbcb4c5efabd93ede05b272296a5a78a67e9b6e82ba7f51a07b8103db06ce01 \
--hash=sha256:ef355fb3802e0fc5a71dadb65a3c317bfc9bdf567d357f8e0b1900b432ffe486 \
--hash=sha256:fe2f61fed5817bf8db01d9a72309ed5990c478a077e9585b58740c26774bce39 \
# via astroid
mccabe==0.6.1 \
--hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
--hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f \
# via pylint
mock==3.0.5 \
--hash=sha256:83657d894c90d5681d62155c82bda9c1187827525880eda8ff5df4ec813437c3 \
--hash=sha256:d157e52d4e5b938c550f39eb2fd15610db062441a9c2747d3dbfa9298211d0f8 \
# via vcrpy
pyflakes==2.2.0 \
--hash=sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92 \
--hash=sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8 \
# via -r contrib/automation/linux-requirements.txt.in
pygments==2.5.2 \
--hash=sha256:2a3fe295e54a20164a9df49c75fa58526d3be48e14aceba6d6b1e8ac0bfd6f1b \
--hash=sha256:98c8aa5a9f778fcd1026a17361ddaf7330d1b7c62ae97c3bb0ae73e0b9b6b0fe \
# via -r contrib/automation/linux-requirements.txt.in
pylint==1.9.5 \
--hash=sha256:367e3d49813d349a905390ac27989eff82ab84958731c5ef0bef867452cfdc42 \
--hash=sha256:97a42df23d436c70132971d1dcb9efad2fe5c0c6add55b90161e773caf729300 \
# via -r contrib/automation/linux-requirements.txt.in
python-levenshtein==0.12.0 \
--hash=sha256:033a11de5e3d19ea25c9302d11224e1a1898fe5abd23c61c7c360c25195e3eb1 \
# via -r contrib/automation/linux-requirements.txt.in
pyyaml==5.3.1 \
--hash=sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97 \
--hash=sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76 \
--hash=sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2 \
--hash=sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648 \
--hash=sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf \
--hash=sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f \
--hash=sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2 \
--hash=sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee \
--hash=sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d \
--hash=sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c \
--hash=sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a \
# via vcrpy
singledispatch==3.4.0.3 \
--hash=sha256:5b06af87df13818d14f08a028e42f566640aef80805c3b50c5056b086e3c2b9c \
--hash=sha256:833b46966687b3de7f438c761ac475213e53b306740f1abfaa86e1d1aae56aa8 \
# via astroid, pylint
six==1.15.0 \
--hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
--hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
# via astroid, mock, pylint, singledispatch, vcrpy
vcrpy==3.0.0 \
--hash=sha256:21168d5ae14263a833d4b71acfd8278d8841114f24be1b4ab4a5719d0c7f07bc \
--hash=sha256:a2e6b653a627f9f3d6ded4d68587e470b91e4c1444e7dae939510dfeacb65276 \
# via -r contrib/automation/linux-requirements.txt.in
wrapt==1.12.1 \
--hash=sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7 \
# via astroid, vcrpy
# WARNING: The following packages were not pinned, but pip requires them to be
# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.
# setuptools
contrib/automation/hgautomation/linux.py