This is an archive of the discontinued Mercurial Phabricator instance.

Differential D6917

automation: support and use Debian Buster by default
ClosedPublic

Authored by indygreg on Sep 29 2019, 3:08 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHGd1d919f679f7: automation: support and use Debian Buster by default
Summary

While Debian Buster (Debian 10) was released in July, the
AWS AMIs were not published until mid September.

This commit teaches the automation system to create AMIs for
Debian Buster. Since Debian Buster is the new stable Debian
release, we make it the default distribution for automation.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

indygreg created this revision.Sep 29 2019, 3:08 PM
indygreg added a commit: rHGd1d919f679f7: automation: support and use Debian Buster by default.Sep 30 2019, 5:06 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHGd1d919f679f7: automation: support and use Debian Buster by default (authored by indygreg). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 16690

contrib/automation/hgautomation/aws.py

'r5ad', 'r5ad',
'x1', 'x1',
'z1d', 'z1d',
} }
AMAZON_ACCOUNT_ID = '801119661308' AMAZON_ACCOUNT_ID = '801119661308'
DEBIAN_ACCOUNT_ID = '379101102735' DEBIAN_ACCOUNT_ID = '379101102735'
DEBIAN_ACCOUNT_ID_2 = '136693071363'
UBUNTU_ACCOUNT_ID = '099720109477' UBUNTU_ACCOUNT_ID = '099720109477'
WINDOWS_BASE_IMAGE_NAME = 'Windows_Server-2019-English-Full-Base-2019.07.12' WINDOWS_BASE_IMAGE_NAME = 'Windows_Server-2019-English-Full-Base-2019.07.12'
KEY_PAIRS = { KEY_PAIRS = {
'automation', 'automation',
ImageIds=[image.id], ImageIds=[image.id],
) )
print('image %s available as %s' % (image.id, image.name)) print('image %s available as %s' % (image.id, image.name))
return image return image
def ensure_linux_dev_ami(c: AWSConnection, distro='debian9', prefix='hg-'): def ensure_linux_dev_ami(c: AWSConnection, distro='debian10', prefix='hg-'):
"""Ensures a Linux development AMI is available and up-to-date. """Ensures a Linux development AMI is available and up-to-date.
Returns an ``ec2.Image`` of either an existing AMI or a newly-built one. Returns an ``ec2.Image`` of either an existing AMI or a newly-built one.
""" """
ec2client = c.ec2client ec2client = c.ec2client
ec2resource = c.ec2resource ec2resource = c.ec2resource
name = '%s%s-%s' % (prefix, 'linux-dev', distro) name = '%s%s-%s' % (prefix, 'linux-dev', distro)
if distro == 'debian9': if distro == 'debian9':
image = find_image( image = find_image(
ec2resource, ec2resource,
DEBIAN_ACCOUNT_ID, DEBIAN_ACCOUNT_ID,
'debian-stretch-hvm-x86_64-gp2-2019-09-08-17994', 'debian-stretch-hvm-x86_64-gp2-2019-09-08-17994',
) )
ssh_username = 'admin' ssh_username = 'admin'
elif distro == 'debian10':
image = find_image(
ec2resource,
DEBIAN_ACCOUNT_ID_2,
'debian-10-amd64-20190909-10',
)
ssh_username = 'admin'
elif distro == 'ubuntu18.04': elif distro == 'ubuntu18.04':
image = find_image( image = find_image(
ec2resource, ec2resource,
UBUNTU_ACCOUNT_ID, UBUNTU_ACCOUNT_ID,
'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190918', 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190918',
) )
ssh_username = 'ubuntu' ssh_username = 'ubuntu'
elif distro == 'ubuntu19.04': elif distro == 'ubuntu19.04':

contrib/automation/hgautomation/cli.py

sp = subparsers.add_parser( sp = subparsers.add_parser(
'run-tests-linux', 'run-tests-linux',
help='Run tests on Linux', help='Run tests on Linux',
) )
sp.add_argument( sp.add_argument(
'--distro', '--distro',
help='Linux distribution to run tests on', help='Linux distribution to run tests on',
choices=linux.DISTROS, choices=linux.DISTROS,
default='debian9', default='debian10',
) )
sp.add_argument( sp.add_argument(
'--filesystem', '--filesystem',
help='Filesystem type to use', help='Filesystem type to use',
choices={'btrfs', 'default', 'ext3', 'ext4', 'jfs', 'tmpfs', 'xfs'}, choices={'btrfs', 'default', 'ext3', 'ext4', 'jfs', 'tmpfs', 'xfs'},
default='default', default='default',
) )
sp.add_argument( sp.add_argument(

contrib/automation/hgautomation/linux.py

from .ssh import ( from .ssh import (
exec_command, exec_command,
) )
# Linux distributions that are supported. # Linux distributions that are supported.
DISTROS = { DISTROS = {
'debian9', 'debian9',
'debian10',
'ubuntu18.04', 'ubuntu18.04',
'ubuntu19.04', 'ubuntu19.04',
} }
INSTALL_PYTHONS = r''' INSTALL_PYTHONS = r'''
PYENV2_VERSIONS="2.7.16 pypy2.7-7.1.1" PYENV2_VERSIONS="2.7.16 pypy2.7-7.1.1"
PYENV3_VERSIONS="3.5.7 3.6.9 3.7.4 3.8-dev pypy3.5-7.0.0 pypy3.6-7.1.1" PYENV3_VERSIONS="3.5.7 3.6.9 3.7.4 3.8-dev pypy3.5-7.0.0 pypy3.6-7.1.1"
EOF EOF
sudo apt-key add docker-apt-key sudo apt-key add docker-apt-key
if [ "$LSB_RELEASE" = "stretch" ]; then if [ "$LSB_RELEASE" = "stretch" ]; then
cat << EOF | sudo tee -a /etc/apt/sources.list cat << EOF | sudo tee -a /etc/apt/sources.list
# Need backports for clang-format-6.0 # Need backports for clang-format-6.0
deb http://deb.debian.org/debian stretch-backports main deb http://deb.debian.org/debian stretch-backports main
EOF
fi
if [ "$LSB_RELEASE" = "stretch" -o "$LSB_RELEASE" = "buster" ]; then
cat << EOF | sudo tee -a /etc/apt/sources.list
# Sources are useful if we want to compile things locally. # Sources are useful if we want to compile things locally.
deb-src http://deb.debian.org/debian stretch main deb-src http://deb.debian.org/debian $LSB_RELEASE main
deb-src http://security.debian.org/debian-security stretch/updates main deb-src http://security.debian.org/debian-security $LSB_RELEASE/updates main
deb-src http://deb.debian.org/debian stretch-updates main deb-src http://deb.debian.org/debian $LSB_RELEASE-updates main
deb-src http://deb.debian.org/debian stretch-backports main deb-src http://deb.debian.org/debian $LSB_RELEASE-backports main
deb [arch=amd64] https://download.docker.com/linux/debian stretch stable deb [arch=amd64] https://download.docker.com/linux/debian $LSB_RELEASE stable
EOF EOF
elif [ "$DISTRO" = "Ubuntu" ]; then elif [ "$DISTRO" = "Ubuntu" ]; then
cat << EOF | sudo tee -a /etc/apt/sources.list cat << EOF | sudo tee -a /etc/apt/sources.list
deb [arch=amd64] https://download.docker.com/linux/ubuntu $LSB_RELEASE stable deb [arch=amd64] https://download.docker.com/linux/ubuntu $LSB_RELEASE stable
EOF EOF
fi fi
zlib1g-dev" zlib1g-dev"
if [ "LSB_RELEASE" = "stretch" ]; then if [ "LSB_RELEASE" = "stretch" ]; then
PACKAGES="$PACKAGES linux-perf" PACKAGES="$PACKAGES linux-perf"
elif [ "$DISTRO" = "Ubuntu" ]; then elif [ "$DISTRO" = "Ubuntu" ]; then
PACKAGES="$PACKAGES linux-tools-common" PACKAGES="$PACKAGES linux-tools-common"
fi fi
# Ubuntu 19.04 removes monotone. # Monotone only available in older releases.
if [ "$LSB_RELEASE" != "disco" ]; then if [ "$LSB_RELEASE" = "stretch" -o "$LSB_RELEASE" = "xenial" ]; then
PACKAGES="$PACKAGES monotone" PACKAGES="$PACKAGES monotone"
fi fi
sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install --no-install-recommends $PACKAGES sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install --no-install-recommends $PACKAGES
# Create clang-format symlink so test harness finds it. # Create clang-format symlink so test harness finds it.
sudo update-alternatives --install /usr/bin/clang-format clang-format \ sudo update-alternatives --install /usr/bin/clang-format clang-format \
/usr/bin/clang-format-6.0 1000 /usr/bin/clang-format-6.0 1000