This is an archive of the discontinued Mercurial Phabricator instance.

Differential D6673

automation: make Windows base image name configurable
ClosedPublic

Authored by indygreg on Jul 22 2019, 10:21 PM.

Details

Reviewers
pulkit
Group Reviewers
hg-reviewers
Commits
rHGd80edcb0b30c: automation: make Windows base image name configurable
Summary

Since automation broke in the middle of the 5.0 release cycle,
there's a good chance it will break again in the future. While
a robust solution might be to search for all available images and
choose the newest one, it does seem useful to be able to explicitly
choose the name of the image to find and use so users can opt in
to using a different image.

This commit implements that functionality.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

indygreg created this revision.Jul 22 2019, 10:21 PM
pulkit accepted this revision.Jul 24 2019, 5:12 AM
This revision is now accepted and ready to land.Jul 24 2019, 5:12 AM
indygreg added a commit: rHGd80edcb0b30c: automation: make Windows base image name configurable.Jul 24 2019, 5:27 AM
Closed by commit rHGd80edcb0b30c: automation: make Windows base image name configurable (authored by indygreg). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 16030

contrib/automation/hgautomation/aws.py

try: try:
yield instances yield instances
finally: finally:
for instance in instances: for instance in instances:
instance.ssh_client.close() instance.ssh_client.close()
def ensure_windows_dev_ami(c: AWSConnection, prefix='hg-'): def ensure_windows_dev_ami(c: AWSConnection, prefix='hg-',
base_image_name=WINDOWS_BASE_IMAGE_NAME):
"""Ensure Windows Development AMI is available and up-to-date. """Ensure Windows Development AMI is available and up-to-date.
If necessary, a modern AMI will be built by starting a temporary EC2 If necessary, a modern AMI will be built by starting a temporary EC2
instance and bootstrapping it. instance and bootstrapping it.
Obsolete AMIs will be deleted so there is only a single AMI having the Obsolete AMIs will be deleted so there is only a single AMI having the
desired name. desired name.
Returns an ``ec2.Image`` of either an existing AMI or a newly-built Returns an ``ec2.Image`` of either an existing AMI or a newly-built
one. one.
""" """
ec2client = c.ec2client ec2client = c.ec2client
ec2resource = c.ec2resource ec2resource = c.ec2resource
ssmclient = c.session.client('ssm') ssmclient = c.session.client('ssm')
name = '%s%s' % (prefix, 'windows-dev') name = '%s%s' % (prefix, 'windows-dev')
image = find_image(ec2resource, AMAZON_ACCOUNT_ID, WINDOWS_BASE_IMAGE_NAME) image = find_image(ec2resource, AMAZON_ACCOUNT_ID, base_image_name)
config = { config = {
'BlockDeviceMappings': [ 'BlockDeviceMappings': [
{ {
'DeviceName': '/dev/sda1', 'DeviceName': '/dev/sda1',
'Ebs': { 'Ebs': {
'DeleteOnTermination': True, 'DeleteOnTermination': True,
'VolumeSize': 32, 'VolumeSize': 32,
# Compute a deterministic fingerprint to determine whether image needs # Compute a deterministic fingerprint to determine whether image needs
# to be regenerated. # to be regenerated.
fingerprint = resolve_fingerprint({ fingerprint = resolve_fingerprint({
'instance_config': config, 'instance_config': config,
'user_data': WINDOWS_USER_DATA, 'user_data': WINDOWS_USER_DATA,
'initial_bootstrap': WINDOWS_BOOTSTRAP_POWERSHELL, 'initial_bootstrap': WINDOWS_BOOTSTRAP_POWERSHELL,
'bootstrap_commands': commands, 'bootstrap_commands': commands,
'base_image_name': base_image_name,
}) })
existing_image = find_and_reconcile_image(ec2resource, name, fingerprint) existing_image = find_and_reconcile_image(ec2resource, name, fingerprint)
if existing_image: if existing_image:
return existing_image return existing_image
print('no suitable Windows development image found; creating one...') print('no suitable Windows development image found; creating one...')

contrib/automation/hgautomation/cli.py

for f in fs: for f in fs:
f.result() f.result()
else: else:
for distro in distros: for distro in distros:
aws.ensure_linux_dev_ami(c, distro=distro) aws.ensure_linux_dev_ami(c, distro=distro)
def bootstrap_windows_dev(hga: HGAutomation, aws_region): def bootstrap_windows_dev(hga: HGAutomation, aws_region, base_image_name):
c = hga.aws_connection(aws_region) c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
print('Windows development AMI available as %s' % image.id) print('Windows development AMI available as %s' % image.id)
def build_inno(hga: HGAutomation, aws_region, arch, revision, version): def build_inno(hga: HGAutomation, aws_region, arch, revision, version,
base_image_name):
c = hga.aws_connection(aws_region) c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
DIST_PATH.mkdir(exist_ok=True) DIST_PATH.mkdir(exist_ok=True)
with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
instance = insts[0] instance = insts[0]
windows.synchronize_hg(SOURCE_ROOT, revision, instance) windows.synchronize_hg(SOURCE_ROOT, revision, instance)
for a in arch: for a in arch:
windows.build_inno_installer(instance.winrm_client, a, windows.build_inno_installer(instance.winrm_client, a,
DIST_PATH, DIST_PATH,
version=version) version=version)
def build_wix(hga: HGAutomation, aws_region, arch, revision, version): def build_wix(hga: HGAutomation, aws_region, arch, revision, version,
base_image_name):
c = hga.aws_connection(aws_region) c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
DIST_PATH.mkdir(exist_ok=True) DIST_PATH.mkdir(exist_ok=True)
with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
instance = insts[0] instance = insts[0]
windows.synchronize_hg(SOURCE_ROOT, revision, instance) windows.synchronize_hg(SOURCE_ROOT, revision, instance)
for a in arch: for a in arch:
windows.build_wix_installer(instance.winrm_client, a, windows.build_wix_installer(instance.winrm_client, a,
DIST_PATH, version=version) DIST_PATH, version=version)
def build_windows_wheel(hga: HGAutomation, aws_region, arch, revision): def build_windows_wheel(hga: HGAutomation, aws_region, arch, revision,
base_image_name):
c = hga.aws_connection(aws_region) c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
DIST_PATH.mkdir(exist_ok=True) DIST_PATH.mkdir(exist_ok=True)
with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
instance = insts[0] instance = insts[0]
windows.synchronize_hg(SOURCE_ROOT, revision, instance) windows.synchronize_hg(SOURCE_ROOT, revision, instance)
for a in arch: for a in arch:
windows.build_wheel(instance.winrm_client, a, DIST_PATH) windows.build_wheel(instance.winrm_client, a, DIST_PATH)
def build_all_windows_packages(hga: HGAutomation, aws_region, revision, def build_all_windows_packages(hga: HGAutomation, aws_region, revision,
version): version, base_image_name):
c = hga.aws_connection(aws_region) c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
DIST_PATH.mkdir(exist_ok=True) DIST_PATH.mkdir(exist_ok=True)
with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
instance = insts[0] instance = insts[0]
winrm_client = instance.winrm_client winrm_client = instance.winrm_client
windows.synchronize_hg(SOURCE_ROOT, revision, instance) windows.synchronize_hg(SOURCE_ROOT, revision, instance)
t_all = t_done - t_start t_all = t_done - t_start
print( print(
'total time: %.1fs; setup: %.1fs; tests: %.1fs; setup overhead: %.1f%%' 'total time: %.1fs; setup: %.1fs; tests: %.1fs; setup overhead: %.1f%%'
% (t_all, t_setup, t_done - t_prepared, t_setup / t_all * 100.0)) % (t_all, t_setup, t_done - t_prepared, t_setup / t_all * 100.0))
def run_tests_windows(hga: HGAutomation, aws_region, instance_type, def run_tests_windows(hga: HGAutomation, aws_region, instance_type,
python_version, arch, test_flags): python_version, arch, test_flags, base_image_name):
c = hga.aws_connection(aws_region) c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
with aws.temporary_windows_dev_instances(c, image, instance_type, with aws.temporary_windows_dev_instances(c, image, instance_type,
disable_antivirus=True) as insts: disable_antivirus=True) as insts:
instance = insts[0] instance = insts[0]
windows.synchronize_hg(SOURCE_ROOT, '.', instance) windows.synchronize_hg(SOURCE_ROOT, '.', instance)
windows.run_tests(instance.winrm_client, python_version, arch, windows.run_tests(instance.winrm_client, python_version, arch,
test_flags) test_flags)
help='Generate AMIs in parallel (not CTRL-c safe)' help='Generate AMIs in parallel (not CTRL-c safe)'
) )
sp.set_defaults(func=bootstrap_linux_dev) sp.set_defaults(func=bootstrap_linux_dev)
sp = subparsers.add_parser( sp = subparsers.add_parser(
'bootstrap-windows-dev', 'bootstrap-windows-dev',
help='Bootstrap the Windows development environment', help='Bootstrap the Windows development environment',
) )
sp.add_argument(
'--base-image-name',
help='AMI name of base image',
default=aws.WINDOWS_BASE_IMAGE_NAME,
)
sp.set_defaults(func=bootstrap_windows_dev) sp.set_defaults(func=bootstrap_windows_dev)
sp = subparsers.add_parser( sp = subparsers.add_parser(
'build-all-windows-packages', 'build-all-windows-packages',
help='Build all Windows packages', help='Build all Windows packages',
) )
sp.add_argument( sp.add_argument(
'--revision', '--revision',
help='Mercurial revision to build', help='Mercurial revision to build',
default='.', default='.',
) )
sp.add_argument( sp.add_argument(
'--version', '--version',
help='Mercurial version string to use', help='Mercurial version string to use',
) )
sp.add_argument(
'--base-image-name',
help='AMI name of base image',
default=aws.WINDOWS_BASE_IMAGE_NAME,
)
sp.set_defaults(func=build_all_windows_packages) sp.set_defaults(func=build_all_windows_packages)
sp = subparsers.add_parser( sp = subparsers.add_parser(
'build-inno', 'build-inno',
help='Build Inno Setup installer(s)', help='Build Inno Setup installer(s)',
) )
sp.add_argument( sp.add_argument(
'--arch', '--arch',
help='Architecture to build for', help='Architecture to build for',
choices={'x86', 'x64'}, choices={'x86', 'x64'},
nargs='*', nargs='*',
default=['x64'], default=['x64'],
) )
sp.add_argument( sp.add_argument(
'--revision', '--revision',
help='Mercurial revision to build', help='Mercurial revision to build',
default='.', default='.',
) )
sp.add_argument( sp.add_argument(
'--version', '--version',
help='Mercurial version string to use in installer', help='Mercurial version string to use in installer',
) )
sp.add_argument(
'--base-image-name',
help='AMI name of base image',
default=aws.WINDOWS_BASE_IMAGE_NAME,
)
sp.set_defaults(func=build_inno) sp.set_defaults(func=build_inno)
sp = subparsers.add_parser( sp = subparsers.add_parser(
'build-windows-wheel', 'build-windows-wheel',
help='Build Windows wheel(s)', help='Build Windows wheel(s)',
) )
sp.add_argument( sp.add_argument(
'--arch', '--arch',
help='Architecture to build for', help='Architecture to build for',
choices={'x86', 'x64'}, choices={'x86', 'x64'},
nargs='*', nargs='*',
default=['x64'], default=['x64'],
) )
sp.add_argument( sp.add_argument(
'--revision', '--revision',
help='Mercurial revision to build', help='Mercurial revision to build',
default='.', default='.',
) )
sp.add_argument(
'--base-image-name',
help='AMI name of base image',
default=aws.WINDOWS_BASE_IMAGE_NAME,
)
sp.set_defaults(func=build_windows_wheel) sp.set_defaults(func=build_windows_wheel)
sp = subparsers.add_parser( sp = subparsers.add_parser(
'build-wix', 'build-wix',
help='Build WiX installer(s)' help='Build WiX installer(s)'
) )
sp.add_argument( sp.add_argument(
'--arch', '--arch',
help='Architecture to build for', help='Architecture to build for',
choices={'x86', 'x64'}, choices={'x86', 'x64'},
nargs='*', nargs='*',
default=['x64'], default=['x64'],
) )
sp.add_argument( sp.add_argument(
'--revision', '--revision',
help='Mercurial revision to build', help='Mercurial revision to build',
default='.', default='.',
) )
sp.add_argument( sp.add_argument(
'--version', '--version',
help='Mercurial version string to use in installer', help='Mercurial version string to use in installer',
) )
sp.add_argument(
'--base-image-name',
help='AMI name of base image',
default=aws.WINDOWS_BASE_IMAGE_NAME,
)
sp.set_defaults(func=build_wix) sp.set_defaults(func=build_wix)
sp = subparsers.add_parser( sp = subparsers.add_parser(
'terminate-ec2-instances', 'terminate-ec2-instances',
help='Terminate all active EC2 instances managed by us', help='Terminate all active EC2 instances managed by us',
) )
sp.set_defaults(func=terminate_ec2_instances) sp.set_defaults(func=terminate_ec2_instances)
help='Architecture to test', help='Architecture to test',
choices={'x86', 'x64'}, choices={'x86', 'x64'},
default='x64', default='x64',
) )
sp.add_argument( sp.add_argument(
'--test-flags', '--test-flags',
help='Extra command line flags to pass to run-tests.py', help='Extra command line flags to pass to run-tests.py',
) )
sp.add_argument(
'--base-image-name',
help='AMI name of base image',
default=aws.WINDOWS_BASE_IMAGE_NAME,
)
sp.set_defaults(func=run_tests_windows) sp.set_defaults(func=run_tests_windows)
return parser return parser
def main(): def main():
parser = get_parser() parser = get_parser()
args = parser.parse_args() args = parser.parse_args()