diff --git a/contrib/automation/README.rst b/contrib/automation/README.rst new file mode 100644 --- /dev/null +++ b/contrib/automation/README.rst @@ -0,0 +1,127 @@ +==================== +Mercurial Automation +==================== + +This directory contains code and utilities for building and testing Mercurial +on remote machines. + +The ``automation.py`` Script +============================ + +``automation.py`` is an executable Python script (requires Python 3.5+) +that serves as a driver to common automation tasks. + +When executed, the script will *bootstrap* a virtualenv in +``/build/venv-automation`` then re-execute itself using +that virtualenv. So there is no need for the caller to have a virtualenv +explicitly activated. This virtualenv will be populated with various +dependencies (as defined by the ``requirements.txt`` file). + +To see what you can do with this script, simply run it:: + + $ ./automation.py + +Local State +=========== + +By default, local state required to interact with remote servers is stored +in the ``~/.hgautomation`` directory. + +We attempt to limit persistent state to this directory. Even when +performing tasks that may have side-effects, we try to limit those +side-effects so they don't impact the local system. e.g. when we SSH +into a remote machine, we create a temporary directory for the SSH +config so the user's known hosts file isn't updated. + +AWS Integration +=============== + +Various automation tasks integrate with AWS to provide access to +resources such as EC2 instances for generic compute. + +This obviously requires an AWS account and credentials to work. + +We use the ``boto3`` library for interacting with AWS APIs. We do not employ +any special functionality for telling ``boto3`` where to find AWS credentials. See +https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html +for how ``boto3`` works. Once you have configured your environment such +that ``boto3`` can find credentials, interaction with AWS should *just work*. + +.. hint:: + + Typically you have a ``~/.aws/credentials`` file containing AWS + credentials. If you manage multiple credentials, you can override which + *profile* to use at run-time by setting the ``AWS_PROFILE`` environment + variable. + +Resource Management +------------------- + +Depending on the task being performed, various AWS services will be accessed. +This of course requires AWS credentials with permissions to access these +services. + +The following AWS services can be accessed by automation tasks: + +* EC2 +* IAM +* Simple Systems Manager (SSM) + +Various resources will also be created as part of performing various tasks. +This also requires various permissions. + +The following AWS resources can be created by automation tasks: + +* EC2 key pairs +* EC2 security groups +* EC2 instances +* IAM roles and instance profiles +* SSM command invocations + +When possible, we prefix resource names with ``hg-`` so they can easily +be identified as belonging to Mercurial. + +.. important:: + + We currently assume that AWS accounts utilized by *us* are single + tenancy. Attempts to have discrete users of ``automation.py`` (including + sharing credentials across machines) using the same AWS account can result + in them interfering with each other and things breaking. + +Cost of Operation +----------------- + +``automation.py`` tries to be frugal with regards to utilization of remote +resources. Persistent remote resources are minimized in order to keep costs +in check. For example, EC2 instances are often ephemeral and only live as long +as the operation being performed. + +Under normal operation, recurring costs are limited to: + +* Storage costs for AMI / EBS snapshots. This should be just a few pennies + per month. + +When running EC2 instances, you'll be billed accordingly. By default, we +use *small* instances, like ``t3.medium``. This instance type costs ~$0.07 per +hour. + +.. note:: + + When running Windows EC2 instances, AWS bills at the full hourly cost, even + if the instance doesn't run for a full hour (per-second billing doesn't + apply to Windows AMIs). + +Managing Remote Resources +------------------------- + +Occassionally, there may be an error purging a temporary resource. Or you +may wish to forcefully purge remote state. Commands can be invoked to manually +purge remote resources. + +To terminate all EC2 instances that we manage:: + + $ automation.py terminate-ec2-instances + +To purge all EC2 resources that we manage:: + + $ automation.py purge-ec2-resources diff --git a/contrib/automation/automation.py b/contrib/automation/automation.py new file mode 100755 --- /dev/null +++ b/contrib/automation/automation.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# +# automation.py - Perform tasks on remote machines +# +# Copyright 2019 Gregory Szorc +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +import os +import pathlib +import subprocess +import sys +import venv + + +HERE = pathlib.Path(os.path.abspath(__file__)).parent +REQUIREMENTS_TXT = HERE / 'requirements.txt' +SOURCE_DIR = HERE.parent.parent +VENV = SOURCE_DIR / 'build' / 'venv-automation' + + +def bootstrap(): + venv_created = not VENV.exists() + + VENV.parent.mkdir(exist_ok=True) + + venv.create(VENV, with_pip=True) + + if os.name == 'nt': + venv_bin = VENV / 'Scripts' + pip = venv_bin / 'pip.exe' + python = venv_bin / 'python.exe' + else: + venv_bin = VENV / 'bin' + pip = venv_bin / 'pip' + python = venv_bin / 'python' + + args = [str(pip), 'install', '-r', str(REQUIREMENTS_TXT), + '--disable-pip-version-check'] + + if not venv_created: + args.append('-q') + + subprocess.run(args, check=True) + + os.environ['HGAUTOMATION_BOOTSTRAPPED'] = '1' + os.environ['PATH'] = '%s%s%s' % ( + venv_bin, os.pathsep, os.environ['PATH']) + + subprocess.run([str(python), __file__] + sys.argv[1:], check=True) + + +def run(): + import hgautomation.cli as cli + + # Need to strip off main Python executable. + cli.main() + + +if __name__ == '__main__': + try: + if 'HGAUTOMATION_BOOTSTRAPPED' not in os.environ: + bootstrap() + else: + run() + except subprocess.CalledProcessError as e: + sys.exit(e.returncode) + except KeyboardInterrupt: + sys.exit(1) diff --git a/contrib/automation/hgautomation/__init__.py b/contrib/automation/hgautomation/__init__.py new file mode 100644 --- /dev/null +++ b/contrib/automation/hgautomation/__init__.py @@ -0,0 +1,59 @@ +# __init__.py - High-level automation interfaces +# +# Copyright 2019 Gregory Szorc +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +# no-check-code because Python 3 native. + +import pathlib +import secrets + +from .aws import ( + AWSConnection, +) + + +class HGAutomation: + """High-level interface for Mercurial automation. + + Holds global state, provides access to other primitives, etc. + """ + + def __init__(self, state_path: pathlib.Path): + self.state_path = state_path + + state_path.mkdir(exist_ok=True) + + def default_password(self): + """Obtain the default password to use for remote machines. + + A new password will be generated if one is not stored. + """ + p = self.state_path / 'default-password' + + try: + with p.open('r', encoding='ascii') as fh: + data = fh.read().strip() + + if data: + return data + + except FileNotFoundError: + pass + + password = secrets.token_urlsafe(24) + + with p.open('w', encoding='ascii') as fh: + fh.write(password) + fh.write('\n') + + p.chmod(0o0600) + + return password + + def aws_connection(self, region: str): + """Obtain an AWSConnection instance bound to a specific region.""" + + return AWSConnection(self, region) diff --git a/contrib/automation/hgautomation/aws.py b/contrib/automation/hgautomation/aws.py new file mode 100644 --- /dev/null +++ b/contrib/automation/hgautomation/aws.py @@ -0,0 +1,879 @@ +# aws.py - Automation code for Amazon Web Services +# +# Copyright 2019 Gregory Szorc +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +# no-check-code because Python 3 native. + +import contextlib +import copy +import hashlib +import json +import os +import pathlib +import subprocess +import time + +import boto3 +import botocore.exceptions + +from .winrm import ( + run_powershell, + wait_for_winrm, +) + + +SOURCE_ROOT = pathlib.Path(os.path.abspath(__file__)).parent.parent.parent.parent + +INSTALL_WINDOWS_DEPENDENCIES = (SOURCE_ROOT / 'contrib' / + 'install-windows-dependencies.ps1') + + +KEY_PAIRS = { + 'automation', +} + + +SECURITY_GROUPS = { + 'windows-dev-1': { + 'description': 'Mercurial Windows instances that perform build automation', + 'ingress': [ + { + 'FromPort': 22, + 'ToPort': 22, + 'IpProtocol': 'tcp', + 'IpRanges': [ + { + 'CidrIp': '0.0.0.0/0', + 'Description': 'SSH from entire Internet', + }, + ], + }, + { + 'FromPort': 3389, + 'ToPort': 3389, + 'IpProtocol': 'tcp', + 'IpRanges': [ + { + 'CidrIp': '0.0.0.0/0', + 'Description': 'RDP from entire Internet', + }, + ], + + }, + { + 'FromPort': 5985, + 'ToPort': 5986, + 'IpProtocol': 'tcp', + 'IpRanges': [ + { + 'CidrIp': '0.0.0.0/0', + 'Description': 'PowerShell Remoting (Windows Remote Management)', + }, + ], + } + ], + }, +} + + +IAM_ROLES = { + 'ephemeral-ec2-role-1': { + 'description': 'Mercurial temporary EC2 instances', + 'policy_arns': [ + 'arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM', + ], + }, +} + + +ASSUME_ROLE_POLICY_DOCUMENT = ''' +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] +} +'''.strip() + + +IAM_INSTANCE_PROFILES = { + 'ephemeral-ec2-1': { + 'roles': [ + 'ephemeral-ec2-role-1', + ], + } +} + + +# User Data for Windows EC2 instance. Mainly used to set the password +# and configure WinRM. +# Inspired by the User Data script used by Packer +# (from https://www.packer.io/intro/getting-started/build-image.html). +WINDOWS_USER_DATA = ''' + + +# TODO enable this once we figure out what is failing. +#$ErrorActionPreference = "stop" + +# Set administrator password +net user Administrator "%s" +wmic useraccount where "name='Administrator'" set PasswordExpires=FALSE + +# First, make sure WinRM can't be connected to +netsh advfirewall firewall set rule name="Windows Remote Management (HTTP-In)" new enable=yes action=block + +# Delete any existing WinRM listeners +winrm delete winrm/config/listener?Address=*+Transport=HTTP 2>$Null +winrm delete winrm/config/listener?Address=*+Transport=HTTPS 2>$Null + +# Create a new WinRM listener and configure +winrm create winrm/config/listener?Address=*+Transport=HTTP +winrm set winrm/config/winrs '@{MaxMemoryPerShellMB="0"}' +winrm set winrm/config '@{MaxTimeoutms="7200000"}' +winrm set winrm/config/service '@{AllowUnencrypted="true"}' +winrm set winrm/config/service '@{MaxConcurrentOperationsPerUser="12000"}' +winrm set winrm/config/service/auth '@{Basic="true"}' +winrm set winrm/config/client/auth '@{Basic="true"}' + +# Configure UAC to allow privilege elevation in remote shells +$Key = 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System' +$Setting = 'LocalAccountTokenFilterPolicy' +Set-ItemProperty -Path $Key -Name $Setting -Value 1 -Force + +# Configure and restart the WinRM Service; Enable the required firewall exception +Stop-Service -Name WinRM +Set-Service -Name WinRM -StartupType Automatic +netsh advfirewall firewall set rule name="Windows Remote Management (HTTP-In)" new action=allow localip=any remoteip=any +Start-Service -Name WinRM + +# Disable firewall on private network interfaces so prompts don't appear. +Set-NetFirewallProfile -Name private -Enabled false + +'''.lstrip() + + +WINDOWS_BOOTSTRAP_POWERSHELL = ''' +Write-Output "installing PowerShell dependencies" +Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force +Set-PSRepository -Name PSGallery -InstallationPolicy Trusted +Install-Module -Name OpenSSHUtils -RequiredVersion 0.0.2.0 + +Write-Output "installing OpenSSL server" +Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0 +# Various tools will attempt to use older versions of .NET. So we enable +# the feature that provides them so it doesn't have to be auto-enabled +# later. +Write-Output "enabling .NET Framework feature" +Install-WindowsFeature -Name Net-Framework-Core +''' + + +class AWSConnection: + """Manages the state of a connection with AWS.""" + + def __init__(self, automation, region: str): + self.automation = automation + self.local_state_path = automation.state_path + + self.prefix = 'hg-' + + self.session = boto3.session.Session(region_name=region) + self.ec2client = self.session.client('ec2') + self.ec2resource = self.session.resource('ec2') + self.iamclient = self.session.client('iam') + self.iamresource = self.session.resource('iam') + + ensure_key_pairs(automation.state_path, self.ec2resource) + + self.security_groups = ensure_security_groups(self.ec2resource) + ensure_iam_state(self.iamresource) + + def key_pair_path_private(self, name): + """Path to a key pair private key file.""" + return self.local_state_path / 'keys' / ('keypair-%s' % name) + + def key_pair_path_public(self, name): + return self.local_state_path / 'keys' / ('keypair-%s.pub' % name) + + +def rsa_key_fingerprint(p: pathlib.Path): + """Compute the fingerprint of an RSA private key.""" + + # TODO use rsa package. + res = subprocess.run( + ['openssl', 'pkcs8', '-in', str(p), '-nocrypt', '-topk8', + '-outform', 'DER'], + capture_output=True, + check=True) + + sha1 = hashlib.sha1(res.stdout).hexdigest() + return ':'.join(a + b for a, b in zip(sha1[::2], sha1[1::2])) + + +def ensure_key_pairs(state_path: pathlib.Path, ec2resource, prefix='hg-'): + remote_existing = {} + + for kpi in ec2resource.key_pairs.all(): + if kpi.name.startswith(prefix): + remote_existing[kpi.name[len(prefix):]] = kpi.key_fingerprint + + # Validate that we have these keys locally. + key_path = state_path / 'keys' + key_path.mkdir(exist_ok=True, mode=0o700) + + def remove_remote(name): + print('deleting key pair %s' % name) + key = ec2resource.KeyPair(name) + key.delete() + + def remove_local(name): + pub_full = key_path / ('keypair-%s.pub' % name) + priv_full = key_path / ('keypair-%s' % name) + + print('removing %s' % pub_full) + pub_full.unlink() + print('removing %s' % priv_full) + priv_full.unlink() + + local_existing = {} + + for f in sorted(os.listdir(key_path)): + if not f.startswith('keypair-') or not f.endswith('.pub'): + continue + + name = f[len('keypair-'):-len('.pub')] + + pub_full = key_path / f + priv_full = key_path / ('keypair-%s' % name) + + with open(pub_full, 'r', encoding='ascii') as fh: + data = fh.read() + + if not data.startswith('ssh-rsa '): + print('unexpected format for key pair file: %s; removing' % + pub_full) + pub_full.unlink() + priv_full.unlink() + continue + + local_existing[name] = rsa_key_fingerprint(priv_full) + + for name in sorted(set(remote_existing) | set(local_existing)): + if name not in local_existing: + actual = '%s%s' % (prefix, name) + print('remote key %s does not exist locally' % name) + remove_remote(actual) + del remote_existing[name] + + elif name not in remote_existing: + print('local key %s does not exist remotely' % name) + remove_local(name) + del local_existing[name] + + elif remote_existing[name] != local_existing[name]: + print('key fingerprint mismatch for %s; ' + 'removing from local and remote' % name) + remove_local(name) + remove_remote('%s%s' % (prefix, name)) + del local_existing[name] + del remote_existing[name] + + missing = KEY_PAIRS - set(remote_existing) + + for name in sorted(missing): + actual = '%s%s' % (prefix, name) + print('creating key pair %s' % actual) + + priv_full = key_path / ('keypair-%s' % name) + pub_full = key_path / ('keypair-%s.pub' % name) + + kp = ec2resource.create_key_pair(KeyName=actual) + + with priv_full.open('w', encoding='ascii') as fh: + fh.write(kp.key_material) + fh.write('\n') + + priv_full.chmod(0o0600) + + # SSH public key can be extracted via `ssh-keygen`. + with pub_full.open('w', encoding='ascii') as fh: + subprocess.run( + ['ssh-keygen', '-y', '-f', str(priv_full)], + stdout=fh, + check=True) + + pub_full.chmod(0o0600) + + +def delete_instance_profile(profile): + for role in profile.roles: + print('removing role %s from instance profile %s' % (role.name, + profile.name)) + profile.remove_role(RoleName=role.name) + + print('deleting instance profile %s' % profile.name) + profile.delete() + + +def ensure_iam_state(iamresource, prefix='hg-'): + """Ensure IAM state is in sync with our canonical definition.""" + + remote_profiles = {} + + for profile in iamresource.instance_profiles.all(): + if profile.name.startswith(prefix): + remote_profiles[profile.name[len(prefix):]] = profile + + for name in sorted(set(remote_profiles) - set(IAM_INSTANCE_PROFILES)): + delete_instance_profile(remote_profiles[name]) + del remote_profiles[name] + + remote_roles = {} + + for role in iamresource.roles.all(): + if role.name.startswith(prefix): + remote_roles[role.name[len(prefix):]] = role + + for name in sorted(set(remote_roles) - set(IAM_ROLES)): + role = remote_roles[name] + + print('removing role %s' % role.name) + role.delete() + del remote_roles[name] + + # We've purged remote state that doesn't belong. Create missing + # instance profiles and roles. + for name in sorted(set(IAM_INSTANCE_PROFILES) - set(remote_profiles)): + actual = '%s%s' % (prefix, name) + print('creating IAM instance profile %s' % actual) + + profile = iamresource.create_instance_profile( + InstanceProfileName=actual) + remote_profiles[name] = profile + + for name in sorted(set(IAM_ROLES) - set(remote_roles)): + entry = IAM_ROLES[name] + + actual = '%s%s' % (prefix, name) + print('creating IAM role %s' % actual) + + role = iamresource.create_role( + RoleName=actual, + Description=entry['description'], + AssumeRolePolicyDocument=ASSUME_ROLE_POLICY_DOCUMENT, + ) + + remote_roles[name] = role + + for arn in entry['policy_arns']: + print('attaching policy %s to %s' % (arn, role.name)) + role.attach_policy(PolicyArn=arn) + + # Now reconcile state of profiles. + for name, meta in sorted(IAM_INSTANCE_PROFILES.items()): + profile = remote_profiles[name] + wanted = {'%s%s' % (prefix, role) for role in meta['roles']} + have = {role.name for role in profile.roles} + + for role in sorted(have - wanted): + print('removing role %s from %s' % (role, profile.name)) + profile.remove_role(RoleName=role) + + for role in sorted(wanted - have): + print('adding role %s to %s' % (role, profile.name)) + profile.add_role(RoleName=role) + + +def find_windows_server_2019_image(ec2resource): + """Find the Amazon published Windows Server 2019 base image.""" + + images = ec2resource.images.filter( + Filters=[ + { + 'Name': 'owner-alias', + 'Values': ['amazon'], + }, + { + 'Name': 'state', + 'Values': ['available'], + }, + { + 'Name': 'image-type', + 'Values': ['machine'], + }, + { + 'Name': 'name', + 'Values': ['Windows_Server-2019-English-Full-Base-2019.02.13'], + }, + ]) + + for image in images: + return image + + raise Exception('unable to find Windows Server 2019 image') + + +def ensure_security_groups(ec2resource, prefix='hg-'): + """Ensure all necessary Mercurial security groups are present. + + All security groups are prefixed with ``hg-`` by default. Any security + groups having this prefix but aren't in our list are deleted. + """ + existing = {} + + for group in ec2resource.security_groups.all(): + if group.group_name.startswith(prefix): + existing[group.group_name[len(prefix):]] = group + + purge = set(existing) - set(SECURITY_GROUPS) + + for name in sorted(purge): + group = existing[name] + print('removing legacy security group: %s' % group.group_name) + group.delete() + + security_groups = {} + + for name, group in sorted(SECURITY_GROUPS.items()): + if name in existing: + security_groups[name] = existing[name] + continue + + actual = '%s%s' % (prefix, name) + print('adding security group %s' % actual) + + group_res = ec2resource.create_security_group( + Description=group['description'], + GroupName=actual, + ) + + group_res.authorize_ingress( + IpPermissions=group['ingress'], + ) + + security_groups[name] = group_res + + return security_groups + + +def terminate_ec2_instances(ec2resource, prefix='hg-'): + """Terminate all EC2 instances managed by us.""" + waiting = [] + + for instance in ec2resource.instances.all(): + if instance.state['Name'] == 'terminated': + continue + + for tag in instance.tags or []: + if tag['Key'] == 'Name' and tag['Value'].startswith(prefix): + print('terminating %s' % instance.id) + instance.terminate() + waiting.append(instance) + + for instance in waiting: + instance.wait_until_terminated() + + +def remove_resources(c, prefix='hg-'): + """Purge all of our resources in this EC2 region.""" + ec2resource = c.ec2resource + iamresource = c.iamresource + + terminate_ec2_instances(ec2resource, prefix=prefix) + + for image in ec2resource.images.all(): + if image.name.startswith(prefix): + remove_ami(ec2resource, image) + + for group in ec2resource.security_groups.all(): + if group.group_name.startswith(prefix): + print('removing security group %s' % group.group_name) + group.delete() + + for profile in iamresource.instance_profiles.all(): + if profile.name.startswith(prefix): + delete_instance_profile(profile) + + for role in iamresource.roles.all(): + if role.name.startswith(prefix): + print('removing role %s' % role.name) + role.delete() + + +def wait_for_ip_addresses(instances): + """Wait for the public IP addresses of an iterable of instances.""" + for instance in instances: + while True: + if not instance.public_ip_address: + time.sleep(2) + instance.reload() + continue + + print('public IP address for %s: %s' % ( + instance.id, instance.public_ip_address)) + break + + +def remove_ami(ec2resource, image): + """Remove an AMI and its underlying snapshots.""" + snapshots = [] + + for device in image.block_device_mappings: + if 'Ebs' in device: + snapshots.append(ec2resource.Snapshot(device['Ebs']['SnapshotId'])) + + print('deregistering %s' % image.id) + image.deregister() + + for snapshot in snapshots: + print('deleting snapshot %s' % snapshot.id) + snapshot.delete() + + +def wait_for_ssm(ssmclient, instances): + """Wait for SSM to come online for an iterable of instance IDs.""" + while True: + res = ssmclient.describe_instance_information( + Filters=[ + { + 'Key': 'InstanceIds', + 'Values': [i.id for i in instances], + }, + ], + ) + + available = len(res['InstanceInformationList']) + wanted = len(instances) + + print('%d/%d instances available in SSM' % (available, wanted)) + + if available == wanted: + return + + time.sleep(2) + + +def run_ssm_command(ssmclient, instances, document_name, parameters): + """Run a PowerShell script on an EC2 instance.""" + + res = ssmclient.send_command( + InstanceIds=[i.id for i in instances], + DocumentName=document_name, + Parameters=parameters, + CloudWatchOutputConfig={ + 'CloudWatchOutputEnabled': True, + }, + ) + + command_id = res['Command']['CommandId'] + + for instance in instances: + while True: + try: + res = ssmclient.get_command_invocation( + CommandId=command_id, + InstanceId=instance.id, + ) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'InvocationDoesNotExist': + print('could not find SSM command invocation; waiting') + time.sleep(1) + continue + else: + raise + + if res['Status'] == 'Success': + break + elif res['Status'] in ('Pending', 'InProgress', 'Delayed'): + time.sleep(2) + else: + raise Exception('command failed on %s: %s' % ( + instance.id, res['Status'])) + + +@contextlib.contextmanager +def temporary_ec2_instances(ec2resource, config): + """Create temporary EC2 instances. + + This is a proxy to ``ec2client.run_instances(**config)`` that takes care of + managing the lifecycle of the instances. + + When the context manager exits, the instances are terminated. + + The context manager evaluates to the list of data structures + describing each created instance. The instances may not be available + for work immediately: it is up to the caller to wait for the instance + to start responding. + """ + + ids = None + + try: + res = ec2resource.create_instances(**config) + + ids = [i.id for i in res] + print('started instances: %s' % ' '.join(ids)) + + yield res + finally: + if ids: + print('terminating instances: %s' % ' '.join(ids)) + for instance in res: + instance.terminate() + print('terminated %d instances' % len(ids)) + + +@contextlib.contextmanager +def create_temp_windows_ec2_instances(c: AWSConnection, config): + """Create temporary Windows EC2 instances. + + This is a higher-level wrapper around ``create_temp_ec2_instances()`` that + configures the Windows instance for Windows Remote Management. The emitted + instances will have a ``winrm_client`` attribute containing a + ``pypsrp.client.Client`` instance bound to the instance. + """ + if 'IamInstanceProfile' in config: + raise ValueError('IamInstanceProfile cannot be provided in config') + if 'UserData' in config: + raise ValueError('UserData cannot be provided in config') + + password = c.automation.default_password() + + config = copy.deepcopy(config) + config['IamInstanceProfile'] = { + 'Name': 'hg-ephemeral-ec2-1', + } + config.setdefault('TagSpecifications', []).append({ + 'ResourceType': 'instance', + 'Tags': [{'Key': 'Name', 'Value': 'hg-temp-windows'}], + }) + config['UserData'] = WINDOWS_USER_DATA % password + + with temporary_ec2_instances(c.ec2resource, config) as instances: + wait_for_ip_addresses(instances) + + print('waiting for Windows Remote Management service...') + + for instance in instances: + client = wait_for_winrm(instance.public_ip_address, 'Administrator', password) + print('established WinRM connection to %s' % instance.id) + instance.winrm_client = client + + yield instances + + +def ensure_windows_dev_ami(c: AWSConnection, prefix='hg-'): + """Ensure Windows Development AMI is available and up-to-date. + + If necessary, a modern AMI will be built by starting a temporary EC2 + instance and bootstrapping it. + + Obsolete AMIs will be deleted so there is only a single AMI having the + desired name. + + Returns an ``ec2.Image`` of either an existing AMI or a newly-built + one. + """ + ec2client = c.ec2client + ec2resource = c.ec2resource + ssmclient = c.session.client('ssm') + + name = '%s%s' % (prefix, 'windows-dev') + + config = { + 'BlockDeviceMappings': [ + { + 'DeviceName': '/dev/sda1', + 'Ebs': { + 'DeleteOnTermination': True, + 'VolumeSize': 32, + 'VolumeType': 'gp2', + }, + } + ], + 'ImageId': find_windows_server_2019_image(ec2resource).id, + 'InstanceInitiatedShutdownBehavior': 'stop', + 'InstanceType': 't3.medium', + 'KeyName': '%sautomation' % prefix, + 'MaxCount': 1, + 'MinCount': 1, + 'SecurityGroupIds': [c.security_groups['windows-dev-1'].id], + } + + commands = [ + # Need to start the service so sshd_config is generated. + 'Start-Service sshd', + 'Write-Output "modifying sshd_config"', + r'$content = Get-Content C:\ProgramData\ssh\sshd_config', + '$content = $content -replace "Match Group administrators","" -replace "AuthorizedKeysFile __PROGRAMDATA__/ssh/administrators_authorized_keys",""', + r'$content | Set-Content C:\ProgramData\ssh\sshd_config', + 'Import-Module OpenSSHUtils', + r'Repair-SshdConfigPermission C:\ProgramData\ssh\sshd_config -Confirm:$false', + 'Restart-Service sshd', + 'Write-Output "installing OpenSSL client"', + 'Add-WindowsCapability -Online -Name OpenSSH.Client~~~~0.0.1.0', + 'Set-Service -Name sshd -StartupType "Automatic"', + 'Write-Output "OpenSSH server running"', + ] + + with INSTALL_WINDOWS_DEPENDENCIES.open('r', encoding='utf-8') as fh: + commands.extend(l.rstrip() for l in fh) + + # Disable Windows Defender when bootstrapping because it just slows + # things down. + commands.insert(0, 'Set-MpPreference -DisableRealtimeMonitoring $true') + commands.append('Set-MpPreference -DisableRealtimeMonitoring $false') + + # Compute a deterministic fingerprint to determine whether image needs + # to be regenerated. + fingerprint = { + 'instance_config': config, + 'user_data': WINDOWS_USER_DATA, + 'initial_bootstrap': WINDOWS_BOOTSTRAP_POWERSHELL, + 'bootstrap_commands': commands, + } + + fingerprint = json.dumps(fingerprint, sort_keys=True) + fingerprint = hashlib.sha256(fingerprint.encode('utf-8')).hexdigest() + + # Find existing AMIs with this name and delete the ones that are invalid. + # Store a reference to a good image so it can be returned one the + # image state is reconciled. + images = ec2resource.images.filter( + Filters=[{'Name': 'name', 'Values': [name]}]) + + existing_image = None + + for image in images: + if image.tags is None: + print('image %s for %s lacks required tags; removing' % ( + image.id, image.name)) + remove_ami(ec2resource, image) + else: + tags = {t['Key']: t['Value'] for t in image.tags} + + if tags.get('HGIMAGEFINGERPRINT') == fingerprint: + existing_image = image + else: + print('image %s for %s has wrong fingerprint; removing' % ( + image.id, image.name)) + remove_ami(ec2resource, image) + + if existing_image: + return existing_image + + print('no suitable Windows development image found; creating one...') + + with create_temp_windows_ec2_instances(c, config) as instances: + assert len(instances) == 1 + instance = instances[0] + + wait_for_ssm(ssmclient, [instance]) + + # On first boot, install various Windows updates. + # We would ideally use PowerShell Remoting for this. However, there are + # trust issues that make it difficult to invoke Windows Update + # remotely. So we use SSM, which has a mechanism for running Windows + # Update. + print('installing Windows features...') + run_ssm_command( + ssmclient, + [instance], + 'AWS-RunPowerShellScript', + { + 'commands': WINDOWS_BOOTSTRAP_POWERSHELL.split('\n'), + }, + ) + + # Reboot so all updates are fully applied. + print('rebooting instance %s' % instance.id) + ec2client.reboot_instances(InstanceIds=[instance.id]) + + time.sleep(15) + + print('waiting for Windows Remote Management to come back...') + client = wait_for_winrm(instance.public_ip_address, 'Administrator', + c.automation.default_password()) + print('established WinRM connection to %s' % instance.id) + instance.winrm_client = client + + print('bootstrapping instance...') + run_powershell(instance.winrm_client, '\n'.join(commands)) + + print('bootstrap completed; stopping %s to create image' % instance.id) + instance.stop() + + ec2client.get_waiter('instance_stopped').wait( + InstanceIds=[instance.id], + WaiterConfig={ + 'Delay': 5, + }) + print('%s is stopped' % instance.id) + + image = instance.create_image( + Name=name, + Description='Mercurial Windows development environment', + ) + + image.create_tags(Tags=[ + { + 'Key': 'HGIMAGEFINGERPRINT', + 'Value': fingerprint, + }, + ]) + + print('waiting for image %s' % image.id) + + ec2client.get_waiter('image_available').wait( + ImageIds=[image.id], + ) + + print('image %s available as %s' % (image.id, image.name)) + + return image + + +@contextlib.contextmanager +def temporary_windows_dev_instances(c: AWSConnection, image, instance_type, + prefix='hg-', disable_antivirus=False): + """Create a temporary Windows development EC2 instance. + + Context manager resolves to the list of ``EC2.Instance`` that were created. + """ + config = { + 'BlockDeviceMappings': [ + { + 'DeviceName': '/dev/sda1', + 'Ebs': { + 'DeleteOnTermination': True, + 'VolumeSize': 32, + 'VolumeType': 'gp2', + }, + } + ], + 'ImageId': image.id, + 'InstanceInitiatedShutdownBehavior': 'stop', + 'InstanceType': instance_type, + 'KeyName': '%sautomation' % prefix, + 'MaxCount': 1, + 'MinCount': 1, + 'SecurityGroupIds': [c.security_groups['windows-dev-1'].id], + } + + with create_temp_windows_ec2_instances(c, config) as instances: + if disable_antivirus: + for instance in instances: + run_powershell( + instance.winrm_client, + 'Set-MpPreference -DisableRealtimeMonitoring $true') + + yield instances diff --git a/contrib/automation/hgautomation/cli.py b/contrib/automation/hgautomation/cli.py new file mode 100644 --- /dev/null +++ b/contrib/automation/hgautomation/cli.py @@ -0,0 +1,273 @@ +# cli.py - Command line interface for automation +# +# Copyright 2019 Gregory Szorc +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +# no-check-code because Python 3 native. + +import argparse +import os +import pathlib + +from . import ( + aws, + HGAutomation, + windows, +) + + +SOURCE_ROOT = pathlib.Path(os.path.abspath(__file__)).parent.parent.parent.parent +DIST_PATH = SOURCE_ROOT / 'dist' + + +def bootstrap_windows_dev(hga: HGAutomation, aws_region): + c = hga.aws_connection(aws_region) + image = aws.ensure_windows_dev_ami(c) + print('Windows development AMI available as %s' % image.id) + + +def build_inno(hga: HGAutomation, aws_region, arch, revision, version): + c = hga.aws_connection(aws_region) + image = aws.ensure_windows_dev_ami(c) + DIST_PATH.mkdir(exist_ok=True) + + with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: + instance = insts[0] + + windows.synchronize_hg(SOURCE_ROOT, revision, instance) + + for a in arch: + windows.build_inno_installer(instance.winrm_client, a, + DIST_PATH, + version=version) + + +def build_wix(hga: HGAutomation, aws_region, arch, revision, version): + c = hga.aws_connection(aws_region) + image = aws.ensure_windows_dev_ami(c) + DIST_PATH.mkdir(exist_ok=True) + + with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: + instance = insts[0] + + windows.synchronize_hg(SOURCE_ROOT, revision, instance) + + for a in arch: + windows.build_wix_installer(instance.winrm_client, a, + DIST_PATH, version=version) + + +def build_windows_wheel(hga: HGAutomation, aws_region, arch, revision): + c = hga.aws_connection(aws_region) + image = aws.ensure_windows_dev_ami(c) + DIST_PATH.mkdir(exist_ok=True) + + with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: + instance = insts[0] + + windows.synchronize_hg(SOURCE_ROOT, revision, instance) + + for a in arch: + windows.build_wheel(instance.winrm_client, a, DIST_PATH) + + +def build_all_windows_packages(hga: HGAutomation, aws_region, revision): + c = hga.aws_connection(aws_region) + image = aws.ensure_windows_dev_ami(c) + DIST_PATH.mkdir(exist_ok=True) + + with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: + instance = insts[0] + + winrm_client = instance.winrm_client + + windows.synchronize_hg(SOURCE_ROOT, revision, instance) + + for arch in ('x86', 'x64'): + windows.purge_hg(winrm_client) + windows.build_wheel(winrm_client, arch, DIST_PATH) + windows.purge_hg(winrm_client) + windows.build_inno_installer(winrm_client, arch, DIST_PATH) + windows.purge_hg(winrm_client) + windows.build_wix_installer(winrm_client, arch, DIST_PATH) + + +def terminate_ec2_instances(hga: HGAutomation, aws_region): + c = hga.aws_connection(aws_region) + aws.terminate_ec2_instances(c.ec2resource) + + +def purge_ec2_resources(hga: HGAutomation, aws_region): + c = hga.aws_connection(aws_region) + aws.remove_resources(c) + + +def run_tests_windows(hga: HGAutomation, aws_region, instance_type, + python_version, arch, test_flags): + c = hga.aws_connection(aws_region) + image = aws.ensure_windows_dev_ami(c) + + with aws.temporary_windows_dev_instances(c, image, instance_type, + disable_antivirus=True) as insts: + instance = insts[0] + + windows.synchronize_hg(SOURCE_ROOT, '.', instance) + windows.run_tests(instance.winrm_client, python_version, arch, + test_flags) + + +def get_parser(): + parser = argparse.ArgumentParser() + + parser.add_argument( + '--state-path', + default='~/.hgautomation', + help='Path for local state files', + ) + parser.add_argument( + '--aws-region', + help='AWS region to use', + default='us-west-1', + ) + + subparsers = parser.add_subparsers() + + sp = subparsers.add_parser( + 'bootstrap-windows-dev', + help='Bootstrap the Windows development environment', + ) + sp.set_defaults(func=bootstrap_windows_dev) + + sp = subparsers.add_parser( + 'build-all-windows-packages', + help='Build all Windows packages', + ) + sp.add_argument( + '--revision', + help='Mercurial revision to build', + default='.', + ) + sp.set_defaults(func=build_all_windows_packages) + + sp = subparsers.add_parser( + 'build-inno', + help='Build Inno Setup installer(s)', + ) + sp.add_argument( + '--arch', + help='Architecture to build for', + choices={'x86', 'x64'}, + nargs='*', + default=['x64'], + ) + sp.add_argument( + '--revision', + help='Mercurial revision to build', + default='.', + ) + sp.add_argument( + '--version', + help='Mercurial version string to use in installer', + ) + sp.set_defaults(func=build_inno) + + sp = subparsers.add_parser( + 'build-windows-wheel', + help='Build Windows wheel(s)', + ) + sp.add_argument( + '--arch', + help='Architecture to build for', + choices={'x86', 'x64'}, + nargs='*', + default=['x64'], + ) + sp.add_argument( + '--revision', + help='Mercurial revision to build', + default='.', + ) + sp.set_defaults(func=build_windows_wheel) + + sp = subparsers.add_parser( + 'build-wix', + help='Build WiX installer(s)' + ) + sp.add_argument( + '--arch', + help='Architecture to build for', + choices={'x86', 'x64'}, + nargs='*', + default=['x64'], + ) + sp.add_argument( + '--revision', + help='Mercurial revision to build', + default='.', + ) + sp.add_argument( + '--version', + help='Mercurial version string to use in installer', + ) + sp.set_defaults(func=build_wix) + + sp = subparsers.add_parser( + 'terminate-ec2-instances', + help='Terminate all active EC2 instances managed by us', + ) + sp.set_defaults(func=terminate_ec2_instances) + + sp = subparsers.add_parser( + 'purge-ec2-resources', + help='Purge all EC2 resources managed by us', + ) + sp.set_defaults(func=purge_ec2_resources) + + sp = subparsers.add_parser( + 'run-tests-windows', + help='Run tests on Windows', + ) + sp.add_argument( + '--instance-type', + help='EC2 instance type to use', + default='t3.medium', + ) + sp.add_argument( + '--python-version', + help='Python version to use', + choices={'2.7', '3.5', '3.6', '3.7', '3.8'}, + default='2.7', + ) + sp.add_argument( + '--arch', + help='Architecture to test', + choices={'x86', 'x64'}, + default='x64', + ) + sp.add_argument( + '--test-flags', + help='Extra command line flags to pass to run-tests.py', + ) + sp.set_defaults(func=run_tests_windows) + + return parser + + +def main(): + parser = get_parser() + args = parser.parse_args() + + local_state_path = pathlib.Path(os.path.expanduser(args.state_path)) + automation = HGAutomation(local_state_path) + + if not hasattr(args, 'func'): + parser.print_help() + return + + kwargs = dict(vars(args)) + del kwargs['func'] + del kwargs['state_path'] + + args.func(automation, **kwargs) diff --git a/contrib/automation/hgautomation/windows.py b/contrib/automation/hgautomation/windows.py new file mode 100644 --- /dev/null +++ b/contrib/automation/hgautomation/windows.py @@ -0,0 +1,287 @@ +# windows.py - Automation specific to Windows +# +# Copyright 2019 Gregory Szorc +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +# no-check-code because Python 3 native. + +import os +import pathlib +import re +import subprocess +import tempfile + +from .winrm import ( + run_powershell, +) + + +# PowerShell commands to activate a Visual Studio 2008 environment. +# This is essentially a port of vcvarsall.bat to PowerShell. +ACTIVATE_VC9_AMD64 = r''' +Write-Output "activating Visual Studio 2008 environment for AMD64" +$root = "$env:LOCALAPPDATA\Programs\Common\Microsoft\Visual C++ for Python\9.0" +$Env:VCINSTALLDIR = "${root}\VC\" +$Env:WindowsSdkDir = "${root}\WinSDK\" +$Env:PATH = "${root}\VC\Bin\amd64;${root}\WinSDK\Bin\x64;${root}\WinSDK\Bin;$Env:PATH" +$Env:INCLUDE = "${root}\VC\Include;${root}\WinSDK\Include;$Env:PATH" +$Env:LIB = "${root}\VC\Lib\amd64;${root}\WinSDK\Lib\x64;$Env:LIB" +$Env:LIBPATH = "${root}\VC\Lib\amd64;${root}\WinSDK\Lib\x64;$Env:LIBPATH" +'''.lstrip() + +ACTIVATE_VC9_X86 = r''' +Write-Output "activating Visual Studio 2008 environment for x86" +$root = "$env:LOCALAPPDATA\Programs\Common\Microsoft\Visual C++ for Python\9.0" +$Env:VCINSTALLDIR = "${root}\VC\" +$Env:WindowsSdkDir = "${root}\WinSDK\" +$Env:PATH = "${root}\VC\Bin;${root}\WinSDK\Bin;$Env:PATH" +$Env:INCLUDE = "${root}\VC\Include;${root}\WinSDK\Include;$Env:INCLUDE" +$Env:LIB = "${root}\VC\Lib;${root}\WinSDK\Lib;$Env:LIB" +$Env:LIBPATH = "${root}\VC\lib;${root}\WinSDK\Lib:$Env:LIBPATH" +'''.lstrip() + +HG_PURGE = r''' +$Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH" +Set-Location C:\hgdev\src +hg.exe --config extensions.purge= purge --all +if ($LASTEXITCODE -ne 0) { + throw "process exited non-0: $LASTEXITCODE" +} +Write-Output "purged Mercurial repo" +''' + +HG_UPDATE_CLEAN = r''' +$Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH" +Set-Location C:\hgdev\src +hg.exe --config extensions.purge= purge --all +if ($LASTEXITCODE -ne 0) {{ + throw "process exited non-0: $LASTEXITCODE" +}} +hg.exe update -C {revision} +if ($LASTEXITCODE -ne 0) {{ + throw "process exited non-0: $LASTEXITCODE" +}} +hg.exe log -r . +Write-Output "updated Mercurial working directory to {revision}" +'''.lstrip() + +BUILD_INNO = r''' +Set-Location C:\hgdev\src +$python = "C:\hgdev\python27-{arch}\python.exe" +C:\hgdev\python37-x64\python.exe contrib\packaging\inno\build.py --python $python +if ($LASTEXITCODE -ne 0) {{ + throw "process exited non-0: $LASTEXITCODE" +}} +'''.lstrip() + +BUILD_WHEEL = r''' +Set-Location C:\hgdev\src +C:\hgdev\python27-{arch}\Scripts\pip.exe wheel --wheel-dir dist . +if ($LASTEXITCODE -ne 0) {{ + throw "process exited non-0: $LASTEXITCODE" +}} +''' + +BUILD_WIX = r''' +Set-Location C:\hgdev\src +$python = "C:\hgdev\python27-{arch}\python.exe" +C:\hgdev\python37-x64\python.exe contrib\packaging\wix\build.py --python $python {extra_args} +if ($LASTEXITCODE -ne 0) {{ + throw "process exited non-0: $LASTEXITCODE" +}} +''' + +RUN_TESTS = r''' +C:\hgdev\MinGW\msys\1.0\bin\sh.exe --login -c "cd /c/hgdev/src/tests && /c/hgdev/{python_path}/python.exe run-tests.py {test_flags}" +if ($LASTEXITCODE -ne 0) {{ + throw "process exited non-0: $LASTEXITCODE" +}} +''' + + +def get_vc_prefix(arch): + if arch == 'x86': + return ACTIVATE_VC9_X86 + elif arch == 'x64': + return ACTIVATE_VC9_AMD64 + else: + raise ValueError('illegal arch: %s; must be x86 or x64' % arch) + + +def fix_authorized_keys_permissions(winrm_client, path): + commands = [ + '$ErrorActionPreference = "Stop"', + 'Repair-AuthorizedKeyPermission -FilePath %s -Confirm:$false' % path, + 'icacls %s /remove:g "NT Service\sshd"' % path, + ] + + run_powershell(winrm_client, '\n'.join(commands)) + + +def synchronize_hg(hg_repo: pathlib.Path, revision: str, ec2_instance): + """Synchronize local Mercurial repo to remote EC2 instance.""" + + winrm_client = ec2_instance.winrm_client + + with tempfile.TemporaryDirectory() as temp_dir: + temp_dir = pathlib.Path(temp_dir) + + ssh_dir = temp_dir / '.ssh' + ssh_dir.mkdir() + ssh_dir.chmod(0o0700) + + # Generate SSH key to use for communication. + subprocess.run([ + 'ssh-keygen', '-t', 'rsa', '-b', '4096', '-N', '', + '-f', str(ssh_dir / 'id_rsa')], + check=True, capture_output=True) + + # Add it to ~/.ssh/authorized_keys on remote. + # This assumes the file doesn't already exist. + authorized_keys = r'c:\Users\Administrator\.ssh\authorized_keys' + winrm_client.execute_cmd(r'mkdir c:\Users\Administrator\.ssh') + winrm_client.copy(str(ssh_dir / 'id_rsa.pub'), authorized_keys) + fix_authorized_keys_permissions(winrm_client, authorized_keys) + + public_ip = ec2_instance.public_ip_address + + ssh_config = temp_dir / '.ssh' / 'config' + + with open(ssh_config, 'w', encoding='utf-8') as fh: + fh.write('Host %s\n' % public_ip) + fh.write(' User Administrator\n') + fh.write(' StrictHostKeyChecking no\n') + fh.write(' UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts')) + fh.write(' IdentityFile %s\n' % (ssh_dir / 'id_rsa')) + + env = dict(os.environ) + env['HGPLAIN'] = '1' + env['HGENCODING'] = 'utf-8' + + hg_bin = hg_repo / 'hg' + + res = subprocess.run( + ['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'], + cwd=str(hg_repo), env=env, check=True, capture_output=True) + + full_revision = res.stdout.decode('ascii') + + args = [ + 'python2.7', hg_bin, + '--config', 'ui.ssh=ssh -F %s' % ssh_config, + '--config', 'ui.remotecmd=c:/hgdev/venv-bootstrap/Scripts/hg.exe', + 'push', '-r', full_revision, 'ssh://%s/c:/hgdev/src' % public_ip, + ] + + subprocess.run(args, cwd=str(hg_repo), env=env, check=True) + + run_powershell(winrm_client, + HG_UPDATE_CLEAN.format(revision=full_revision)) + + # TODO detect dirty local working directory and synchronize accordingly. + + +def purge_hg(winrm_client): + """Purge the Mercurial source repository on an EC2 instance.""" + run_powershell(winrm_client, HG_PURGE) + + +def find_latest_dist(winrm_client, pattern): + """Find path to newest file in dist/ directory matching a pattern.""" + + res = winrm_client.execute_ps( + '$v = Get-ChildItem -Path C:\hgdev\src\dist -Filter "%s" ' + '| Sort-Object LastWriteTime -Descending ' + '| Select-Object -First 1\n' + '$v.name' % pattern + ) + return res[0] + + +def copy_latest_dist(winrm_client, pattern, dest_path): + """Copy latest file matching pattern in dist/ directory. + + Given a WinRM client and a file pattern, find the latest file on the remote + matching that pattern and copy it to the ``dest_path`` directory on the + local machine. + """ + latest = find_latest_dist(winrm_client, pattern) + source = r'C:\hgdev\src\dist\%s' % latest + dest = dest_path / latest + print('copying %s to %s' % (source, dest)) + winrm_client.fetch(source, str(dest)) + + +def build_inno_installer(winrm_client, arch: str, dest_path: pathlib.Path, + version=None): + """Build the Inno Setup installer on a remote machine. + + Using a WinRM client, remote commands are executed to build + a Mercurial Inno Setup installer. + """ + print('building Inno Setup installer for %s' % arch) + + extra_args = [] + if version: + extra_args.extend(['--version', version]) + + ps = get_vc_prefix(arch) + BUILD_INNO.format(arch=arch, + extra_args=' '.join(extra_args)) + run_powershell(winrm_client, ps) + copy_latest_dist(winrm_client, '*.exe', dest_path) + + +def build_wheel(winrm_client, arch: str, dest_path: pathlib.Path): + """Build Python wheels on a remote machine. + + Using a WinRM client, remote commands are executed to build a Python wheel + for Mercurial. + """ + print('Building Windows wheel for %s' % arch) + ps = get_vc_prefix(arch) + BUILD_WHEEL.format(arch=arch) + run_powershell(winrm_client, ps) + copy_latest_dist(winrm_client, '*.whl', dest_path) + + +def build_wix_installer(winrm_client, arch: str, dest_path: pathlib.Path, + version=None): + """Build the WiX installer on a remote machine. + + Using a WinRM client, remote commands are executed to build a WiX installer. + """ + print('Building WiX installer for %s' % arch) + extra_args = [] + if version: + extra_args.extend(['--version', version]) + + ps = get_vc_prefix(arch) + BUILD_WIX.format(arch=arch, + extra_args=' '.join(extra_args)) + run_powershell(winrm_client, ps) + copy_latest_dist(winrm_client, '*.msi', dest_path) + + +def run_tests(winrm_client, python_version, arch, test_flags=''): + """Run tests on a remote Windows machine. + + ``python_version`` is a ``X.Y`` string like ``2.7`` or ``3.7``. + ``arch`` is ``x86`` or ``x64``. + ``test_flags`` is a str representing extra arguments to pass to + ``run-tests.py``. + """ + if not re.match('\d\.\d', python_version): + raise ValueError('python_version must be \d.\d; got %s' % + python_version) + + if arch not in ('x86', 'x64'): + raise ValueError('arch must be x86 or x64; got %s' % arch) + + python_path = 'python%s-%s' % (python_version.replace('.', ''), arch) + + ps = RUN_TESTS.format( + python_path=python_path, + test_flags=test_flags or '', + ) + + run_powershell(winrm_client, ps) diff --git a/contrib/automation/hgautomation/winrm.py b/contrib/automation/hgautomation/winrm.py new file mode 100644 --- /dev/null +++ b/contrib/automation/hgautomation/winrm.py @@ -0,0 +1,82 @@ +# winrm.py - Interact with Windows Remote Management (WinRM) +# +# Copyright 2019 Gregory Szorc +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +# no-check-code because Python 3 native. + +import logging +import pprint +import time + +from pypsrp.client import ( + Client, +) +from pypsrp.powershell import ( + PowerShell, + PSInvocationState, + RunspacePool, +) +import requests.exceptions + + +logger = logging.getLogger(__name__) + + +def wait_for_winrm(host, username, password, timeout=120, ssl=False): + """Wait for the Windows Remoting (WinRM) service to become available. + + Returns a ``psrpclient.Client`` instance. + """ + + end_time = time.time() + timeout + + while True: + try: + client = Client(host, username=username, password=password, + ssl=ssl, connection_timeout=5) + client.execute_cmd('echo "hello world"') + return client + except requests.exceptions.ConnectionError: + if time.time() >= end_time: + raise + + time.sleep(1) + + +def format_object(o): + if isinstance(o, str): + return o + + try: + o = str(o) + except TypeError: + o = pprint.pformat(o.extended_properties) + + return o + + +def run_powershell(client, script): + with RunspacePool(client.wsman) as pool: + ps = PowerShell(pool) + ps.add_script(script) + + ps.begin_invoke() + + while ps.state == PSInvocationState.RUNNING: + ps.poll_invoke() + for o in ps.output: + print(format_object(o)) + + ps.output[:] = [] + + ps.end_invoke() + + for o in ps.output: + print(format_object(o)) + + if ps.state == PSInvocationState.FAILED: + raise Exception('PowerShell execution failed: %s' % + ' '.join(map(format_object, ps.streams.error))) diff --git a/contrib/automation/requirements.txt b/contrib/automation/requirements.txt new file mode 100644 --- /dev/null +++ b/contrib/automation/requirements.txt @@ -0,0 +1,119 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile -U --generate-hashes --output-file contrib/automation/requirements.txt contrib/automation/requirements.txt.in +# +asn1crypto==0.24.0 \ + --hash=sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87 \ + --hash=sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49 \ + # via cryptography +boto3==1.9.111 \ + --hash=sha256:06414c75d1f62af7d04fd652b38d1e4fd3cfd6b35bad978466af88e2aaecd00d \ + --hash=sha256:f3b77dff382374773d02411fa47ee408f4f503aeebd837fd9dc9ed8635bc5e8e +botocore==1.12.111 \ + --hash=sha256:6af473c52d5e3e7ff82de5334e9fee96b2d5ec2df5d78bc00cd9937e2573a7a8 \ + --hash=sha256:9f5123c7be704b17aeacae99b5842ab17bda1f799dd29134de8c70e0a50a45d7 \ + # via boto3, s3transfer +certifi==2019.3.9 \ + --hash=sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5 \ + --hash=sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae \ + # via requests +cffi==1.12.2 \ + --hash=sha256:00b97afa72c233495560a0793cdc86c2571721b4271c0667addc83c417f3d90f \ + --hash=sha256:0ba1b0c90f2124459f6966a10c03794082a2f3985cd699d7d63c4a8dae113e11 \ + --hash=sha256:0bffb69da295a4fc3349f2ec7cbe16b8ba057b0a593a92cbe8396e535244ee9d \ + --hash=sha256:21469a2b1082088d11ccd79dd84157ba42d940064abbfa59cf5f024c19cf4891 \ + --hash=sha256:2e4812f7fa984bf1ab253a40f1f4391b604f7fc424a3e21f7de542a7f8f7aedf \ + --hash=sha256:2eac2cdd07b9049dd4e68449b90d3ef1adc7c759463af5beb53a84f1db62e36c \ + --hash=sha256:2f9089979d7456c74d21303c7851f158833d48fb265876923edcb2d0194104ed \ + --hash=sha256:3dd13feff00bddb0bd2d650cdb7338f815c1789a91a6f68fdc00e5c5ed40329b \ + --hash=sha256:4065c32b52f4b142f417af6f33a5024edc1336aa845b9d5a8d86071f6fcaac5a \ + --hash=sha256:51a4ba1256e9003a3acf508e3b4f4661bebd015b8180cc31849da222426ef585 \ + --hash=sha256:59888faac06403767c0cf8cfb3f4a777b2939b1fbd9f729299b5384f097f05ea \ + --hash=sha256:59c87886640574d8b14910840327f5cd15954e26ed0bbd4e7cef95fa5aef218f \ + --hash=sha256:610fc7d6db6c56a244c2701575f6851461753c60f73f2de89c79bbf1cc807f33 \ + --hash=sha256:70aeadeecb281ea901bf4230c6222af0248c41044d6f57401a614ea59d96d145 \ + --hash=sha256:71e1296d5e66c59cd2c0f2d72dc476d42afe02aeddc833d8e05630a0551dad7a \ + --hash=sha256:8fc7a49b440ea752cfdf1d51a586fd08d395ff7a5d555dc69e84b1939f7ddee3 \ + --hash=sha256:9b5c2afd2d6e3771d516045a6cfa11a8da9a60e3d128746a7fe9ab36dfe7221f \ + --hash=sha256:9c759051ebcb244d9d55ee791259ddd158188d15adee3c152502d3b69005e6bd \ + --hash=sha256:b4d1011fec5ec12aa7cc10c05a2f2f12dfa0adfe958e56ae38dc140614035804 \ + --hash=sha256:b4f1d6332339ecc61275bebd1f7b674098a66fea11a00c84d1c58851e618dc0d \ + --hash=sha256:c030cda3dc8e62b814831faa4eb93dd9a46498af8cd1d5c178c2de856972fd92 \ + --hash=sha256:c2e1f2012e56d61390c0e668c20c4fb0ae667c44d6f6a2eeea5d7148dcd3df9f \ + --hash=sha256:c37c77d6562074452120fc6c02ad86ec928f5710fbc435a181d69334b4de1d84 \ + --hash=sha256:c8149780c60f8fd02752d0429246088c6c04e234b895c4a42e1ea9b4de8d27fb \ + --hash=sha256:cbeeef1dc3c4299bd746b774f019de9e4672f7cc666c777cd5b409f0b746dac7 \ + --hash=sha256:e113878a446c6228669144ae8a56e268c91b7f1fafae927adc4879d9849e0ea7 \ + --hash=sha256:e21162bf941b85c0cda08224dade5def9360f53b09f9f259adb85fc7dd0e7b35 \ + --hash=sha256:fb6934ef4744becbda3143d30c6604718871495a5e36c408431bf33d9c146889 \ + # via cryptography +chardet==3.0.4 \ + --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ + --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \ + # via requests +cryptography==2.6.1 \ + --hash=sha256:066f815f1fe46020877c5983a7e747ae140f517f1b09030ec098503575265ce1 \ + --hash=sha256:210210d9df0afba9e000636e97810117dc55b7157c903a55716bb73e3ae07705 \ + --hash=sha256:26c821cbeb683facb966045e2064303029d572a87ee69ca5a1bf54bf55f93ca6 \ + --hash=sha256:2afb83308dc5c5255149ff7d3fb9964f7c9ee3d59b603ec18ccf5b0a8852e2b1 \ + --hash=sha256:2db34e5c45988f36f7a08a7ab2b69638994a8923853dec2d4af121f689c66dc8 \ + --hash=sha256:409c4653e0f719fa78febcb71ac417076ae5e20160aec7270c91d009837b9151 \ + --hash=sha256:45a4f4cf4f4e6a55c8128f8b76b4c057027b27d4c67e3fe157fa02f27e37830d \ + --hash=sha256:48eab46ef38faf1031e58dfcc9c3e71756a1108f4c9c966150b605d4a1a7f659 \ + --hash=sha256:6b9e0ae298ab20d371fc26e2129fd683cfc0cfde4d157c6341722de645146537 \ + --hash=sha256:6c4778afe50f413707f604828c1ad1ff81fadf6c110cb669579dea7e2e98a75e \ + --hash=sha256:8c33fb99025d353c9520141f8bc989c2134a1f76bac6369cea060812f5b5c2bb \ + --hash=sha256:9873a1760a274b620a135054b756f9f218fa61ca030e42df31b409f0fb738b6c \ + --hash=sha256:9b069768c627f3f5623b1cbd3248c5e7e92aec62f4c98827059eed7053138cc9 \ + --hash=sha256:9e4ce27a507e4886efbd3c32d120db5089b906979a4debf1d5939ec01b9dd6c5 \ + --hash=sha256:acb424eaca214cb08735f1a744eceb97d014de6530c1ea23beb86d9c6f13c2ad \ + --hash=sha256:c8181c7d77388fe26ab8418bb088b1a1ef5fde058c6926790c8a0a3d94075a4a \ + --hash=sha256:d4afbb0840f489b60f5a580a41a1b9c3622e08ecb5eec8614d4fb4cd914c4460 \ + --hash=sha256:d9ed28030797c00f4bc43c86bf819266c76a5ea61d006cd4078a93ebf7da6bfd \ + --hash=sha256:e603aa7bb52e4e8ed4119a58a03b60323918467ef209e6ff9db3ac382e5cf2c6 \ + # via pypsrp +docutils==0.14 \ + --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \ + --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \ + --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6 \ + # via botocore +idna==2.8 \ + --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ + --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \ + # via requests +jmespath==0.9.4 \ + --hash=sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6 \ + --hash=sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c \ + # via boto3, botocore +ntlm-auth==1.2.0 \ + --hash=sha256:7bc02a3fbdfee7275d3dc20fce8028ed8eb6d32364637f28be9e9ae9160c6d5c \ + --hash=sha256:9b13eaf88f16a831637d75236a93d60c0049536715aafbf8190ba58a590b023e \ + # via pypsrp +pycparser==2.19 \ + --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \ + # via cffi +pypsrp==0.3.1 \ + --hash=sha256:309853380fe086090a03cc6662a778ee69b1cae355ae4a932859034fd76e9d0b \ + --hash=sha256:90f946254f547dc3493cea8493c819ab87e152a755797c93aa2668678ba8ae85 +python-dateutil==2.8.0 \ + --hash=sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb \ + --hash=sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e \ + # via botocore +requests==2.21.0 \ + --hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \ + --hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b \ + # via pypsrp +s3transfer==0.2.0 \ + --hash=sha256:7b9ad3213bff7d357f888e0fab5101b56fa1a0548ee77d121c3a3dbfbef4cb2e \ + --hash=sha256:f23d5cb7d862b104401d9021fc82e5fa0e0cf57b7660a1331425aab0c691d021 \ + # via boto3 +six==1.12.0 \ + --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ + --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \ + # via cryptography, pypsrp, python-dateutil +urllib3==1.24.1 \ + --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \ + --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \ + # via botocore, requests diff --git a/contrib/automation/requirements.txt.in b/contrib/automation/requirements.txt.in new file mode 100644 --- /dev/null +++ b/contrib/automation/requirements.txt.in @@ -0,0 +1,2 @@ +boto3 +pypsrp diff --git a/tests/test-check-code.t b/tests/test-check-code.t --- a/tests/test-check-code.t +++ b/tests/test-check-code.t @@ -12,6 +12,11 @@ > -X hgext/fsmonitor/pywatchman \ > -X mercurial/thirdparty \ > | sed 's-\\-/-g' | "$check_code" --warnings --per-file=0 - || false + Skipping contrib/automation/hgautomation/__init__.py it has no-che?k-code (glob) + Skipping contrib/automation/hgautomation/aws.py it has no-che?k-code (glob) + Skipping contrib/automation/hgautomation/cli.py it has no-che?k-code (glob) + Skipping contrib/automation/hgautomation/windows.py it has no-che?k-code (glob) + Skipping contrib/automation/hgautomation/winrm.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/downloads.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/inno.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/py2exe.py it has no-che?k-code (glob) diff --git a/tests/test-check-module-imports.t b/tests/test-check-module-imports.t --- a/tests/test-check-module-imports.t +++ b/tests/test-check-module-imports.t @@ -18,6 +18,7 @@ > 'tests/**.t' \ > -X hgweb.cgi \ > -X setup.py \ + > -X contrib/automation/ \ > -X contrib/debugshell.py \ > -X contrib/hgweb.fcgi \ > -X contrib/packaging/hg-docker \ diff --git a/tests/test-check-py3-compat.t b/tests/test-check-py3-compat.t --- a/tests/test-check-py3-compat.t +++ b/tests/test-check-py3-compat.t @@ -5,6 +5,7 @@ #if no-py3 $ testrepohg files 'set:(**.py)' \ + > -X contrib/automation/ \ > -X contrib/packaging/hgpackaging/ \ > -X contrib/packaging/inno/ \ > -X contrib/packaging/wix/ \