diff --git a/contrib/automation/automation.py b/contrib/automation/automation.py --- a/contrib/automation/automation.py +++ b/contrib/automation/automation.py @@ -36,8 +36,13 @@ pip = venv_bin / 'pip' python = venv_bin / 'python' - args = [str(pip), 'install', '-r', str(REQUIREMENTS_TXT), - '--disable-pip-version-check'] + args = [ + str(pip), + 'install', + '-r', + str(REQUIREMENTS_TXT), + '--disable-pip-version-check', + ] if not venv_created: args.append('-q') @@ -45,8 +50,7 @@ subprocess.run(args, check=True) os.environ['HGAUTOMATION_BOOTSTRAPPED'] = '1' - os.environ['PATH'] = '%s%s%s' % ( - venv_bin, os.pathsep, os.environ['PATH']) + os.environ['PATH'] = '%s%s%s' % (venv_bin, os.pathsep, os.environ['PATH']) subprocess.run([str(python), __file__] + sys.argv[1:], check=True) diff --git a/contrib/automation/hgautomation/__init__.py b/contrib/automation/hgautomation/__init__.py --- a/contrib/automation/hgautomation/__init__.py +++ b/contrib/automation/hgautomation/__init__.py @@ -10,9 +10,7 @@ import pathlib import secrets -from .aws import ( - AWSConnection, -) +from .aws import AWSConnection class HGAutomation: @@ -53,7 +51,7 @@ return password - def aws_connection(self, region: str, ensure_ec2_state: bool=True): + def aws_connection(self, region: str, ensure_ec2_state: bool = True): """Obtain an AWSConnection instance bound to a specific region.""" return AWSConnection(self, region, ensure_ec2_state=ensure_ec2_state) diff --git a/contrib/automation/hgautomation/aws.py b/contrib/automation/hgautomation/aws.py --- a/contrib/automation/hgautomation/aws.py +++ b/contrib/automation/hgautomation/aws.py @@ -19,9 +19,7 @@ import boto3 import botocore.exceptions -from .linux import ( - BOOTSTRAP_DEBIAN, -) +from .linux import BOOTSTRAP_DEBIAN from .ssh import ( exec_command as ssh_exec_command, wait_for_ssh, @@ -32,10 +30,13 @@ ) -SOURCE_ROOT = pathlib.Path(os.path.abspath(__file__)).parent.parent.parent.parent +SOURCE_ROOT = pathlib.Path( + os.path.abspath(__file__) +).parent.parent.parent.parent -INSTALL_WINDOWS_DEPENDENCIES = (SOURCE_ROOT / 'contrib' / - 'install-windows-dependencies.ps1') +INSTALL_WINDOWS_DEPENDENCIES = ( + SOURCE_ROOT / 'contrib' / 'install-windows-dependencies.ps1' +) INSTANCE_TYPES_WITH_STORAGE = { @@ -107,7 +108,6 @@ 'Description': 'RDP from entire Internet', }, ], - }, { 'FromPort': 5985, @@ -119,7 +119,7 @@ 'Description': 'PowerShell Remoting (Windows Remote Management)', }, ], - } + }, ], }, } @@ -152,11 +152,7 @@ IAM_INSTANCE_PROFILES = { - 'ephemeral-ec2-1': { - 'roles': [ - 'ephemeral-ec2-role-1', - ], - } + 'ephemeral-ec2-1': {'roles': ['ephemeral-ec2-role-1',],} } @@ -226,7 +222,7 @@ class AWSConnection: """Manages the state of a connection with AWS.""" - def __init__(self, automation, region: str, ensure_ec2_state: bool=True): + def __init__(self, automation, region: str, ensure_ec2_state: bool = True): self.automation = automation self.local_state_path = automation.state_path @@ -257,10 +253,19 @@ # TODO use rsa package. res = subprocess.run( - ['openssl', 'pkcs8', '-in', str(p), '-nocrypt', '-topk8', - '-outform', 'DER'], + [ + 'openssl', + 'pkcs8', + '-in', + str(p), + '-nocrypt', + '-topk8', + '-outform', + 'DER', + ], capture_output=True, - check=True) + check=True, + ) sha1 = hashlib.sha1(res.stdout).hexdigest() return ':'.join(a + b for a, b in zip(sha1[::2], sha1[1::2])) @@ -271,7 +276,7 @@ for kpi in ec2resource.key_pairs.all(): if kpi.name.startswith(prefix): - remote_existing[kpi.name[len(prefix):]] = kpi.key_fingerprint + remote_existing[kpi.name[len(prefix) :]] = kpi.key_fingerprint # Validate that we have these keys locally. key_path = state_path / 'keys' @@ -297,7 +302,7 @@ if not f.startswith('keypair-') or not f.endswith('.pub'): continue - name = f[len('keypair-'):-len('.pub')] + name = f[len('keypair-') : -len('.pub')] pub_full = key_path / f priv_full = key_path / ('keypair-%s' % name) @@ -306,8 +311,9 @@ data = fh.read() if not data.startswith('ssh-rsa '): - print('unexpected format for key pair file: %s; removing' % - pub_full) + print( + 'unexpected format for key pair file: %s; removing' % pub_full + ) pub_full.unlink() priv_full.unlink() continue @@ -327,8 +333,10 @@ del local_existing[name] elif remote_existing[name] != local_existing[name]: - print('key fingerprint mismatch for %s; ' - 'removing from local and remote' % name) + print( + 'key fingerprint mismatch for %s; ' + 'removing from local and remote' % name + ) remove_local(name) remove_remote('%s%s' % (prefix, name)) del local_existing[name] @@ -356,15 +364,18 @@ subprocess.run( ['ssh-keygen', '-y', '-f', str(priv_full)], stdout=fh, - check=True) + check=True, + ) pub_full.chmod(0o0600) def delete_instance_profile(profile): for role in profile.roles: - print('removing role %s from instance profile %s' % (role.name, - profile.name)) + print( + 'removing role %s from instance profile %s' + % (role.name, profile.name) + ) profile.remove_role(RoleName=role.name) print('deleting instance profile %s' % profile.name) @@ -378,7 +389,7 @@ for profile in iamresource.instance_profiles.all(): if profile.name.startswith(prefix): - remote_profiles[profile.name[len(prefix):]] = profile + remote_profiles[profile.name[len(prefix) :]] = profile for name in sorted(set(remote_profiles) - set(IAM_INSTANCE_PROFILES)): delete_instance_profile(remote_profiles[name]) @@ -388,7 +399,7 @@ for role in iamresource.roles.all(): if role.name.startswith(prefix): - remote_roles[role.name[len(prefix):]] = role + remote_roles[role.name[len(prefix) :]] = role for name in sorted(set(remote_roles) - set(IAM_ROLES)): role = remote_roles[name] @@ -404,7 +415,8 @@ print('creating IAM instance profile %s' % actual) profile = iamresource.create_instance_profile( - InstanceProfileName=actual) + InstanceProfileName=actual + ) remote_profiles[name] = profile waiter = iamclient.get_waiter('instance_profile_exists') @@ -453,23 +465,12 @@ images = ec2resource.images.filter( Filters=[ - { - 'Name': 'owner-id', - 'Values': [owner_id], - }, - { - 'Name': 'state', - 'Values': ['available'], - }, - { - 'Name': 'image-type', - 'Values': ['machine'], - }, - { - 'Name': 'name', - 'Values': [name], - }, - ]) + {'Name': 'owner-id', 'Values': [owner_id],}, + {'Name': 'state', 'Values': ['available'],}, + {'Name': 'image-type', 'Values': ['machine'],}, + {'Name': 'name', 'Values': [name],}, + ] + ) for image in images: return image @@ -487,7 +488,7 @@ for group in ec2resource.security_groups.all(): if group.group_name.startswith(prefix): - existing[group.group_name[len(prefix):]] = group + existing[group.group_name[len(prefix) :]] = group purge = set(existing) - set(SECURITY_GROUPS) @@ -507,13 +508,10 @@ print('adding security group %s' % actual) group_res = ec2resource.create_security_group( - Description=group['description'], - GroupName=actual, + Description=group['description'], GroupName=actual, ) - group_res.authorize_ingress( - IpPermissions=group['ingress'], - ) + group_res.authorize_ingress(IpPermissions=group['ingress'],) security_groups[name] = group_res @@ -577,8 +575,10 @@ instance.reload() continue - print('public IP address for %s: %s' % ( - instance.id, instance.public_ip_address)) + print( + 'public IP address for %s: %s' + % (instance.id, instance.public_ip_address) + ) break @@ -603,10 +603,7 @@ while True: res = ssmclient.describe_instance_information( Filters=[ - { - 'Key': 'InstanceIds', - 'Values': [i.id for i in instances], - }, + {'Key': 'InstanceIds', 'Values': [i.id for i in instances],}, ], ) @@ -628,9 +625,7 @@ InstanceIds=[i.id for i in instances], DocumentName=document_name, Parameters=parameters, - CloudWatchOutputConfig={ - 'CloudWatchOutputEnabled': True, - }, + CloudWatchOutputConfig={'CloudWatchOutputEnabled': True,}, ) command_id = res['Command']['CommandId'] @@ -639,8 +634,7 @@ while True: try: res = ssmclient.get_command_invocation( - CommandId=command_id, - InstanceId=instance.id, + CommandId=command_id, InstanceId=instance.id, ) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'InvocationDoesNotExist': @@ -655,8 +649,9 @@ elif res['Status'] in ('Pending', 'InProgress', 'Delayed'): time.sleep(2) else: - raise Exception('command failed on %s: %s' % ( - instance.id, res['Status'])) + raise Exception( + 'command failed on %s: %s' % (instance.id, res['Status']) + ) @contextlib.contextmanager @@ -711,10 +706,12 @@ config['IamInstanceProfile'] = { 'Name': 'hg-ephemeral-ec2-1', } - config.setdefault('TagSpecifications', []).append({ - 'ResourceType': 'instance', - 'Tags': [{'Key': 'Name', 'Value': 'hg-temp-windows'}], - }) + config.setdefault('TagSpecifications', []).append( + { + 'ResourceType': 'instance', + 'Tags': [{'Key': 'Name', 'Value': 'hg-temp-windows'}], + } + ) config['UserData'] = WINDOWS_USER_DATA % password with temporary_ec2_instances(c.ec2resource, config) as instances: @@ -723,7 +720,9 @@ print('waiting for Windows Remote Management service...') for instance in instances: - client = wait_for_winrm(instance.public_ip_address, 'Administrator', password) + client = wait_for_winrm( + instance.public_ip_address, 'Administrator', password + ) print('established WinRM connection to %s' % instance.id) instance.winrm_client = client @@ -748,14 +747,17 @@ # Store a reference to a good image so it can be returned one the # image state is reconciled. images = ec2resource.images.filter( - Filters=[{'Name': 'name', 'Values': [name]}]) + Filters=[{'Name': 'name', 'Values': [name]}] + ) existing_image = None for image in images: if image.tags is None: - print('image %s for %s lacks required tags; removing' % ( - image.id, image.name)) + print( + 'image %s for %s lacks required tags; removing' + % (image.id, image.name) + ) remove_ami(ec2resource, image) else: tags = {t['Key']: t['Value'] for t in image.tags} @@ -763,15 +765,18 @@ if tags.get('HGIMAGEFINGERPRINT') == fingerprint: existing_image = image else: - print('image %s for %s has wrong fingerprint; removing' % ( - image.id, image.name)) + print( + 'image %s for %s has wrong fingerprint; removing' + % (image.id, image.name) + ) remove_ami(ec2resource, image) return existing_image -def create_ami_from_instance(ec2client, instance, name, description, - fingerprint): +def create_ami_from_instance( + ec2client, instance, name, description, fingerprint +): """Create an AMI from a running instance. Returns the ``ec2resource.Image`` representing the created AMI. @@ -779,29 +784,19 @@ instance.stop() ec2client.get_waiter('instance_stopped').wait( - InstanceIds=[instance.id], - WaiterConfig={ - 'Delay': 5, - }) + InstanceIds=[instance.id], WaiterConfig={'Delay': 5,} + ) print('%s is stopped' % instance.id) - image = instance.create_image( - Name=name, - Description=description, - ) + image = instance.create_image(Name=name, Description=description,) - image.create_tags(Tags=[ - { - 'Key': 'HGIMAGEFINGERPRINT', - 'Value': fingerprint, - }, - ]) + image.create_tags( + Tags=[{'Key': 'HGIMAGEFINGERPRINT', 'Value': fingerprint,},] + ) print('waiting for image %s' % image.id) - ec2client.get_waiter('image_available').wait( - ImageIds=[image.id], - ) + ec2client.get_waiter('image_available').wait(ImageIds=[image.id],) print('image %s available as %s' % (image.id, image.name)) @@ -827,9 +822,7 @@ ssh_username = 'admin' elif distro == 'debian10': image = find_image( - ec2resource, - DEBIAN_ACCOUNT_ID_2, - 'debian-10-amd64-20190909-10', + ec2resource, DEBIAN_ACCOUNT_ID_2, 'debian-10-amd64-20190909-10', ) ssh_username = 'admin' elif distro == 'ubuntu18.04': @@ -871,10 +864,12 @@ 'SecurityGroupIds': [c.security_groups['linux-dev-1'].id], } - requirements2_path = (pathlib.Path(__file__).parent.parent / - 'linux-requirements-py2.txt') - requirements3_path = (pathlib.Path(__file__).parent.parent / - 'linux-requirements-py3.txt') + requirements2_path = ( + pathlib.Path(__file__).parent.parent / 'linux-requirements-py2.txt' + ) + requirements3_path = ( + pathlib.Path(__file__).parent.parent / 'linux-requirements-py3.txt' + ) with requirements2_path.open('r', encoding='utf-8') as fh: requirements2 = fh.read() with requirements3_path.open('r', encoding='utf-8') as fh: @@ -882,12 +877,14 @@ # Compute a deterministic fingerprint to determine whether image needs to # be regenerated. - fingerprint = resolve_fingerprint({ - 'instance_config': config, - 'bootstrap_script': BOOTSTRAP_DEBIAN, - 'requirements_py2': requirements2, - 'requirements_py3': requirements3, - }) + fingerprint = resolve_fingerprint( + { + 'instance_config': config, + 'bootstrap_script': BOOTSTRAP_DEBIAN, + 'requirements_py2': requirements2, + 'requirements_py3': requirements3, + } + ) existing_image = find_and_reconcile_image(ec2resource, name, fingerprint) @@ -902,9 +899,11 @@ instance = instances[0] client = wait_for_ssh( - instance.public_ip_address, 22, + instance.public_ip_address, + 22, username=ssh_username, - key_filename=str(c.key_pair_path_private('automation'))) + key_filename=str(c.key_pair_path_private('automation')), + ) home = '/home/%s' % ssh_username @@ -926,8 +925,9 @@ fh.chmod(0o0700) print('executing bootstrap') - chan, stdin, stdout = ssh_exec_command(client, - '%s/bootstrap' % home) + chan, stdin, stdout = ssh_exec_command( + client, '%s/bootstrap' % home + ) stdin.close() for line in stdout: @@ -937,17 +937,28 @@ if res: raise Exception('non-0 exit from bootstrap: %d' % res) - print('bootstrap completed; stopping %s to create %s' % ( - instance.id, name)) + print( + 'bootstrap completed; stopping %s to create %s' + % (instance.id, name) + ) - return create_ami_from_instance(ec2client, instance, name, - 'Mercurial Linux development environment', - fingerprint) + return create_ami_from_instance( + ec2client, + instance, + name, + 'Mercurial Linux development environment', + fingerprint, + ) @contextlib.contextmanager -def temporary_linux_dev_instances(c: AWSConnection, image, instance_type, - prefix='hg-', ensure_extra_volume=False): +def temporary_linux_dev_instances( + c: AWSConnection, + image, + instance_type, + prefix='hg-', + ensure_extra_volume=False, +): """Create temporary Linux development EC2 instances. Context manager resolves to a list of ``ec2.Instance`` that were created @@ -979,8 +990,9 @@ # This is not an exhaustive list of instance types having instance storage. # But - if (ensure_extra_volume - and not instance_type.startswith(tuple(INSTANCE_TYPES_WITH_STORAGE))): + if ensure_extra_volume and not instance_type.startswith( + tuple(INSTANCE_TYPES_WITH_STORAGE) + ): main_device = block_device_mappings[0]['DeviceName'] if main_device == 'xvda': @@ -988,17 +1000,20 @@ elif main_device == '/dev/sda1': second_device = '/dev/sdb' else: - raise ValueError('unhandled primary EBS device name: %s' % - main_device) + raise ValueError( + 'unhandled primary EBS device name: %s' % main_device + ) - block_device_mappings.append({ - 'DeviceName': second_device, - 'Ebs': { - 'DeleteOnTermination': True, - 'VolumeSize': 8, - 'VolumeType': 'gp2', + block_device_mappings.append( + { + 'DeviceName': second_device, + 'Ebs': { + 'DeleteOnTermination': True, + 'VolumeSize': 8, + 'VolumeType': 'gp2', + }, } - }) + ) config = { 'BlockDeviceMappings': block_device_mappings, @@ -1019,9 +1034,11 @@ for instance in instances: client = wait_for_ssh( - instance.public_ip_address, 22, + instance.public_ip_address, + 22, username='hg', - key_filename=ssh_private_key_path) + key_filename=ssh_private_key_path, + ) instance.ssh_client = client instance.ssh_private_key_path = ssh_private_key_path @@ -1033,8 +1050,9 @@ instance.ssh_client.close() -def ensure_windows_dev_ami(c: AWSConnection, prefix='hg-', - base_image_name=WINDOWS_BASE_IMAGE_NAME): +def ensure_windows_dev_ami( + c: AWSConnection, prefix='hg-', base_image_name=WINDOWS_BASE_IMAGE_NAME +): """Ensure Windows Development AMI is available and up-to-date. If necessary, a modern AMI will be built by starting a temporary EC2 @@ -1100,13 +1118,15 @@ # Compute a deterministic fingerprint to determine whether image needs # to be regenerated. - fingerprint = resolve_fingerprint({ - 'instance_config': config, - 'user_data': WINDOWS_USER_DATA, - 'initial_bootstrap': WINDOWS_BOOTSTRAP_POWERSHELL, - 'bootstrap_commands': commands, - 'base_image_name': base_image_name, - }) + fingerprint = resolve_fingerprint( + { + 'instance_config': config, + 'user_data': WINDOWS_USER_DATA, + 'initial_bootstrap': WINDOWS_BOOTSTRAP_POWERSHELL, + 'bootstrap_commands': commands, + 'base_image_name': base_image_name, + } + ) existing_image = find_and_reconcile_image(ec2resource, name, fingerprint) @@ -1131,9 +1151,7 @@ ssmclient, [instance], 'AWS-RunPowerShellScript', - { - 'commands': WINDOWS_BOOTSTRAP_POWERSHELL.split('\n'), - }, + {'commands': WINDOWS_BOOTSTRAP_POWERSHELL.split('\n'),}, ) # Reboot so all updates are fully applied. @@ -1145,10 +1163,8 @@ print('rebooting instance %s' % instance.id) instance.stop() ec2client.get_waiter('instance_stopped').wait( - InstanceIds=[instance.id], - WaiterConfig={ - 'Delay': 5, - }) + InstanceIds=[instance.id], WaiterConfig={'Delay': 5,} + ) instance.start() wait_for_ip_addresses([instance]) @@ -1159,8 +1175,11 @@ # TODO figure out a workaround. print('waiting for Windows Remote Management to come back...') - client = wait_for_winrm(instance.public_ip_address, 'Administrator', - c.automation.default_password()) + client = wait_for_winrm( + instance.public_ip_address, + 'Administrator', + c.automation.default_password(), + ) print('established WinRM connection to %s' % instance.id) instance.winrm_client = client @@ -1168,14 +1187,23 @@ run_powershell(instance.winrm_client, '\n'.join(commands)) print('bootstrap completed; stopping %s to create image' % instance.id) - return create_ami_from_instance(ec2client, instance, name, - 'Mercurial Windows development environment', - fingerprint) + return create_ami_from_instance( + ec2client, + instance, + name, + 'Mercurial Windows development environment', + fingerprint, + ) @contextlib.contextmanager -def temporary_windows_dev_instances(c: AWSConnection, image, instance_type, - prefix='hg-', disable_antivirus=False): +def temporary_windows_dev_instances( + c: AWSConnection, + image, + instance_type, + prefix='hg-', + disable_antivirus=False, +): """Create a temporary Windows development EC2 instance. Context manager resolves to the list of ``EC2.Instance`` that were created. @@ -1205,6 +1233,7 @@ for instance in instances: run_powershell( instance.winrm_client, - 'Set-MpPreference -DisableRealtimeMonitoring $true') + 'Set-MpPreference -DisableRealtimeMonitoring $true', + ) yield instances diff --git a/contrib/automation/hgautomation/cli.py b/contrib/automation/hgautomation/cli.py --- a/contrib/automation/hgautomation/cli.py +++ b/contrib/automation/hgautomation/cli.py @@ -22,12 +22,15 @@ ) -SOURCE_ROOT = pathlib.Path(os.path.abspath(__file__)).parent.parent.parent.parent +SOURCE_ROOT = pathlib.Path( + os.path.abspath(__file__) +).parent.parent.parent.parent DIST_PATH = SOURCE_ROOT / 'dist' -def bootstrap_linux_dev(hga: HGAutomation, aws_region, distros=None, - parallel=False): +def bootstrap_linux_dev( + hga: HGAutomation, aws_region, distros=None, parallel=False +): c = hga.aws_connection(aws_region) if distros: @@ -59,8 +62,9 @@ print('Windows development AMI available as %s' % image.id) -def build_inno(hga: HGAutomation, aws_region, arch, revision, version, - base_image_name): +def build_inno( + hga: HGAutomation, aws_region, arch, revision, version, base_image_name +): c = hga.aws_connection(aws_region) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) DIST_PATH.mkdir(exist_ok=True) @@ -71,13 +75,14 @@ windows.synchronize_hg(SOURCE_ROOT, revision, instance) for a in arch: - windows.build_inno_installer(instance.winrm_client, a, - DIST_PATH, - version=version) + windows.build_inno_installer( + instance.winrm_client, a, DIST_PATH, version=version + ) -def build_wix(hga: HGAutomation, aws_region, arch, revision, version, - base_image_name): +def build_wix( + hga: HGAutomation, aws_region, arch, revision, version, base_image_name +): c = hga.aws_connection(aws_region) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) DIST_PATH.mkdir(exist_ok=True) @@ -88,12 +93,14 @@ windows.synchronize_hg(SOURCE_ROOT, revision, instance) for a in arch: - windows.build_wix_installer(instance.winrm_client, a, - DIST_PATH, version=version) + windows.build_wix_installer( + instance.winrm_client, a, DIST_PATH, version=version + ) -def build_windows_wheel(hga: HGAutomation, aws_region, arch, revision, - base_image_name): +def build_windows_wheel( + hga: HGAutomation, aws_region, arch, revision, base_image_name +): c = hga.aws_connection(aws_region) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) DIST_PATH.mkdir(exist_ok=True) @@ -107,8 +114,9 @@ windows.build_wheel(instance.winrm_client, a, DIST_PATH) -def build_all_windows_packages(hga: HGAutomation, aws_region, revision, - version, base_image_name): +def build_all_windows_packages( + hga: HGAutomation, aws_region, revision, version, base_image_name +): c = hga.aws_connection(aws_region) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) DIST_PATH.mkdir(exist_ok=True) @@ -124,11 +132,13 @@ windows.purge_hg(winrm_client) windows.build_wheel(winrm_client, arch, DIST_PATH) windows.purge_hg(winrm_client) - windows.build_inno_installer(winrm_client, arch, DIST_PATH, - version=version) + windows.build_inno_installer( + winrm_client, arch, DIST_PATH, version=version + ) windows.purge_hg(winrm_client) - windows.build_wix_installer(winrm_client, arch, DIST_PATH, - version=version) + windows.build_wix_installer( + winrm_client, arch, DIST_PATH, version=version + ) def terminate_ec2_instances(hga: HGAutomation, aws_region): @@ -141,8 +151,15 @@ aws.remove_resources(c) -def run_tests_linux(hga: HGAutomation, aws_region, instance_type, - python_version, test_flags, distro, filesystem): +def run_tests_linux( + hga: HGAutomation, + aws_region, + instance_type, + python_version, + test_flags, + distro, + filesystem, +): c = hga.aws_connection(aws_region) image = aws.ensure_linux_dev_ami(c, distro=distro) @@ -151,17 +168,17 @@ ensure_extra_volume = filesystem not in ('default', 'tmpfs') with aws.temporary_linux_dev_instances( - c, image, instance_type, - ensure_extra_volume=ensure_extra_volume) as insts: + c, image, instance_type, ensure_extra_volume=ensure_extra_volume + ) as insts: instance = insts[0] - linux.prepare_exec_environment(instance.ssh_client, - filesystem=filesystem) + linux.prepare_exec_environment( + instance.ssh_client, filesystem=filesystem + ) linux.synchronize_hg(SOURCE_ROOT, instance, '.') t_prepared = time.time() - linux.run_tests(instance.ssh_client, python_version, - test_flags) + linux.run_tests(instance.ssh_client, python_version, test_flags) t_done = time.time() t_setup = t_prepared - t_start @@ -169,29 +186,48 @@ print( 'total time: %.1fs; setup: %.1fs; tests: %.1fs; setup overhead: %.1f%%' - % (t_all, t_setup, t_done - t_prepared, t_setup / t_all * 100.0)) + % (t_all, t_setup, t_done - t_prepared, t_setup / t_all * 100.0) + ) -def run_tests_windows(hga: HGAutomation, aws_region, instance_type, - python_version, arch, test_flags, base_image_name): +def run_tests_windows( + hga: HGAutomation, + aws_region, + instance_type, + python_version, + arch, + test_flags, + base_image_name, +): c = hga.aws_connection(aws_region) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) - with aws.temporary_windows_dev_instances(c, image, instance_type, - disable_antivirus=True) as insts: + with aws.temporary_windows_dev_instances( + c, image, instance_type, disable_antivirus=True + ) as insts: instance = insts[0] windows.synchronize_hg(SOURCE_ROOT, '.', instance) - windows.run_tests(instance.winrm_client, python_version, arch, - test_flags) + windows.run_tests( + instance.winrm_client, python_version, arch, test_flags + ) -def publish_windows_artifacts(hg: HGAutomation, aws_region, version: str, - pypi: bool, mercurial_scm_org: bool, - ssh_username: str): - windows.publish_artifacts(DIST_PATH, version, - pypi=pypi, mercurial_scm_org=mercurial_scm_org, - ssh_username=ssh_username) +def publish_windows_artifacts( + hg: HGAutomation, + aws_region, + version: str, + pypi: bool, + mercurial_scm_org: bool, + ssh_username: str, +): + windows.publish_artifacts( + DIST_PATH, + version, + pypi=pypi, + mercurial_scm_org=mercurial_scm_org, + ssh_username=ssh_username, + ) def run_try(hga: HGAutomation, aws_region: str, rev: str): @@ -208,25 +244,21 @@ help='Path for local state files', ) parser.add_argument( - '--aws-region', - help='AWS region to use', - default='us-west-2', + '--aws-region', help='AWS region to use', default='us-west-2', ) subparsers = parser.add_subparsers() sp = subparsers.add_parser( - 'bootstrap-linux-dev', - help='Bootstrap Linux development environments', + 'bootstrap-linux-dev', help='Bootstrap Linux development environments', ) sp.add_argument( - '--distros', - help='Comma delimited list of distros to bootstrap', + '--distros', help='Comma delimited list of distros to bootstrap', ) sp.add_argument( '--parallel', action='store_true', - help='Generate AMIs in parallel (not CTRL-c safe)' + help='Generate AMIs in parallel (not CTRL-c safe)', ) sp.set_defaults(func=bootstrap_linux_dev) @@ -242,17 +274,13 @@ sp.set_defaults(func=bootstrap_windows_dev) sp = subparsers.add_parser( - 'build-all-windows-packages', - help='Build all Windows packages', + 'build-all-windows-packages', help='Build all Windows packages', ) sp.add_argument( - '--revision', - help='Mercurial revision to build', - default='.', + '--revision', help='Mercurial revision to build', default='.', ) sp.add_argument( - '--version', - help='Mercurial version string to use', + '--version', help='Mercurial version string to use', ) sp.add_argument( '--base-image-name', @@ -262,8 +290,7 @@ sp.set_defaults(func=build_all_windows_packages) sp = subparsers.add_parser( - 'build-inno', - help='Build Inno Setup installer(s)', + 'build-inno', help='Build Inno Setup installer(s)', ) sp.add_argument( '--arch', @@ -273,13 +300,10 @@ default=['x64'], ) sp.add_argument( - '--revision', - help='Mercurial revision to build', - default='.', + '--revision', help='Mercurial revision to build', default='.', ) sp.add_argument( - '--version', - help='Mercurial version string to use in installer', + '--version', help='Mercurial version string to use in installer', ) sp.add_argument( '--base-image-name', @@ -289,8 +313,7 @@ sp.set_defaults(func=build_inno) sp = subparsers.add_parser( - 'build-windows-wheel', - help='Build Windows wheel(s)', + 'build-windows-wheel', help='Build Windows wheel(s)', ) sp.add_argument( '--arch', @@ -300,9 +323,7 @@ default=['x64'], ) sp.add_argument( - '--revision', - help='Mercurial revision to build', - default='.', + '--revision', help='Mercurial revision to build', default='.', ) sp.add_argument( '--base-image-name', @@ -311,10 +332,7 @@ ) sp.set_defaults(func=build_windows_wheel) - sp = subparsers.add_parser( - 'build-wix', - help='Build WiX installer(s)' - ) + sp = subparsers.add_parser('build-wix', help='Build WiX installer(s)') sp.add_argument( '--arch', help='Architecture to build for', @@ -323,13 +341,10 @@ default=['x64'], ) sp.add_argument( - '--revision', - help='Mercurial revision to build', - default='.', + '--revision', help='Mercurial revision to build', default='.', ) sp.add_argument( - '--version', - help='Mercurial version string to use in installer', + '--version', help='Mercurial version string to use in installer', ) sp.add_argument( '--base-image-name', @@ -345,15 +360,11 @@ sp.set_defaults(func=terminate_ec2_instances) sp = subparsers.add_parser( - 'purge-ec2-resources', - help='Purge all EC2 resources managed by us', + 'purge-ec2-resources', help='Purge all EC2 resources managed by us', ) sp.set_defaults(func=purge_ec2_resources) - sp = subparsers.add_parser( - 'run-tests-linux', - help='Run tests on Linux', - ) + sp = subparsers.add_parser('run-tests-linux', help='Run tests on Linux',) sp.add_argument( '--distro', help='Linux distribution to run tests on', @@ -374,8 +385,18 @@ sp.add_argument( '--python-version', help='Python version to use', - choices={'system2', 'system3', '2.7', '3.5', '3.6', '3.7', '3.8', - 'pypy', 'pypy3.5', 'pypy3.6'}, + choices={ + 'system2', + 'system3', + '2.7', + '3.5', + '3.6', + '3.7', + '3.8', + 'pypy', + 'pypy3.5', + 'pypy3.6', + }, default='system2', ) sp.add_argument( @@ -386,13 +407,10 @@ sp.set_defaults(func=run_tests_linux) sp = subparsers.add_parser( - 'run-tests-windows', - help='Run tests on Windows', + 'run-tests-windows', help='Run tests on Windows', ) sp.add_argument( - '--instance-type', - help='EC2 instance type to use', - default='t3.medium', + '--instance-type', help='EC2 instance type to use', default='t3.medium', ) sp.add_argument( '--python-version', @@ -407,8 +425,7 @@ default='x64', ) sp.add_argument( - '--test-flags', - help='Extra command line flags to pass to run-tests.py', + '--test-flags', help='Extra command line flags to pass to run-tests.py', ) sp.add_argument( '--base-image-name', @@ -419,7 +436,7 @@ sp = subparsers.add_parser( 'publish-windows-artifacts', - help='Publish built Windows artifacts (wheels, installers, etc)' + help='Publish built Windows artifacts (wheels, installers, etc)', ) sp.add_argument( '--no-pypi', @@ -436,22 +453,17 @@ help='Skip uploading to www.mercurial-scm.org', ) sp.add_argument( - '--ssh-username', - help='SSH username for mercurial-scm.org', + '--ssh-username', help='SSH username for mercurial-scm.org', ) sp.add_argument( - 'version', - help='Mercurial version string to locate local packages', + 'version', help='Mercurial version string to locate local packages', ) sp.set_defaults(func=publish_windows_artifacts) sp = subparsers.add_parser( - 'try', - help='Run CI automation against a custom changeset' + 'try', help='Run CI automation against a custom changeset' ) - sp.add_argument('-r', '--rev', - default='.', - help='Revision to run CI on') + sp.add_argument('-r', '--rev', default='.', help='Revision to run CI on') sp.set_defaults(func=run_try) return parser diff --git a/contrib/automation/hgautomation/linux.py b/contrib/automation/hgautomation/linux.py --- a/contrib/automation/hgautomation/linux.py +++ b/contrib/automation/hgautomation/linux.py @@ -13,9 +13,7 @@ import subprocess import tempfile -from .ssh import ( - exec_command, -) +from .ssh import exec_command # Linux distributions that are supported. @@ -62,7 +60,9 @@ done pyenv global ${PYENV2_VERSIONS} ${PYENV3_VERSIONS} system -'''.lstrip().replace('\r\n', '\n') +'''.lstrip().replace( + '\r\n', '\n' +) INSTALL_RUST = r''' @@ -87,10 +87,13 @@ echo "${HG_SHA256} ${HG_TARBALL}" | sha256sum --check - /hgdev/venv-bootstrap/bin/pip install ${HG_TARBALL} -'''.lstrip().replace('\r\n', '\n') +'''.lstrip().replace( + '\r\n', '\n' +) -BOOTSTRAP_DEBIAN = r''' +BOOTSTRAP_DEBIAN = ( + r''' #!/bin/bash set -ex @@ -323,11 +326,14 @@ EOF sudo chown -R hg:hg /hgdev -'''.lstrip().format( - install_rust=INSTALL_RUST, - install_pythons=INSTALL_PYTHONS, - bootstrap_virtualenv=BOOTSTRAP_VIRTUALENV -).replace('\r\n', '\n') +'''.lstrip() + .format( + install_rust=INSTALL_RUST, + install_pythons=INSTALL_PYTHONS, + bootstrap_virtualenv=BOOTSTRAP_VIRTUALENV, + ) + .replace('\r\n', '\n') +) # Prepares /hgdev for operations. @@ -409,7 +415,9 @@ chown hg:hg /hgwork/tmp rsync -a /hgdev/src /hgwork/ -'''.lstrip().replace('\r\n', '\n') +'''.lstrip().replace( + '\r\n', '\n' +) HG_UPDATE_CLEAN = ''' @@ -421,7 +429,9 @@ ${HG} --config extensions.purge= purge --all ${HG} update -C $1 ${HG} log -r . -'''.lstrip().replace('\r\n', '\n') +'''.lstrip().replace( + '\r\n', '\n' +) def prepare_exec_environment(ssh_client, filesystem='default'): @@ -456,11 +466,12 @@ res = chan.recv_exit_status() if res: - raise Exception('non-0 exit code updating working directory; %d' - % res) + raise Exception('non-0 exit code updating working directory; %d' % res) -def synchronize_hg(source_path: pathlib.Path, ec2_instance, revision: str=None): +def synchronize_hg( + source_path: pathlib.Path, ec2_instance, revision: str = None +): """Synchronize a local Mercurial source path to remote EC2 instance.""" with tempfile.TemporaryDirectory() as temp_dir: @@ -482,8 +493,10 @@ fh.write(' IdentityFile %s\n' % ec2_instance.ssh_private_key_path) if not (source_path / '.hg').is_dir(): - raise Exception('%s is not a Mercurial repository; synchronization ' - 'not yet supported' % source_path) + raise Exception( + '%s is not a Mercurial repository; synchronization ' + 'not yet supported' % source_path + ) env = dict(os.environ) env['HGPLAIN'] = '1' @@ -493,17 +506,29 @@ res = subprocess.run( ['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'], - cwd=str(source_path), env=env, check=True, capture_output=True) + cwd=str(source_path), + env=env, + check=True, + capture_output=True, + ) full_revision = res.stdout.decode('ascii') args = [ - 'python2.7', str(hg_bin), - '--config', 'ui.ssh=ssh -F %s' % ssh_config, - '--config', 'ui.remotecmd=/hgdev/venv-bootstrap/bin/hg', + 'python2.7', + str(hg_bin), + '--config', + 'ui.ssh=ssh -F %s' % ssh_config, + '--config', + 'ui.remotecmd=/hgdev/venv-bootstrap/bin/hg', # Also ensure .hgtags changes are present so auto version # calculation works. - 'push', '-f', '-r', full_revision, '-r', 'file(.hgtags)', + 'push', + '-f', + '-r', + full_revision, + '-r', + 'file(.hgtags)', 'ssh://%s//hgwork/src' % public_ip, ] @@ -522,7 +547,8 @@ fh.chmod(0o0700) chan, stdin, stdout = exec_command( - ec2_instance.ssh_client, '/hgdev/hgup %s' % full_revision) + ec2_instance.ssh_client, '/hgdev/hgup %s' % full_revision + ) stdin.close() for line in stdout: @@ -531,8 +557,9 @@ res = chan.recv_exit_status() if res: - raise Exception('non-0 exit code updating working directory; %d' - % res) + raise Exception( + 'non-0 exit code updating working directory; %d' % res + ) def run_tests(ssh_client, python_version, test_flags=None): @@ -554,8 +581,8 @@ command = ( '/bin/sh -c "export TMPDIR=/hgwork/tmp; ' - 'cd /hgwork/src/tests && %s run-tests.py %s"' % ( - python, test_flags)) + 'cd /hgwork/src/tests && %s run-tests.py %s"' % (python, test_flags) + ) chan, stdin, stdout = exec_command(ssh_client, command) diff --git a/contrib/automation/hgautomation/pypi.py b/contrib/automation/hgautomation/pypi.py --- a/contrib/automation/hgautomation/pypi.py +++ b/contrib/automation/hgautomation/pypi.py @@ -7,12 +7,8 @@ # no-check-code because Python 3 native. -from twine.commands.upload import ( - upload as twine_upload, -) -from twine.settings import ( - Settings, -) +from twine.commands.upload import upload as twine_upload +from twine.settings import Settings def upload(paths): diff --git a/contrib/automation/hgautomation/ssh.py b/contrib/automation/hgautomation/ssh.py --- a/contrib/automation/hgautomation/ssh.py +++ b/contrib/automation/hgautomation/ssh.py @@ -11,14 +11,13 @@ import time import warnings -from cryptography.utils import ( - CryptographyDeprecationWarning, -) +from cryptography.utils import CryptographyDeprecationWarning import paramiko def wait_for_ssh(hostname, port, timeout=60, username=None, key_filename=None): """Wait for an SSH server to start on the specified host and port.""" + class IgnoreHostKeyPolicy(paramiko.MissingHostKeyPolicy): def missing_host_key(self, client, hostname, key): return @@ -28,17 +27,23 @@ # paramiko triggers a CryptographyDeprecationWarning in the cryptography # package. Let's suppress with warnings.catch_warnings(): - warnings.filterwarnings('ignore', - category=CryptographyDeprecationWarning) + warnings.filterwarnings( + 'ignore', category=CryptographyDeprecationWarning + ) while True: client = paramiko.SSHClient() client.set_missing_host_key_policy(IgnoreHostKeyPolicy()) try: - client.connect(hostname, port=port, username=username, - key_filename=key_filename, - timeout=5.0, allow_agent=False, - look_for_keys=False) + client.connect( + hostname, + port=port, + username=username, + key_filename=key_filename, + timeout=5.0, + allow_agent=False, + look_for_keys=False, + ) return client except socket.error: diff --git a/contrib/automation/hgautomation/windows.py b/contrib/automation/hgautomation/windows.py --- a/contrib/automation/hgautomation/windows.py +++ b/contrib/automation/hgautomation/windows.py @@ -15,12 +15,8 @@ import subprocess import tempfile -from .pypi import ( - upload as pypi_upload, -) -from .winrm import ( - run_powershell, -) +from .pypi import upload as pypi_upload +from .winrm import run_powershell # PowerShell commands to activate a Visual Studio 2008 environment. @@ -117,14 +113,21 @@ X86_USER_AGENT_PATTERN = '.*Windows.*' X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*' -X86_EXE_DESCRIPTION = ('Mercurial {version} Inno Setup installer - x86 Windows ' - '- does not require admin rights') -X64_EXE_DESCRIPTION = ('Mercurial {version} Inno Setup installer - x64 Windows ' - '- does not require admin rights') -X86_MSI_DESCRIPTION = ('Mercurial {version} MSI installer - x86 Windows ' - '- requires admin rights') -X64_MSI_DESCRIPTION = ('Mercurial {version} MSI installer - x64 Windows ' - '- requires admin rights') +X86_EXE_DESCRIPTION = ( + 'Mercurial {version} Inno Setup installer - x86 Windows ' + '- does not require admin rights' +) +X64_EXE_DESCRIPTION = ( + 'Mercurial {version} Inno Setup installer - x64 Windows ' + '- does not require admin rights' +) +X86_MSI_DESCRIPTION = ( + 'Mercurial {version} MSI installer - x86 Windows ' '- requires admin rights' +) +X64_MSI_DESCRIPTION = ( + 'Mercurial {version} MSI installer - x64 Windows ' '- requires admin rights' +) + def get_vc_prefix(arch): if arch == 'x86': @@ -158,10 +161,21 @@ ssh_dir.chmod(0o0700) # Generate SSH key to use for communication. - subprocess.run([ - 'ssh-keygen', '-t', 'rsa', '-b', '4096', '-N', '', - '-f', str(ssh_dir / 'id_rsa')], - check=True, capture_output=True) + subprocess.run( + [ + 'ssh-keygen', + '-t', + 'rsa', + '-b', + '4096', + '-N', + '', + '-f', + str(ssh_dir / 'id_rsa'), + ], + check=True, + capture_output=True, + ) # Add it to ~/.ssh/authorized_keys on remote. # This assumes the file doesn't already exist. @@ -182,8 +196,10 @@ fh.write(' IdentityFile %s\n' % (ssh_dir / 'id_rsa')) if not (hg_repo / '.hg').is_dir(): - raise Exception('%s is not a Mercurial repository; ' - 'synchronization not yet supported' % hg_repo) + raise Exception( + '%s is not a Mercurial repository; ' + 'synchronization not yet supported' % hg_repo + ) env = dict(os.environ) env['HGPLAIN'] = '1' @@ -193,17 +209,29 @@ res = subprocess.run( ['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'], - cwd=str(hg_repo), env=env, check=True, capture_output=True) + cwd=str(hg_repo), + env=env, + check=True, + capture_output=True, + ) full_revision = res.stdout.decode('ascii') args = [ - 'python2.7', hg_bin, - '--config', 'ui.ssh=ssh -F %s' % ssh_config, - '--config', 'ui.remotecmd=c:/hgdev/venv-bootstrap/Scripts/hg.exe', + 'python2.7', + hg_bin, + '--config', + 'ui.ssh=ssh -F %s' % ssh_config, + '--config', + 'ui.remotecmd=c:/hgdev/venv-bootstrap/Scripts/hg.exe', # Also ensure .hgtags changes are present so auto version # calculation works. - 'push', '-f', '-r', full_revision, '-r', 'file(.hgtags)', + 'push', + '-f', + '-r', + full_revision, + '-r', + 'file(.hgtags)', 'ssh://%s/c:/hgdev/src' % public_ip, ] @@ -213,8 +241,9 @@ if res.returncode not in (0, 1): res.check_returncode() - run_powershell(winrm_client, - HG_UPDATE_CLEAN.format(revision=full_revision)) + run_powershell( + winrm_client, HG_UPDATE_CLEAN.format(revision=full_revision) + ) # TODO detect dirty local working directory and synchronize accordingly. @@ -250,8 +279,9 @@ winrm_client.fetch(source, str(dest)) -def build_inno_installer(winrm_client, arch: str, dest_path: pathlib.Path, - version=None): +def build_inno_installer( + winrm_client, arch: str, dest_path: pathlib.Path, version=None +): """Build the Inno Setup installer on a remote machine. Using a WinRM client, remote commands are executed to build @@ -263,8 +293,9 @@ if version: extra_args.extend(['--version', version]) - ps = get_vc_prefix(arch) + BUILD_INNO.format(arch=arch, - extra_args=' '.join(extra_args)) + ps = get_vc_prefix(arch) + BUILD_INNO.format( + arch=arch, extra_args=' '.join(extra_args) + ) run_powershell(winrm_client, ps) copy_latest_dist(winrm_client, '*.exe', dest_path) @@ -281,8 +312,9 @@ copy_latest_dist(winrm_client, '*.whl', dest_path) -def build_wix_installer(winrm_client, arch: str, dest_path: pathlib.Path, - version=None): +def build_wix_installer( + winrm_client, arch: str, dest_path: pathlib.Path, version=None +): """Build the WiX installer on a remote machine. Using a WinRM client, remote commands are executed to build a WiX installer. @@ -292,8 +324,9 @@ if version: extra_args.extend(['--version', version]) - ps = get_vc_prefix(arch) + BUILD_WIX.format(arch=arch, - extra_args=' '.join(extra_args)) + ps = get_vc_prefix(arch) + BUILD_WIX.format( + arch=arch, extra_args=' '.join(extra_args) + ) run_powershell(winrm_client, ps) copy_latest_dist(winrm_client, '*.msi', dest_path) @@ -307,18 +340,16 @@ ``run-tests.py``. """ if not re.match(r'\d\.\d', python_version): - raise ValueError(r'python_version must be \d.\d; got %s' % - python_version) + raise ValueError( + r'python_version must be \d.\d; got %s' % python_version + ) if arch not in ('x86', 'x64'): raise ValueError('arch must be x86 or x64; got %s' % arch) python_path = 'python%s-%s' % (python_version.replace('.', ''), arch) - ps = RUN_TESTS.format( - python_path=python_path, - test_flags=test_flags or '', - ) + ps = RUN_TESTS.format(python_path=python_path, test_flags=test_flags or '',) run_powershell(winrm_client, ps) @@ -374,8 +405,8 @@ version, X64_USER_AGENT_PATTERN, '%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_msi_filename), - X64_MSI_DESCRIPTION.format(version=version) - ) + X64_MSI_DESCRIPTION.format(version=version), + ), ) lines = ['\t'.join(e) for e in entries] @@ -396,8 +427,9 @@ pypi_upload(wheel_paths) -def publish_artifacts_mercurial_scm_org(dist_path: pathlib.Path, version: str, - ssh_username=None): +def publish_artifacts_mercurial_scm_org( + dist_path: pathlib.Path, version: str, ssh_username=None +): """Publish Windows release artifacts to mercurial-scm.org.""" all_paths = resolve_all_artifacts(dist_path, version) @@ -436,7 +468,8 @@ now = datetime.datetime.utcnow() backup_path = dist_path / ( - 'latest-windows-%s.dat' % now.strftime('%Y%m%dT%H%M%S')) + 'latest-windows-%s.dat' % now.strftime('%Y%m%dT%H%M%S') + ) print('backing up %s to %s' % (latest_dat_path, backup_path)) with sftp.open(latest_dat_path, 'rb') as fh: @@ -453,9 +486,13 @@ fh.write(latest_dat_content.encode('ascii')) -def publish_artifacts(dist_path: pathlib.Path, version: str, - pypi=True, mercurial_scm_org=True, - ssh_username=None): +def publish_artifacts( + dist_path: pathlib.Path, + version: str, + pypi=True, + mercurial_scm_org=True, + ssh_username=None, +): """Publish Windows release artifacts. Files are found in `dist_path`. We will look for files with version string @@ -468,5 +505,6 @@ publish_artifacts_pypi(dist_path, version) if mercurial_scm_org: - publish_artifacts_mercurial_scm_org(dist_path, version, - ssh_username=ssh_username) + publish_artifacts_mercurial_scm_org( + dist_path, version, ssh_username=ssh_username + ) diff --git a/contrib/automation/hgautomation/winrm.py b/contrib/automation/hgautomation/winrm.py --- a/contrib/automation/hgautomation/winrm.py +++ b/contrib/automation/hgautomation/winrm.py @@ -11,9 +11,7 @@ import pprint import time -from pypsrp.client import ( - Client, -) +from pypsrp.client import Client from pypsrp.powershell import ( PowerShell, PSInvocationState, @@ -35,8 +33,13 @@ while True: try: - client = Client(host, username=username, password=password, - ssl=ssl, connection_timeout=5) + client = Client( + host, + username=username, + password=password, + ssl=ssl, + connection_timeout=5, + ) client.execute_ps("Write-Host 'Hello, World!'") return client except requests.exceptions.ConnectionError: @@ -78,5 +81,7 @@ print(format_object(o)) if ps.state == PSInvocationState.FAILED: - raise Exception('PowerShell execution failed: %s' % - ' '.join(map(format_object, ps.streams.error))) + raise Exception( + 'PowerShell execution failed: %s' + % ' '.join(map(format_object, ps.streams.error)) + ) diff --git a/contrib/bdiff-torture.py b/contrib/bdiff-torture.py --- a/contrib/bdiff-torture.py +++ b/contrib/bdiff-torture.py @@ -9,15 +9,20 @@ pycompat, ) + def reducetest(a, b): tries = 0 reductions = 0 print("reducing...") while tries < 1000: - a2 = "\n".join(l for l in a.splitlines() - if random.randint(0, 100) > 0) + "\n" - b2 = "\n".join(l for l in b.splitlines() - if random.randint(0, 100) > 0) + "\n" + a2 = ( + "\n".join(l for l in a.splitlines() if random.randint(0, 100) > 0) + + "\n" + ) + b2 = ( + "\n".join(l for l in b.splitlines() if random.randint(0, 100) > 0) + + "\n" + ) if a2 == a and b2 == b: continue if a2 == b2: @@ -32,8 +37,7 @@ a = a2 b = b2 - print("reduced:", reductions, len(a) + len(b), - repr(a), repr(b)) + print("reduced:", reductions, len(a) + len(b), repr(a), repr(b)) try: test1(a, b) except Exception as inst: @@ -41,6 +45,7 @@ sys.exit(0) + def test1(a, b): d = mdiff.textdiff(a, b) if not d: @@ -49,6 +54,7 @@ if c != b: raise ValueError("bad") + def testwrap(a, b): try: test1(a, b) @@ -57,10 +63,12 @@ print("exception:", inst) reducetest(a, b) + def test(a, b): testwrap(a, b) testwrap(b, a) + def rndtest(size, noise): a = [] src = " aaaaaaaabbbbccd" @@ -82,6 +90,7 @@ test(a, b) + maxvol = 10000 startsize = 2 while True: diff --git a/contrib/benchmarks/__init__.py b/contrib/benchmarks/__init__.py --- a/contrib/benchmarks/__init__.py +++ b/contrib/benchmarks/__init__.py @@ -44,15 +44,24 @@ util, ) -basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), - os.path.pardir, os.path.pardir)) +basedir = os.path.abspath( + os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir) +) reposdir = os.environ['REPOS_DIR'] -reposnames = [name for name in os.listdir(reposdir) - if os.path.isdir(os.path.join(reposdir, name, ".hg"))] +reposnames = [ + name + for name in os.listdir(reposdir) + if os.path.isdir(os.path.join(reposdir, name, ".hg")) +] if not reposnames: raise ValueError("No repositories found in $REPO_DIR") -outputre = re.compile((r'! wall (\d+.\d+) comb \d+.\d+ user \d+.\d+ sys ' - r'\d+.\d+ \(best of \d+\)')) +outputre = re.compile( + ( + r'! wall (\d+.\d+) comb \d+.\d+ user \d+.\d+ sys ' + r'\d+.\d+ \(best of \d+\)' + ) +) + def runperfcommand(reponame, command, *args, **kwargs): os.environ["HGRCPATH"] = os.environ.get("ASVHGRCPATH", "") @@ -63,8 +72,9 @@ else: ui = uimod.ui() repo = hg.repository(ui, os.path.join(reposdir, reponame)) - perfext = extensions.load(ui, 'perfext', - os.path.join(basedir, 'contrib', 'perf.py')) + perfext = extensions.load( + ui, 'perfext', os.path.join(basedir, 'contrib', 'perf.py') + ) cmd = getattr(perfext, command) ui.pushbuffer() cmd(ui, repo, *args, **kwargs) @@ -74,6 +84,7 @@ raise ValueError("Invalid output {0}".format(output)) return float(match.group(1)) + def perfbench(repos=reposnames, name=None, params=None): """decorator to declare ASV benchmark based on contrib/perf.py extension @@ -104,10 +115,12 @@ def wrapped(repo, *args): def perf(command, *a, **kw): return runperfcommand(repo, command, *a, **kw) + return func(perf, *args) wrapped.params = [p[1] for p in params] wrapped.param_names = [p[0] for p in params] wrapped.pretty_name = name return wrapped + return decorator diff --git a/contrib/benchmarks/perf.py b/contrib/benchmarks/perf.py --- a/contrib/benchmarks/perf.py +++ b/contrib/benchmarks/perf.py @@ -9,18 +9,22 @@ from . import perfbench + @perfbench() def track_tags(perf): return perf("perftags") + @perfbench() def track_status(perf): return perf("perfstatus", unknown=False) + @perfbench(params=[('rev', ['1000', '10000', 'tip'])]) def track_manifest(perf, rev): return perf("perfmanifest", rev) + @perfbench() def track_heads(perf): return perf("perfheads") diff --git a/contrib/benchmarks/revset.py b/contrib/benchmarks/revset.py --- a/contrib/benchmarks/revset.py +++ b/contrib/benchmarks/revset.py @@ -18,15 +18,16 @@ from . import basedir, perfbench + def createrevsetbenchmark(baseset, variants=None): if variants is None: # Default variants - variants = ["plain", "first", "last", "sort", "sort+first", - "sort+last"] - fname = "track_" + "_".join("".join([ - c if c in string.digits + string.letters else " " - for c in baseset - ]).split()) + variants = ["plain", "first", "last", "sort", "sort+first", "sort+last"] + fname = "track_" + "_".join( + "".join( + [c if c in string.digits + string.letters else " " for c in baseset] + ).split() + ) def wrap(fname, baseset): @perfbench(name=baseset, params=[("variant", variants)]) @@ -36,18 +37,21 @@ for var in variant.split("+"): revset = "%s(%s)" % (var, revset) return perf("perfrevset", revset) + f.__name__ = fname return f + return wrap(fname, baseset) + def initializerevsetbenchmarks(): mod = sys.modules[__name__] - with open(os.path.join(basedir, 'contrib', 'base-revsets.txt'), - 'rb') as fh: + with open(os.path.join(basedir, 'contrib', 'base-revsets.txt'), 'rb') as fh: for line in fh: baseset = line.strip() if baseset and not baseset.startswith('#'): func = createrevsetbenchmark(baseset) setattr(mod, func.__name__, func) + initializerevsetbenchmarks() diff --git a/contrib/byteify-strings.py b/contrib/byteify-strings.py --- a/contrib/byteify-strings.py +++ b/contrib/byteify-strings.py @@ -18,10 +18,13 @@ import token import tokenize + def adjusttokenpos(t, ofs): """Adjust start/end column of the given token""" - return t._replace(start=(t.start[0], t.start[1] + ofs), - end=(t.end[0], t.end[1] + ofs)) + return t._replace( + start=(t.start[0], t.start[1] + ofs), end=(t.end[0], t.end[1] + ofs) + ) + def replacetokens(tokens, opts): """Transform a stream of tokens from raw to Python 3. @@ -82,9 +85,8 @@ currtoken = tokens[k] while currtoken.type in (token.STRING, token.NEWLINE, tokenize.NL): k += 1 - if ( - currtoken.type == token.STRING - and currtoken.string.startswith(("'", '"')) + if currtoken.type == token.STRING and currtoken.string.startswith( + ("'", '"') ): sysstrtokens.add(currtoken) try: @@ -126,7 +128,7 @@ coloffset = -1 # column offset for the current line (-1: TBD) parens = [(0, 0, 0, -1)] # stack of (line, end-column, column-offset, type) ignorenextline = False # don't transform the next line - insideignoreblock = False # don't transform until turned off + insideignoreblock = False # don't transform until turned off for i, t in enumerate(tokens): # Compute the column offset for the current line, such that # the current line will be aligned to the last opening paren @@ -135,9 +137,9 @@ lastparen = parens[-1] if t.start[1] == lastparen[1]: coloffset = lastparen[2] - elif ( - t.start[1] + 1 == lastparen[1] - and lastparen[3] not in (token.NEWLINE, tokenize.NL) + elif t.start[1] + 1 == lastparen[1] and lastparen[3] not in ( + token.NEWLINE, + tokenize.NL, ): # fix misaligned indent of s/util.Abort/error.Abort/ coloffset = lastparen[2] + (lastparen[1] - t.start[1]) @@ -202,8 +204,7 @@ continue # String literal. Prefix to make a b'' string. - yield adjusttokenpos(t._replace(string='b%s' % t.string), - coloffset) + yield adjusttokenpos(t._replace(string='b%s' % t.string), coloffset) coldelta += 1 continue @@ -213,8 +214,13 @@ # *attr() builtins don't accept byte strings to 2nd argument. if fn in ( - 'getattr', 'setattr', 'hasattr', 'safehasattr', 'wrapfunction', - 'wrapclass', 'addattr' + 'getattr', + 'setattr', + 'hasattr', + 'safehasattr', + 'wrapfunction', + 'wrapclass', + 'addattr', ) and (opts['allow-attr-methods'] or not _isop(i - 1, '.')): arg1idx = _findargnofcall(1) if arg1idx is not None: @@ -241,18 +247,23 @@ _ensuresysstr(i + 4) # Looks like "if __name__ == '__main__'". - if (t.type == token.NAME and t.string == '__name__' - and _isop(i + 1, '==')): + if ( + t.type == token.NAME + and t.string == '__name__' + and _isop(i + 1, '==') + ): _ensuresysstr(i + 2) # Emit unmodified token. yield adjusttokenpos(t, coloffset) + def process(fin, fout, opts): tokens = tokenize.tokenize(fin.readline) tokens = replacetokens(list(tokens), opts) fout.write(tokenize.untokenize(tokens)) + def tryunlink(fname): try: os.unlink(fname) @@ -260,12 +271,14 @@ if err.errno != errno.ENOENT: raise + @contextlib.contextmanager def editinplace(fname): n = os.path.basename(fname) d = os.path.dirname(fname) - fp = tempfile.NamedTemporaryFile(prefix='.%s-' % n, suffix='~', dir=d, - delete=False) + fp = tempfile.NamedTemporaryFile( + prefix='.%s-' % n, suffix='~', dir=d, delete=False + ) try: yield fp fp.close() @@ -276,19 +289,37 @@ fp.close() tryunlink(fp.name) + def main(): ap = argparse.ArgumentParser() - ap.add_argument('--version', action='version', - version='Byteify strings 1.0') - ap.add_argument('-i', '--inplace', action='store_true', default=False, - help='edit files in place') - ap.add_argument('--dictiter', action='store_true', default=False, - help='rewrite iteritems() and itervalues()'), - ap.add_argument('--allow-attr-methods', action='store_true', - default=False, - help='also handle attr*() when they are methods'), - ap.add_argument('--treat-as-kwargs', nargs="+", default=[], - help="ignore kwargs-like objects"), + ap.add_argument( + '--version', action='version', version='Byteify strings 1.0' + ) + ap.add_argument( + '-i', + '--inplace', + action='store_true', + default=False, + help='edit files in place', + ) + ap.add_argument( + '--dictiter', + action='store_true', + default=False, + help='rewrite iteritems() and itervalues()', + ), + ap.add_argument( + '--allow-attr-methods', + action='store_true', + default=False, + help='also handle attr*() when they are methods', + ), + ap.add_argument( + '--treat-as-kwargs', + nargs="+", + default=[], + help="ignore kwargs-like objects", + ), ap.add_argument('files', metavar='FILE', nargs='+', help='source file') args = ap.parse_args() opts = { @@ -306,6 +337,7 @@ fout = sys.stdout.buffer process(fin, fout, opts) + if __name__ == '__main__': if sys.version_info.major < 3: print('This script must be run under Python 3.') diff --git a/contrib/casesmash.py b/contrib/casesmash.py --- a/contrib/casesmash.py +++ b/contrib/casesmash.py @@ -1,12 +1,12 @@ from __future__ import absolute_import import __builtin__ import os -from mercurial import ( - util, -) +from mercurial import util + def lowerwrap(scope, funcname): f = getattr(scope, funcname) + def wrap(fname, *args, **kwargs): d, base = os.path.split(fname) try: @@ -19,11 +19,14 @@ if fn.lower() == base.lower(): return f(os.path.join(d, fn), *args, **kwargs) return f(fname, *args, **kwargs) + scope.__dict__[funcname] = wrap + def normcase(path): return path.lower() + os.path.normcase = normcase for f in 'file open'.split(): diff --git a/contrib/catapipe.py b/contrib/catapipe.py --- a/contrib/catapipe.py +++ b/contrib/catapipe.py @@ -53,15 +53,28 @@ # Python version and OS timer = timeit.default_timer + def main(): parser = argparse.ArgumentParser() - parser.add_argument('pipe', type=str, nargs=1, - help='Path of named pipe to create and listen on.') - parser.add_argument('output', default='trace.json', type=str, nargs='?', - help='Path of json file to create where the traces ' - 'will be stored.') - parser.add_argument('--debug', default=False, action='store_true', - help='Print useful debug messages') + parser.add_argument( + 'pipe', + type=str, + nargs=1, + help='Path of named pipe to create and listen on.', + ) + parser.add_argument( + 'output', + default='trace.json', + type=str, + nargs='?', + help='Path of json file to create where the traces ' 'will be stored.', + ) + parser.add_argument( + '--debug', + default=False, + action='store_true', + help='Print useful debug messages', + ) args = parser.parse_args() fn = args.pipe[0] os.mkfifo(fn) @@ -86,19 +99,23 @@ payload_args = {} pid = _threadmap[session] ts_micros = (now - start) * 1000000 - out.write(json.dumps( - { - "name": label, - "cat": "misc", - "ph": _TYPEMAP[verb], - "ts": ts_micros, - "pid": pid, - "tid": 1, - "args": payload_args, - })) + out.write( + json.dumps( + { + "name": label, + "cat": "misc", + "ph": _TYPEMAP[verb], + "ts": ts_micros, + "pid": pid, + "tid": 1, + "args": payload_args, + } + ) + ) out.write(',\n') finally: os.unlink(fn) + if __name__ == '__main__': main() diff --git a/contrib/check-code.py b/contrib/check-code.py --- a/contrib/check-code.py +++ b/contrib/check-code.py @@ -26,11 +26,15 @@ import os import re import sys + if sys.version_info[0] < 3: opentext = open else: + def opentext(f): return open(f, encoding='latin1') + + try: xrange except NameError: @@ -42,6 +46,7 @@ import testparseutil + def compilere(pat, multiline=False): if multiline: pat = '(?m)' + pat @@ -52,10 +57,22 @@ pass return re.compile(pat) + # check "rules depending on implementation of repquote()" in each # patterns (especially pypats), before changing around repquote() -_repquotefixedmap = {' ': ' ', '\n': '\n', '.': 'p', ':': 'q', - '%': '%', '\\': 'b', '*': 'A', '+': 'P', '-': 'M'} +_repquotefixedmap = { + ' ': ' ', + '\n': '\n', + '.': 'p', + ':': 'q', + '%': '%', + '\\': 'b', + '*': 'A', + '+': 'P', + '-': 'M', +} + + def _repquoteencodechr(i): if i > 255: return 'u' @@ -67,13 +84,17 @@ if c.isdigit(): return 'n' return 'o' + + _repquotett = ''.join(_repquoteencodechr(i) for i in xrange(256)) + def repquote(m): t = m.group('text') t = t.translate(_repquotett) return m.group('quote') + t + m.group('quote') + def reppython(m): comment = m.group('comment') if comment: @@ -81,86 +102,103 @@ return "#" * l + comment[l:] return repquote(m) + def repcomment(m): return m.group(1) + "#" * len(m.group(2)) + def repccomment(m): t = re.sub(r"((?<=\n) )|\S", "x", m.group(2)) return m.group(1) + t + "*/" + def repcallspaces(m): t = re.sub(r"\n\s+", "\n", m.group(2)) return m.group(1) + t + def repinclude(m): return m.group(1) + "" + def rephere(m): t = re.sub(r"\S", "x", m.group(2)) return m.group(1) + t testpats = [ - [ - (r'\b(push|pop)d\b', "don't use 'pushd' or 'popd', use 'cd'"), - (r'\W\$?\(\([^\)\n]*\)\)', "don't use (()) or $(()), use 'expr'"), - (r'grep.*-q', "don't use 'grep -q', redirect to /dev/null"), - (r'(?'"), - (r'sha1sum', "don't use sha1sum, use $TESTDIR/md5sum.py"), - (r'\bls\b.*-\w*R', "don't use 'ls -R', use 'find'"), - (r'printf.*[^\\]\\([1-9]|0\d)', r"don't use 'printf \NNN', use Python"), - (r'printf.*[^\\]\\x', "don't use printf \\x, use Python"), - (r'rm -rf \*', "don't use naked rm -rf, target a directory"), - (r'\[[^\]]+==', '[ foo == bar ] is a bashism, use [ foo = bar ] instead'), - (r'(^|\|\s*)grep (-\w\s+)*[^|]*[(|]\w', - "use egrep for extended grep syntax"), - (r'(^|\|\s*)e?grep .*\\S', "don't use \\S in regular expression"), - (r'(?\n]>\s*\$HGRCPATH', "don't overwrite $HGRCPATH, append to it"), - (r'^stop\(\)', "don't use 'stop' as a shell function name"), - (r'(\[|\btest\b).*-e ', "don't use 'test -e', use 'test -f'"), - (r'\[\[\s+[^\]]*\]\]', "don't use '[[ ]]', use '[ ]'"), - (r'^alias\b.*=', "don't use alias, use a function"), - (r'if\s*!', "don't use '!' to negate exit status"), - (r'/dev/u?random', "don't use entropy, use /dev/zero"), - (r'do\s*true;\s*done', "don't use true as loop body, use sleep 0"), - (r'sed (-e )?\'(\d+|/[^/]*/)i(?!\\\n)', - "put a backslash-escaped newline after sed 'i' command"), - (r'^diff *-\w*[uU].*$\n(^ \$ |^$)', "prefix diff -u/-U with cmp"), - (r'^\s+(if)? diff *-\w*[uU]', "prefix diff -u/-U with cmp"), - (r'[\s="`\']python\s(?!bindings)', "don't use 'python', use '$PYTHON'"), - (r'seq ', "don't use 'seq', use $TESTDIR/seq.py"), - (r'\butil\.Abort\b', "directly use error.Abort"), - (r'\|&', "don't use |&, use 2>&1"), - (r'\w = +\w', "only one space after = allowed"), - (r'\bsed\b.*[^\\]\\n', "don't use 'sed ... \\n', use a \\ and a newline"), - (r'env.*-u', "don't use 'env -u VAR', use 'unset VAR'"), - (r'cp.* -r ', "don't use 'cp -r', use 'cp -R'"), - (r'grep.* -[ABC]', "don't use grep's context flags"), - (r'find.*-printf', - "don't use 'find -printf', it doesn't exist on BSD find(1)"), - (r'\$RANDOM ', "don't use bash-only $RANDOM to generate random values"), - ], - # warnings - [ - (r'^function', "don't use 'function', use old style"), - (r'^diff.*-\w*N', "don't use 'diff -N'"), - (r'\$PWD|\${PWD}', "don't use $PWD, use `pwd`"), - (r'^([^"\'\n]|("[^"\n]*")|(\'[^\'\n]*\'))*\^', "^ must be quoted"), - (r'kill (`|\$\()', "don't use kill, use killdaemons.py") - ] + [ + (r'\b(push|pop)d\b', "don't use 'pushd' or 'popd', use 'cd'"), + (r'\W\$?\(\([^\)\n]*\)\)', "don't use (()) or $(()), use 'expr'"), + (r'grep.*-q', "don't use 'grep -q', redirect to /dev/null"), + (r'(?'"), + (r'sha1sum', "don't use sha1sum, use $TESTDIR/md5sum.py"), + (r'\bls\b.*-\w*R', "don't use 'ls -R', use 'find'"), + (r'printf.*[^\\]\\([1-9]|0\d)', r"don't use 'printf \NNN', use Python"), + (r'printf.*[^\\]\\x', "don't use printf \\x, use Python"), + (r'rm -rf \*', "don't use naked rm -rf, target a directory"), + ( + r'\[[^\]]+==', + '[ foo == bar ] is a bashism, use [ foo = bar ] instead', + ), + ( + r'(^|\|\s*)grep (-\w\s+)*[^|]*[(|]\w', + "use egrep for extended grep syntax", + ), + (r'(^|\|\s*)e?grep .*\\S', "don't use \\S in regular expression"), + (r'(?\n]>\s*\$HGRCPATH', "don't overwrite $HGRCPATH, append to it"), + (r'^stop\(\)', "don't use 'stop' as a shell function name"), + (r'(\[|\btest\b).*-e ', "don't use 'test -e', use 'test -f'"), + (r'\[\[\s+[^\]]*\]\]', "don't use '[[ ]]', use '[ ]'"), + (r'^alias\b.*=', "don't use alias, use a function"), + (r'if\s*!', "don't use '!' to negate exit status"), + (r'/dev/u?random', "don't use entropy, use /dev/zero"), + (r'do\s*true;\s*done', "don't use true as loop body, use sleep 0"), + ( + r'sed (-e )?\'(\d+|/[^/]*/)i(?!\\\n)', + "put a backslash-escaped newline after sed 'i' command", + ), + (r'^diff *-\w*[uU].*$\n(^ \$ |^$)', "prefix diff -u/-U with cmp"), + (r'^\s+(if)? diff *-\w*[uU]', "prefix diff -u/-U with cmp"), + (r'[\s="`\']python\s(?!bindings)', "don't use 'python', use '$PYTHON'"), + (r'seq ', "don't use 'seq', use $TESTDIR/seq.py"), + (r'\butil\.Abort\b', "directly use error.Abort"), + (r'\|&', "don't use |&, use 2>&1"), + (r'\w = +\w', "only one space after = allowed"), + ( + r'\bsed\b.*[^\\]\\n', + "don't use 'sed ... \\n', use a \\ and a newline", + ), + (r'env.*-u', "don't use 'env -u VAR', use 'unset VAR'"), + (r'cp.* -r ', "don't use 'cp -r', use 'cp -R'"), + (r'grep.* -[ABC]', "don't use grep's context flags"), + ( + r'find.*-printf', + "don't use 'find -printf', it doesn't exist on BSD find(1)", + ), + (r'\$RANDOM ', "don't use bash-only $RANDOM to generate random values"), + ], + # warnings + [ + (r'^function', "don't use 'function', use old style"), + (r'^diff.*-\w*N', "don't use 'diff -N'"), + (r'\$PWD|\${PWD}', "don't use $PWD, use `pwd`"), + (r'^([^"\'\n]|("[^"\n]*")|(\'[^\'\n]*\'))*\^', "^ must be quoted"), + (r'kill (`|\$\()', "don't use kill, use killdaemons.py"), + ], ] testfilters = [ @@ -170,45 +208,72 @@ uprefix = r"^ \$ " utestpats = [ - [ - (r'^(\S.*|| [$>] \S.*)[ \t]\n', "trailing whitespace on non-output"), - (uprefix + r'.*\|\s*sed[^|>\n]*\n', - "use regex test output patterns instead of sed"), - (uprefix + r'(true|exit 0)', "explicit zero exit unnecessary"), - (uprefix + r'.*(? for continued lines"), - (uprefix + r'.*:\.\S*/', "x:.y in a path does not work on msys, rewrite " - "as x://.y, or see `hg log -k msys` for alternatives", r'-\S+:\.|' #-Rxxx - '# no-msys'), # in test-pull.t which is skipped on windows - (r'^ [^$>].*27\.0\.0\.1', - 'use $LOCALIP not an explicit loopback address'), - (r'^ (?![>$] ).*\$LOCALIP.*[^)]$', - 'mark $LOCALIP output lines with (glob) to help tests in BSD jails'), - (r'^ (cat|find): .*: \$ENOENT\$', - 'use test -f to test for file existence'), - (r'^ diff -[^ -]*p', - "don't use (external) diff with -p for portability"), - (r' readlink ', 'use readlink.py instead of readlink'), - (r'^ [-+][-+][-+] .* [-+]0000 \(glob\)', - "glob timezone field in diff output for portability"), - (r'^ @@ -[0-9]+ [+][0-9]+,[0-9]+ @@', - "use '@@ -N* +N,n @@ (glob)' style chunk header for portability"), - (r'^ @@ -[0-9]+,[0-9]+ [+][0-9]+ @@', - "use '@@ -N,n +N* @@ (glob)' style chunk header for portability"), - (r'^ @@ -[0-9]+ [+][0-9]+ @@', - "use '@@ -N* +N* @@ (glob)' style chunk header for portability"), - (uprefix + r'hg( +-[^ ]+( +[^ ]+)?)* +extdiff' - r'( +(-[^ po-]+|--(?!program|option)[^ ]+|[^-][^ ]*))*$', - "use $RUNTESTDIR/pdiff via extdiff (or -o/-p for false-positives)"), - ], - # warnings - [ - (r'^ (?!.*\$LOCALIP)[^*?/\n]* \(glob\)$', - "glob match with no glob string (?, *, /, and $LOCALIP)"), - ] + [ + (r'^(\S.*|| [$>] \S.*)[ \t]\n', "trailing whitespace on non-output"), + ( + uprefix + r'.*\|\s*sed[^|>\n]*\n', + "use regex test output patterns instead of sed", + ), + (uprefix + r'(true|exit 0)', "explicit zero exit unnecessary"), + (uprefix + r'.*(? for continued lines"), + ( + uprefix + r'.*:\.\S*/', + "x:.y in a path does not work on msys, rewrite " + "as x://.y, or see `hg log -k msys` for alternatives", + r'-\S+:\.|' '# no-msys', # -Rxxx + ), # in test-pull.t which is skipped on windows + ( + r'^ [^$>].*27\.0\.0\.1', + 'use $LOCALIP not an explicit loopback address', + ), + ( + r'^ (?![>$] ).*\$LOCALIP.*[^)]$', + 'mark $LOCALIP output lines with (glob) to help tests in BSD jails', + ), + ( + r'^ (cat|find): .*: \$ENOENT\$', + 'use test -f to test for file existence', + ), + ( + r'^ diff -[^ -]*p', + "don't use (external) diff with -p for portability", + ), + (r' readlink ', 'use readlink.py instead of readlink'), + ( + r'^ [-+][-+][-+] .* [-+]0000 \(glob\)', + "glob timezone field in diff output for portability", + ), + ( + r'^ @@ -[0-9]+ [+][0-9]+,[0-9]+ @@', + "use '@@ -N* +N,n @@ (glob)' style chunk header for portability", + ), + ( + r'^ @@ -[0-9]+,[0-9]+ [+][0-9]+ @@', + "use '@@ -N,n +N* @@ (glob)' style chunk header for portability", + ), + ( + r'^ @@ -[0-9]+ [+][0-9]+ @@', + "use '@@ -N* +N* @@ (glob)' style chunk header for portability", + ), + ( + uprefix + r'hg( +-[^ ]+( +[^ ]+)?)* +extdiff' + r'( +(-[^ po-]+|--(?!program|option)[^ ]+|[^-][^ ]*))*$', + "use $RUNTESTDIR/pdiff via extdiff (or -o/-p for false-positives)", + ), + ], + # warnings + [ + ( + r'^ (?!.*\$LOCALIP)[^*?/\n]* \(glob\)$', + "glob match with no glob string (?, *, /, and $LOCALIP)", + ), + ], ] # transform plain test rules to unified test's @@ -234,148 +299,212 @@ # common patterns to check *.py commonpypats = [ - [ - (r'\\$', 'Use () to wrap long lines in Python, not \\'), - (r'^\s*def\s*\w+\s*\(.*,\s*\(', - "tuple parameter unpacking not available in Python 3+"), - (r'lambda\s*\(.*,.*\)', - "tuple parameter unpacking not available in Python 3+"), - (r'(?\s', '<> operator is not available in Python 3+, use !='), - (r'^\s*\t', "don't use tabs"), - (r'\S;\s*\n', "semicolon"), - (r'[^_]_\([ \t\n]*(?:"[^"]+"[ \t\n+]*)+%', "don't use % inside _()"), - (r"[^_]_\([ \t\n]*(?:'[^']+'[ \t\n+]*)+%", "don't use % inside _()"), - (r'(\w|\)),\w', "missing whitespace after ,"), - (r'(\w|\))[+/*\-<>]\w', "missing whitespace in expression"), - (r'\w\s=\s\s+\w', "gratuitous whitespace after ="), - (( - # a line ending with a colon, potentially with trailing comments - r':([ \t]*#[^\n]*)?\n' - # one that is not a pass and not only a comment - r'(?P[ \t]+)[^#][^\n]+\n' - # more lines at the same indent level - r'((?P=indent)[^\n]+\n)*' - # a pass at the same indent level, which is bogus - r'(?P=indent)pass[ \t\n#]' - ), 'omit superfluous pass'), - (r'[^\n]\Z', "no trailing newline"), - (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"), -# (r'^\s+[^_ \n][^_. \n]+_[^_\n]+\s*=', -# "don't use underbars in identifiers"), - (r'^\s+(self\.)?[A-Za-z][a-z0-9]+[A-Z]\w* = ', - "don't use camelcase in identifiers", r'#.*camelcase-required'), - (r'^\s*(if|while|def|class|except|try)\s[^[\n]*:\s*[^\\n]#\s]+', - "linebreak after :"), - (r'class\s[^( \n]+:', "old-style class, use class foo(object)", - r'#.*old-style'), - (r'class\s[^( \n]+\(\):', - "class foo() creates old style object, use class foo(object)", - r'#.*old-style'), - (r'\b(%s)\(' % '|'.join(k for k in keyword.kwlist - if k not in ('print', 'exec')), - "Python keyword is not a function"), -# (r'class\s[A-Z][^\(]*\((?!Exception)', -# "don't capitalize non-exception classes"), -# (r'in range\(', "use xrange"), -# (r'^\s*print\s+', "avoid using print in core and extensions"), - (r'[\x80-\xff]', "non-ASCII character literal"), - (r'("\')\.format\(', "str.format() has no bytes counterpart, use %"), - (r'([\(\[][ \t]\S)|(\S[ \t][\)\]])', "gratuitous whitespace in () or []"), -# (r'\s\s=', "gratuitous whitespace before ="), - (r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\S', - "missing whitespace around operator"), - (r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\s', - "missing whitespace around operator"), - (r'\s(\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\S', - "missing whitespace around operator"), - (r'[^^+=*/!<>&| %-](\s=|=\s)[^= ]', - "wrong whitespace around ="), - (r'\([^()]*( =[^=]|[^<>!=]= )', - "no whitespace around = for named parameters"), - (r'raise [^,(]+, (\([^\)]+\)|[^,\(\)]+)$', - "don't use old-style two-argument raise, use Exception(message)"), - (r' is\s+(not\s+)?["\'0-9-]', "object comparison with literal"), - (r' [=!]=\s+(True|False|None)', - "comparison with singleton, use 'is' or 'is not' instead"), - (r'^\s*(while|if) [01]:', - "use True/False for constant Boolean expression"), - (r'^\s*if False(:| +and)', 'Remove code instead of using `if False`'), - (r'(?:(?\s', '<> operator is not available in Python 3+, use !='), + (r'^\s*\t', "don't use tabs"), + (r'\S;\s*\n', "semicolon"), + (r'[^_]_\([ \t\n]*(?:"[^"]+"[ \t\n+]*)+%', "don't use % inside _()"), + (r"[^_]_\([ \t\n]*(?:'[^']+'[ \t\n+]*)+%", "don't use % inside _()"), + (r'(\w|\)),\w', "missing whitespace after ,"), + (r'(\w|\))[+/*\-<>]\w', "missing whitespace in expression"), + (r'\w\s=\s\s+\w', "gratuitous whitespace after ="), + ( + ( + # a line ending with a colon, potentially with trailing comments + r':([ \t]*#[^\n]*)?\n' + # one that is not a pass and not only a comment + r'(?P[ \t]+)[^#][^\n]+\n' + # more lines at the same indent level + r'((?P=indent)[^\n]+\n)*' + # a pass at the same indent level, which is bogus + r'(?P=indent)pass[ \t\n#]' + ), + 'omit superfluous pass', + ), + (r'[^\n]\Z', "no trailing newline"), + (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"), + # (r'^\s+[^_ \n][^_. \n]+_[^_\n]+\s*=', + # "don't use underbars in identifiers"), + ( + r'^\s+(self\.)?[A-Za-z][a-z0-9]+[A-Z]\w* = ', + "don't use camelcase in identifiers", + r'#.*camelcase-required', + ), + ( + r'^\s*(if|while|def|class|except|try)\s[^[\n]*:\s*[^\\n]#\s]+', + "linebreak after :", + ), + ( + r'class\s[^( \n]+:', + "old-style class, use class foo(object)", + r'#.*old-style', + ), + ( + r'class\s[^( \n]+\(\):', + "class foo() creates old style object, use class foo(object)", + r'#.*old-style', + ), + ( + r'\b(%s)\(' + % '|'.join(k for k in keyword.kwlist if k not in ('print', 'exec')), + "Python keyword is not a function", + ), + # (r'class\s[A-Z][^\(]*\((?!Exception)', + # "don't capitalize non-exception classes"), + # (r'in range\(', "use xrange"), + # (r'^\s*print\s+', "avoid using print in core and extensions"), + (r'[\x80-\xff]', "non-ASCII character literal"), + (r'("\')\.format\(', "str.format() has no bytes counterpart, use %"), + ( + r'([\(\[][ \t]\S)|(\S[ \t][\)\]])', + "gratuitous whitespace in () or []", + ), + # (r'\s\s=', "gratuitous whitespace before ="), + ( + r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\S', + "missing whitespace around operator", + ), + ( + r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\s', + "missing whitespace around operator", + ), + ( + r'\s(\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\S', + "missing whitespace around operator", + ), + (r'[^^+=*/!<>&| %-](\s=|=\s)[^= ]', "wrong whitespace around ="), + ( + r'\([^()]*( =[^=]|[^<>!=]= )', + "no whitespace around = for named parameters", + ), + ( + r'raise [^,(]+, (\([^\)]+\)|[^,\(\)]+)$', + "don't use old-style two-argument raise, use Exception(message)", + ), + (r' is\s+(not\s+)?["\'0-9-]', "object comparison with literal"), + ( + r' [=!]=\s+(True|False|None)', + "comparison with singleton, use 'is' or 'is not' instead", + ), + ( + r'^\s*(while|if) [01]:', + "use True/False for constant Boolean expression", + ), + (r'^\s*if False(:| +and)', 'Remove code instead of using `if False`'), + ( + r'(?:(?\#.*?$)| + ( + r"""(?msx)(?P\#.*?$)| ((?P('''|\"\"\"|(?(([^\\]|\\.)*?)) - (?P=quote))""", reppython), + (?P=quote))""", + reppython, + ), ] # filters to convert normal *.py files -pyfilters = [ -] + commonpyfilters +pyfilters = [] + commonpyfilters # non-filter patterns pynfpats = [ [ - (r'pycompat\.osname\s*[=!]=\s*[\'"]nt[\'"]', "use pycompat.iswindows"), - (r'pycompat\.osname\s*[=!]=\s*[\'"]posix[\'"]', "use pycompat.isposix"), - (r'pycompat\.sysplatform\s*[!=]=\s*[\'"]darwin[\'"]', - "use pycompat.isdarwin"), + (r'pycompat\.osname\s*[=!]=\s*[\'"]nt[\'"]', "use pycompat.iswindows"), + (r'pycompat\.osname\s*[=!]=\s*[\'"]posix[\'"]', "use pycompat.isposix"), + ( + r'pycompat\.sysplatform\s*[!=]=\s*[\'"]darwin[\'"]', + "use pycompat.isdarwin", + ), ], # warnings [], ] # filters to convert *.py for embedded ones in test script -embeddedpyfilters = [ -] + commonpyfilters +embeddedpyfilters = [] + commonpyfilters # extension non-filter patterns pyextnfpats = [ @@ -445,41 +578,40 @@ txtfilters = [] txtpats = [ - [ - (r'\s$', 'trailing whitespace'), - ('.. note::[ \n][^\n]', 'add two newlines after note::') - ], - [] + [ + (r'\s$', 'trailing whitespace'), + ('.. note::[ \n][^\n]', 'add two newlines after note::'), + ], + [], ] cpats = [ - [ - (r'//', "don't use //-style comments"), - (r'\S\t', "don't use tabs except for indent"), - (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"), - (r'(while|if|do|for)\(', "use space after while/if/do/for"), - (r'return\(', "return is not a function"), - (r' ;', "no space before ;"), - (r'[^;] \)', "no space before )"), - (r'[)][{]', "space between ) and {"), - (r'\w+\* \w+', "use int *foo, not int* foo"), - (r'\W\([^\)]+\) \w+', "use (int)foo, not (int) foo"), - (r'\w+ (\+\+|--)', "use foo++, not foo ++"), - (r'\w,\w', "missing whitespace after ,"), - (r'^[^#]\w[+/*]\w', "missing whitespace in expression"), - (r'\w\s=\s\s+\w', "gratuitous whitespace after ="), - (r'^#\s+\w', "use #foo, not # foo"), - (r'[^\n]\Z', "no trailing newline"), - (r'^\s*#import\b', "use only #include in standard C code"), - (r'strcpy\(', "don't use strcpy, use strlcpy or memcpy"), - (r'strcat\(', "don't use strcat"), - - # rules depending on implementation of repquote() - ], - # warnings - [ - # rules depending on implementation of repquote() - ] + [ + (r'//', "don't use //-style comments"), + (r'\S\t', "don't use tabs except for indent"), + (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"), + (r'(while|if|do|for)\(', "use space after while/if/do/for"), + (r'return\(', "return is not a function"), + (r' ;', "no space before ;"), + (r'[^;] \)', "no space before )"), + (r'[)][{]', "space between ) and {"), + (r'\w+\* \w+', "use int *foo, not int* foo"), + (r'\W\([^\)]+\) \w+', "use (int)foo, not (int) foo"), + (r'\w+ (\+\+|--)', "use foo++, not foo ++"), + (r'\w,\w', "missing whitespace after ,"), + (r'^[^#]\w[+/*]\w', "missing whitespace in expression"), + (r'\w\s=\s\s+\w', "gratuitous whitespace after ="), + (r'^#\s+\w', "use #foo, not # foo"), + (r'[^\n]\Z', "no trailing newline"), + (r'^\s*#import\b', "use only #include in standard C code"), + (r'strcpy\(', "don't use strcpy, use strlcpy or memcpy"), + (r'strcat\(', "don't use strcat"), + # rules depending on implementation of repquote() + ], + # warnings + [ + # rules depending on implementation of repquote() + ], ] cfilters = [ @@ -490,82 +622,109 @@ ] inutilpats = [ - [ - (r'\bui\.', "don't use ui in util"), - ], - # warnings - [] + [(r'\bui\.', "don't use ui in util"),], + # warnings + [], ] inrevlogpats = [ - [ - (r'\brepo\.', "don't use repo in revlog"), - ], - # warnings - [] + [(r'\brepo\.', "don't use repo in revlog"),], + # warnings + [], ] webtemplatefilters = [] webtemplatepats = [ - [], - [ - (r'{desc(\|(?!websub|firstline)[^\|]*)+}', - 'follow desc keyword with either firstline or websub'), - ] + [], + [ + ( + r'{desc(\|(?!websub|firstline)[^\|]*)+}', + 'follow desc keyword with either firstline or websub', + ), + ], ] allfilesfilters = [] allfilespats = [ - [ - (r'(http|https)://[a-zA-Z0-9./]*selenic.com/', - 'use mercurial-scm.org domain URL'), - (r'mercurial@selenic\.com', - 'use mercurial-scm.org domain for mercurial ML address'), - (r'mercurial-devel@selenic\.com', - 'use mercurial-scm.org domain for mercurial-devel ML address'), - ], - # warnings - [], + [ + ( + r'(http|https)://[a-zA-Z0-9./]*selenic.com/', + 'use mercurial-scm.org domain URL', + ), + ( + r'mercurial@selenic\.com', + 'use mercurial-scm.org domain for mercurial ML address', + ), + ( + r'mercurial-devel@selenic\.com', + 'use mercurial-scm.org domain for mercurial-devel ML address', + ), + ], + # warnings + [], ] py3pats = [ - [ - (r'os\.environ', "use encoding.environ instead (py3)", r'#.*re-exports'), - (r'os\.name', "use pycompat.osname instead (py3)"), - (r'os\.getcwd', "use encoding.getcwd instead (py3)", r'#.*re-exports'), - (r'os\.sep', "use pycompat.ossep instead (py3)"), - (r'os\.pathsep', "use pycompat.ospathsep instead (py3)"), - (r'os\.altsep', "use pycompat.osaltsep instead (py3)"), - (r'sys\.platform', "use pycompat.sysplatform instead (py3)"), - (r'getopt\.getopt', "use pycompat.getoptb instead (py3)"), - (r'os\.getenv', "use encoding.environ.get instead"), - (r'os\.setenv', "modifying the environ dict is not preferred"), - (r'(?|int|bool|list)\( # First argument. @@ -23,9 +24,12 @@ # Second argument ['"](?P