This is an archive of the discontinued Mercurial Phabricator instance.

Differential D8484

automation: support building Python 3 MSI installers
ClosedPublic

Authored by indygreg on Apr 24 2020, 7:23 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG5e788dc7fb5d: automation: support building Python 3 MSI installers
rHG5c1f356b108e: automation: support building Python 3 MSI installers
Summary

This is very similar to what we just did for Inno.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

indygreg created this revision.Apr 24 2020, 7:23 PM
martinvonz added a subscriber: martinvonz.Apr 30 2020, 12:52 AM

I'm queuing this series. Thanks for all your work on packaging!

indygreg added a commit: rHG5c1f356b108e: automation: support building Python 3 MSI installers.Apr 30 2020, 1:54 AM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG5c1f356b108e: automation: support building Python 3 MSI installers (authored by indygreg). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 21250

contrib/automation/hgautomation/cli.py

py_version, py_version,
a, a,
DIST_PATH, DIST_PATH,
version=version, version=version,
) )
def build_wix( def build_wix(
hga: HGAutomation, aws_region, arch, revision, version, base_image_name hga: HGAutomation,
aws_region,
python_version,
arch,
revision,
version,
base_image_name,
): ):
c = hga.aws_connection(aws_region) c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
DIST_PATH.mkdir(exist_ok=True) DIST_PATH.mkdir(exist_ok=True)
with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
instance = insts[0] instance = insts[0]
windows.synchronize_hg(SOURCE_ROOT, revision, instance) windows.synchronize_hg(SOURCE_ROOT, revision, instance)
for py_version in python_version:
for a in arch: for a in arch:
windows.build_wix_installer( windows.build_wix_installer(
instance.winrm_client, a, DIST_PATH, version=version instance.winrm_client,
py_version,
a,
DIST_PATH,
version=version,
) )
def build_windows_wheel( def build_windows_wheel(
hga: HGAutomation, hga: HGAutomation,
aws_region, aws_region,
python_version, python_version,
arch, arch,
revision, revision,
) )
for py_version in (2, 3): for py_version in (2, 3):
for arch in ('x86', 'x64'): for arch in ('x86', 'x64'):
windows.purge_hg(winrm_client) windows.purge_hg(winrm_client)
windows.build_inno_installer( windows.build_inno_installer(
winrm_client, py_version, arch, DIST_PATH, version=version winrm_client, py_version, arch, DIST_PATH, version=version
) )
for arch in ('x86', 'x64'):
windows.purge_hg(winrm_client)
windows.build_wix_installer( windows.build_wix_installer(
winrm_client, arch, DIST_PATH, version=version winrm_client, py_version, arch, DIST_PATH, version=version
) )
def terminate_ec2_instances(hga: HGAutomation, aws_region): def terminate_ec2_instances(hga: HGAutomation, aws_region):
c = hga.aws_connection(aws_region, ensure_ec2_state=False) c = hga.aws_connection(aws_region, ensure_ec2_state=False)
aws.terminate_ec2_instances(c.ec2resource) aws.terminate_ec2_instances(c.ec2resource)
def purge_ec2_resources(hga: HGAutomation, aws_region): def purge_ec2_resources(hga: HGAutomation, aws_region):
'--base-image-name', '--base-image-name',
help='AMI name of base image', help='AMI name of base image',
default=aws.WINDOWS_BASE_IMAGE_NAME, default=aws.WINDOWS_BASE_IMAGE_NAME,
) )
sp.set_defaults(func=build_windows_wheel) sp.set_defaults(func=build_windows_wheel)
sp = subparsers.add_parser('build-wix', help='Build WiX installer(s)') sp = subparsers.add_parser('build-wix', help='Build WiX installer(s)')
sp.add_argument( sp.add_argument(
'--python-version',
help='Which version of Python to target',
choices={2, 3},
type=int,
nargs='*',
default=[3],
)
sp.add_argument(
'--arch', '--arch',
help='Architecture to build for', help='Architecture to build for',
choices={'x86', 'x64'}, choices={'x86', 'x64'},
nargs='*', nargs='*',
default=['x64'], default=['x64'],
) )
sp.add_argument( sp.add_argument(
'--revision', help='Mercurial revision to build', default='.', '--revision', help='Mercurial revision to build', default='.',

contrib/automation/hgautomation/windows.py

BUILD_WHEEL = r''' BUILD_WHEEL = r'''
Set-Location C:\hgdev\src Set-Location C:\hgdev\src
C:\hgdev\python{python_version}-{arch}\python.exe -m pip wheel --wheel-dir dist . C:\hgdev\python{python_version}-{arch}\python.exe -m pip wheel --wheel-dir dist .
if ($LASTEXITCODE -ne 0) {{ if ($LASTEXITCODE -ne 0) {{
throw "process exited non-0: $LASTEXITCODE" throw "process exited non-0: $LASTEXITCODE"
}} }}
''' '''
BUILD_WIX = r''' BUILD_WIX_PYTHON3 = r'''
$Env:RUSTUP_HOME = "C:\hgdev\rustup"
$Env:CARGO_HOME = "C:\hgdev\cargo"
Set-Location C:\hgdev\src
C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py wix --pyoxidizer-target {pyoxidizer_target} --version {version}
if ($LASTEXITCODE -ne 0) {{
throw "process exited non-0: $LASTEXITCODE"
}}
'''
BUILD_WIX_PYTHON2 = r'''
Set-Location C:\hgdev\src Set-Location C:\hgdev\src
$python = "C:\hgdev\python27-{arch}\python.exe" $python = "C:\hgdev\python27-{arch}\python.exe"
C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py wix --python $python {extra_args} C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py wix --python $python {extra_args}
if ($LASTEXITCODE -ne 0) {{ if ($LASTEXITCODE -ne 0) {{
throw "process exited non-0: $LASTEXITCODE" throw "process exited non-0: $LASTEXITCODE"
}} }}
''' '''
WHEEL_FILENAME_PYTHON37_X64 = 'mercurial-{version}-cp37-cp37m-win_amd64.whl' WHEEL_FILENAME_PYTHON37_X64 = 'mercurial-{version}-cp37-cp37m-win_amd64.whl'
WHEEL_FILENAME_PYTHON38_X86 = 'mercurial-{version}-cp38-cp38-win32.whl' WHEEL_FILENAME_PYTHON38_X86 = 'mercurial-{version}-cp38-cp38-win32.whl'
WHEEL_FILENAME_PYTHON38_X64 = 'mercurial-{version}-cp38-cp38-win_amd64.whl' WHEEL_FILENAME_PYTHON38_X64 = 'mercurial-{version}-cp38-cp38-win_amd64.whl'
EXE_FILENAME_PYTHON2_X86 = 'Mercurial-{version}-x86-python2.exe' EXE_FILENAME_PYTHON2_X86 = 'Mercurial-{version}-x86-python2.exe'
EXE_FILENAME_PYTHON2_X64 = 'Mercurial-{version}-x64-python2.exe' EXE_FILENAME_PYTHON2_X64 = 'Mercurial-{version}-x64-python2.exe'
EXE_FILENAME_PYTHON3_X86 = 'Mercurial-{version}-x86.exe' EXE_FILENAME_PYTHON3_X86 = 'Mercurial-{version}-x86.exe'
EXE_FILENAME_PYTHON3_X64 = 'Mercurial-{version}-x64.exe' EXE_FILENAME_PYTHON3_X64 = 'Mercurial-{version}-x64.exe'
X86_MSI_FILENAME = 'mercurial-{version}-x86-python2.msi'
X64_MSI_FILENAME = 'mercurial-{version}-x64-python2.msi' MSI_FILENAME_PYTHON2_X86 = 'mercurial-{version}-x86-python2.msi'
MSI_FILENAME_PYTHON2_X64 = 'mercurial-{version}-x64-python2.msi'
MSI_FILENAME_PYTHON3_X86 = 'mercurial-{version}-x86.msi'
MSI_FILENAME_PYTHON3_X64 = 'mercurial-{version}-x64.msi'
MERCURIAL_SCM_BASE_URL = 'https://mercurial-scm.org/release/windows' MERCURIAL_SCM_BASE_URL = 'https://mercurial-scm.org/release/windows'
X86_USER_AGENT_PATTERN = '.*Windows.*' X86_USER_AGENT_PATTERN = '.*Windows.*'
X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*' X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*'
EXE_PYTHON2_X86_DESCRIPTION = ( EXE_PYTHON2_X86_DESCRIPTION = (
'Mercurial {version} Inno Setup installer - x86 Windows (Python 2) ' 'Mercurial {version} Inno Setup installer - x86 Windows (Python 2) '
'- does not require admin rights' '- does not require admin rights'
) )
EXE_PYTHON2_X64_DESCRIPTION = ( EXE_PYTHON2_X64_DESCRIPTION = (
'Mercurial {version} Inno Setup installer - x64 Windows (Python 2) ' 'Mercurial {version} Inno Setup installer - x64 Windows (Python 2) '
'- does not require admin rights' '- does not require admin rights'
) )
# TODO remove Python version once Python 2 is dropped. # TODO remove Python version once Python 2 is dropped.
EXE_PYTHON3_X86_DESCRIPTION = ( EXE_PYTHON3_X86_DESCRIPTION = (
'Mercurial {version} Inno Setup installer - x86 Windows (Python 3) ' 'Mercurial {version} Inno Setup installer - x86 Windows (Python 3) '
'- does not require admin rights' '- does not require admin rights'
) )
EXE_PYTHON3_X64_DESCRIPTION = ( EXE_PYTHON3_X64_DESCRIPTION = (
'Mercurial {version} Inno Setup installer - x64 Windows (Python 3) ' 'Mercurial {version} Inno Setup installer - x64 Windows (Python 3) '
'- does not require admin rights' '- does not require admin rights'
) )
X86_MSI_DESCRIPTION = ( MSI_PYTHON2_X86_DESCRIPTION = (
'Mercurial {version} MSI installer - x86 Windows ' '- requires admin rights' 'Mercurial {version} MSI installer - x86 Windows (Python 2) '
) '- requires admin rights'
X64_MSI_DESCRIPTION = ( )
'Mercurial {version} MSI installer - x64 Windows ' '- requires admin rights' MSI_PYTHON2_X64_DESCRIPTION = (
'Mercurial {version} MSI installer - x64 Windows (Python 2) '
'- requires admin rights'
)
MSI_PYTHON3_X86_DESCRIPTION = (
'Mercurial {version} MSI installer - x86 Windows (Python 3) '
'- requires admin rights'
)
MSI_PYTHON3_X64_DESCRIPTION = (
'Mercurial {version} MSI installer - x64 Windows (Python 3) '
'- requires admin rights'
) )
def get_vc_prefix(arch): def get_vc_prefix(arch):
if arch == 'x86': if arch == 'x86':
return ACTIVATE_VC9_X86 return ACTIVATE_VC9_X86
elif arch == 'x64': elif arch == 'x64':
return ACTIVATE_VC9_AMD64 return ACTIVATE_VC9_AMD64
if python_version == "2.7": if python_version == "2.7":
ps = get_vc_prefix(arch) + ps ps = get_vc_prefix(arch) + ps
run_powershell(winrm_client, ps) run_powershell(winrm_client, ps)
copy_latest_dist(winrm_client, '*.whl', dest_path) copy_latest_dist(winrm_client, '*.whl', dest_path)
def build_wix_installer( def build_wix_installer(
winrm_client, arch: str, dest_path: pathlib.Path, version=None winrm_client,
python_version: int,
arch: str,
dest_path: pathlib.Path,
version=None,
): ):
"""Build the WiX installer on a remote machine. """Build the WiX installer on a remote machine.
Using a WinRM client, remote commands are executed to build a WiX installer. Using a WinRM client, remote commands are executed to build a WiX installer.
""" """
print('Building WiX installer for %s' % arch) print('Building WiX installer for Python %d %s' % (python_version, arch))
if python_version == 3:
# TODO fix this limitation in packaging code
if not version:
raise Exception(
"version string is required when building for Python 3"
)
if arch == "x86":
target_triple = "i686-pc-windows-msvc"
elif arch == "x64":
target_triple = "x86_64-pc-windows-msvc"
else:
raise Exception("unhandled arch: %s" % arch)
ps = BUILD_WIX_PYTHON3.format(
pyoxidizer_target=target_triple, version=version,
)
else:
extra_args = [] extra_args = []
if version: if version:
extra_args.extend(['--version', version]) extra_args.extend(['--version', version])
ps = get_vc_prefix(arch) + BUILD_WIX.format( ps = get_vc_prefix(arch) + BUILD_WIX_PYTHON2.format(
arch=arch, extra_args=' '.join(extra_args) arch=arch, extra_args=' '.join(extra_args)
) )
run_powershell(winrm_client, ps) run_powershell(winrm_client, ps)
copy_latest_dist(winrm_client, '*.msi', dest_path) copy_latest_dist(winrm_client, '*.msi', dest_path)
def run_tests(winrm_client, python_version, arch, test_flags=''): def run_tests(winrm_client, python_version, arch, test_flags=''):
"""Run tests on a remote Windows machine. """Run tests on a remote Windows machine.
``python_version`` is a ``X.Y`` string like ``2.7`` or ``3.7``. ``python_version`` is a ``X.Y`` string like ``2.7`` or ``3.7``.
dist_path / WHEEL_FILENAME_PYTHON37_X86.format(version=version), dist_path / WHEEL_FILENAME_PYTHON37_X86.format(version=version),
dist_path / WHEEL_FILENAME_PYTHON37_X64.format(version=version), dist_path / WHEEL_FILENAME_PYTHON37_X64.format(version=version),
dist_path / WHEEL_FILENAME_PYTHON38_X86.format(version=version), dist_path / WHEEL_FILENAME_PYTHON38_X86.format(version=version),
dist_path / WHEEL_FILENAME_PYTHON38_X64.format(version=version), dist_path / WHEEL_FILENAME_PYTHON38_X64.format(version=version),
dist_path / EXE_FILENAME_PYTHON2_X86.format(version=version), dist_path / EXE_FILENAME_PYTHON2_X86.format(version=version),
dist_path / EXE_FILENAME_PYTHON2_X64.format(version=version), dist_path / EXE_FILENAME_PYTHON2_X64.format(version=version),
dist_path / EXE_FILENAME_PYTHON3_X86.format(version=version), dist_path / EXE_FILENAME_PYTHON3_X86.format(version=version),
dist_path / EXE_FILENAME_PYTHON3_X64.format(version=version), dist_path / EXE_FILENAME_PYTHON3_X64.format(version=version),
dist_path / X86_MSI_FILENAME.format(version=version), dist_path / MSI_FILENAME_PYTHON2_X86.format(version=version),
dist_path / X64_MSI_FILENAME.format(version=version), dist_path / MSI_FILENAME_PYTHON2_X64.format(version=version),
dist_path / MSI_FILENAME_PYTHON3_X86.format(version=version),
dist_path / MSI_FILENAME_PYTHON3_X64.format(version=version),
) )
def generate_latest_dat(version: str): def generate_latest_dat(version: str):
python2_x86_exe_filename = EXE_FILENAME_PYTHON2_X86.format(version=version) python2_x86_exe_filename = EXE_FILENAME_PYTHON2_X86.format(version=version)
python2_x64_exe_filename = EXE_FILENAME_PYTHON2_X64.format(version=version) python2_x64_exe_filename = EXE_FILENAME_PYTHON2_X64.format(version=version)
python3_x86_exe_filename = EXE_FILENAME_PYTHON3_X86.format(version=version) python3_x86_exe_filename = EXE_FILENAME_PYTHON3_X86.format(version=version)
python3_x64_exe_filename = EXE_FILENAME_PYTHON3_X64.format(version=version) python3_x64_exe_filename = EXE_FILENAME_PYTHON3_X64.format(version=version)
x86_msi_filename = X86_MSI_FILENAME.format(version=version) python2_x86_msi_filename = MSI_FILENAME_PYTHON2_X86.format(version=version)
x64_msi_filename = X64_MSI_FILENAME.format(version=version) python2_x64_msi_filename = MSI_FILENAME_PYTHON2_X64.format(version=version)
python3_x86_msi_filename = MSI_FILENAME_PYTHON3_X86.format(version=version)
python3_x64_msi_filename = MSI_FILENAME_PYTHON3_X64.format(version=version)
entries = ( entries = (
( (
'10', '10',
version, version,
X86_USER_AGENT_PATTERN, X86_USER_AGENT_PATTERN,
'%s/%s' % (MERCURIAL_SCM_BASE_URL, python3_x86_exe_filename), '%s/%s' % (MERCURIAL_SCM_BASE_URL, python3_x86_exe_filename),
EXE_PYTHON3_X86_DESCRIPTION.format(version=version), EXE_PYTHON3_X86_DESCRIPTION.format(version=version),
X64_USER_AGENT_PATTERN, X64_USER_AGENT_PATTERN,
'%s/%s' % (MERCURIAL_SCM_BASE_URL, python2_x64_exe_filename), '%s/%s' % (MERCURIAL_SCM_BASE_URL, python2_x64_exe_filename),
EXE_PYTHON2_X64_DESCRIPTION.format(version=version), EXE_PYTHON2_X64_DESCRIPTION.format(version=version),
), ),
( (
'10', '10',
version, version,
X86_USER_AGENT_PATTERN, X86_USER_AGENT_PATTERN,
'%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_msi_filename), '%s/%s' % (MERCURIAL_SCM_BASE_URL, python3_x86_msi_filename),
X86_MSI_DESCRIPTION.format(version=version), MSI_PYTHON3_X86_DESCRIPTION.format(version=version),
), ),
( (
'10', '10',
version, version,
X64_USER_AGENT_PATTERN, X64_USER_AGENT_PATTERN,
'%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_msi_filename), '%s/%s' % (MERCURIAL_SCM_BASE_URL, python3_x64_msi_filename),
X64_MSI_DESCRIPTION.format(version=version), MSI_PYTHON3_X64_DESCRIPTION.format(version=version),
),
(
'9',
version,
X86_USER_AGENT_PATTERN,
'%s/%s' % (MERCURIAL_SCM_BASE_URL, python2_x86_msi_filename),
MSI_PYTHON2_X86_DESCRIPTION.format(version=version),
),
(
'9',
version,
X64_USER_AGENT_PATTERN,
'%s/%s' % (MERCURIAL_SCM_BASE_URL, python2_x64_msi_filename),
MSI_PYTHON2_X64_DESCRIPTION.format(version=version),
), ),
) )
lines = ['\t'.join(e) for e in entries] lines = ['\t'.join(e) for e in entries]
return '\n'.join(lines) + '\n' return '\n'.join(lines) + '\n'