( )⚙ D8483 automation: support building Python 3 Inno installers

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D8483

automation: support building Python 3 Inno installers
ClosedPublic

Authored by indygreg on Apr 24 2020, 7:23 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG802ee93c205d: automation: support building Python 3 Inno installers
rHG3943dab662c6: automation: support building Python 3 Inno installers
Summary

The core packaging code now supports building Python 3 installers
using PyOxidizer. Let's teach the automation code to invoke it so
that we produce both Python 2 and Python 3 based exe installers.

When publishing the artifacts, the Python 3 versions are preferred
over the Python 2 versions given their higher weight (10 versus 9).
This may be a controversial change. But I think making Python 3
the default is warranted, as it is the future. The Python 2 installers
are still fully supported and can be installed should issues with
Python 3 arise.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

indygreg created this revision.Apr 24 2020, 7:23 PM
indygreg added a commit: rHG3943dab662c6: automation: support building Python 3 Inno installers.Apr 30 2020, 1:53 AM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG3943dab662c6: automation: support building Python 3 Inno installers (authored by indygreg). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 21249

contrib/automation/hgautomation/cli.py

def bootstrap_windows_dev(hga: HGAutomation, aws_region, base_image_name): def bootstrap_windows_dev(hga: HGAutomation, aws_region, base_image_name):
c = hga.aws_connection(aws_region) c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
print('Windows development AMI available as %s' % image.id) print('Windows development AMI available as %s' % image.id)
def build_inno( def build_inno(
hga: HGAutomation, aws_region, arch, revision, version, base_image_name hga: HGAutomation,
aws_region,
python_version,
arch,
revision,
version,
base_image_name,
): ):
c = hga.aws_connection(aws_region) c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
DIST_PATH.mkdir(exist_ok=True) DIST_PATH.mkdir(exist_ok=True)
with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts: with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
instance = insts[0] instance = insts[0]
windows.synchronize_hg(SOURCE_ROOT, revision, instance) windows.synchronize_hg(SOURCE_ROOT, revision, instance)
for py_version in python_version:
for a in arch: for a in arch:
windows.build_inno_installer( windows.build_inno_installer(
instance.winrm_client, a, DIST_PATH, version=version instance.winrm_client,
py_version,
a,
DIST_PATH,
version=version,
) )
def build_wix( def build_wix(
hga: HGAutomation, aws_region, arch, revision, version, base_image_name hga: HGAutomation, aws_region, arch, revision, version, base_image_name
): ):
c = hga.aws_connection(aws_region) c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name) image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
DIST_PATH.mkdir(exist_ok=True) DIST_PATH.mkdir(exist_ok=True)
windows.purge_hg(winrm_client) windows.purge_hg(winrm_client)
windows.build_wheel( windows.build_wheel(
winrm_client, winrm_client,
python_version=py_version, python_version=py_version,
arch=arch, arch=arch,
dest_path=DIST_PATH, dest_path=DIST_PATH,
) )
for py_version in (2, 3):
for arch in ('x86', 'x64'): for arch in ('x86', 'x64'):
windows.purge_hg(winrm_client) windows.purge_hg(winrm_client)
windows.build_inno_installer( windows.build_inno_installer(
winrm_client, arch, DIST_PATH, version=version winrm_client, py_version, arch, DIST_PATH, version=version
) )
for arch in ('x86', 'x64'):
windows.purge_hg(winrm_client) windows.purge_hg(winrm_client)
windows.build_wix_installer( windows.build_wix_installer(
winrm_client, arch, DIST_PATH, version=version winrm_client, arch, DIST_PATH, version=version
) )
def terminate_ec2_instances(hga: HGAutomation, aws_region): def terminate_ec2_instances(hga: HGAutomation, aws_region):
c = hga.aws_connection(aws_region, ensure_ec2_state=False) c = hga.aws_connection(aws_region, ensure_ec2_state=False)
default=aws.WINDOWS_BASE_IMAGE_NAME, default=aws.WINDOWS_BASE_IMAGE_NAME,
) )
sp.set_defaults(func=build_all_windows_packages) sp.set_defaults(func=build_all_windows_packages)
sp = subparsers.add_parser( sp = subparsers.add_parser(
'build-inno', help='Build Inno Setup installer(s)', 'build-inno', help='Build Inno Setup installer(s)',
) )
sp.add_argument( sp.add_argument(
'--python-version',
help='Which version of Python to target',
choices={2, 3},
type=int,
nargs='*',
default=[3],
)
sp.add_argument(
'--arch', '--arch',
help='Architecture to build for', help='Architecture to build for',
choices={'x86', 'x64'}, choices={'x86', 'x64'},
nargs='*', nargs='*',
default=['x64'], default=['x64'],
) )
sp.add_argument( sp.add_argument(
'--revision', help='Mercurial revision to build', default='.', '--revision', help='Mercurial revision to build', default='.',

contrib/automation/hgautomation/windows.py

hg.exe update -C {revision} hg.exe update -C {revision}
if ($LASTEXITCODE -ne 0) {{ if ($LASTEXITCODE -ne 0) {{
throw "process exited non-0: $LASTEXITCODE" throw "process exited non-0: $LASTEXITCODE"
}} }}
hg.exe log -r . hg.exe log -r .
Write-Output "updated Mercurial working directory to {revision}" Write-Output "updated Mercurial working directory to {revision}"
'''.lstrip() '''.lstrip()
BUILD_INNO = r''' BUILD_INNO_PYTHON3 = r'''
$Env:RUSTUP_HOME = "C:\hgdev\rustup"
$Env:CARGO_HOME = "C:\hgdev\cargo"
Set-Location C:\hgdev\src
C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py inno --pyoxidizer-target {pyoxidizer_target} --version {version}
if ($LASTEXITCODE -ne 0) {{
throw "process exited non-0: $LASTEXITCODE"
}}
'''
BUILD_INNO_PYTHON2 = r'''
Set-Location C:\hgdev\src Set-Location C:\hgdev\src
$python = "C:\hgdev\python27-{arch}\python.exe" $python = "C:\hgdev\python27-{arch}\python.exe"
C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py inno --python $python {extra_args} C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py inno --python $python {extra_args}
if ($LASTEXITCODE -ne 0) {{ if ($LASTEXITCODE -ne 0) {{
throw "process exited non-0: $LASTEXITCODE" throw "process exited non-0: $LASTEXITCODE"
}} }}
'''.lstrip() '''.lstrip()
WHEEL_FILENAME_PYTHON27_X86 = 'mercurial-{version}-cp27-cp27m-win32.whl' WHEEL_FILENAME_PYTHON27_X86 = 'mercurial-{version}-cp27-cp27m-win32.whl'
WHEEL_FILENAME_PYTHON27_X64 = 'mercurial-{version}-cp27-cp27m-win_amd64.whl' WHEEL_FILENAME_PYTHON27_X64 = 'mercurial-{version}-cp27-cp27m-win_amd64.whl'
WHEEL_FILENAME_PYTHON37_X86 = 'mercurial-{version}-cp37-cp37m-win32.whl' WHEEL_FILENAME_PYTHON37_X86 = 'mercurial-{version}-cp37-cp37m-win32.whl'
WHEEL_FILENAME_PYTHON37_X64 = 'mercurial-{version}-cp37-cp37m-win_amd64.whl' WHEEL_FILENAME_PYTHON37_X64 = 'mercurial-{version}-cp37-cp37m-win_amd64.whl'
WHEEL_FILENAME_PYTHON38_X86 = 'mercurial-{version}-cp38-cp38-win32.whl' WHEEL_FILENAME_PYTHON38_X86 = 'mercurial-{version}-cp38-cp38-win32.whl'
WHEEL_FILENAME_PYTHON38_X64 = 'mercurial-{version}-cp38-cp38-win_amd64.whl' WHEEL_FILENAME_PYTHON38_X64 = 'mercurial-{version}-cp38-cp38-win_amd64.whl'
X86_EXE_FILENAME = 'Mercurial-{version}-x86-python2.exe' EXE_FILENAME_PYTHON2_X86 = 'Mercurial-{version}-x86-python2.exe'
X64_EXE_FILENAME = 'Mercurial-{version}-x64-python2.exe' EXE_FILENAME_PYTHON2_X64 = 'Mercurial-{version}-x64-python2.exe'
EXE_FILENAME_PYTHON3_X86 = 'Mercurial-{version}-x86.exe'
EXE_FILENAME_PYTHON3_X64 = 'Mercurial-{version}-x64.exe'
X86_MSI_FILENAME = 'mercurial-{version}-x86-python2.msi' X86_MSI_FILENAME = 'mercurial-{version}-x86-python2.msi'
X64_MSI_FILENAME = 'mercurial-{version}-x64-python2.msi' X64_MSI_FILENAME = 'mercurial-{version}-x64-python2.msi'
MERCURIAL_SCM_BASE_URL = 'https://mercurial-scm.org/release/windows' MERCURIAL_SCM_BASE_URL = 'https://mercurial-scm.org/release/windows'
X86_USER_AGENT_PATTERN = '.*Windows.*' X86_USER_AGENT_PATTERN = '.*Windows.*'
X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*' X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*'
X86_EXE_DESCRIPTION = ( EXE_PYTHON2_X86_DESCRIPTION = (
'Mercurial {version} Inno Setup installer - x86 Windows ' 'Mercurial {version} Inno Setup installer - x86 Windows (Python 2) '
'- does not require admin rights'
)
EXE_PYTHON2_X64_DESCRIPTION = (
'Mercurial {version} Inno Setup installer - x64 Windows (Python 2) '
'- does not require admin rights'
)
# TODO remove Python version once Python 2 is dropped.
EXE_PYTHON3_X86_DESCRIPTION = (
'Mercurial {version} Inno Setup installer - x86 Windows (Python 3) '
'- does not require admin rights' '- does not require admin rights'
) )
X64_EXE_DESCRIPTION = ( EXE_PYTHON3_X64_DESCRIPTION = (
'Mercurial {version} Inno Setup installer - x64 Windows ' 'Mercurial {version} Inno Setup installer - x64 Windows (Python 3) '
'- does not require admin rights' '- does not require admin rights'
) )
X86_MSI_DESCRIPTION = ( X86_MSI_DESCRIPTION = (
'Mercurial {version} MSI installer - x86 Windows ' '- requires admin rights' 'Mercurial {version} MSI installer - x86 Windows ' '- requires admin rights'
) )
X64_MSI_DESCRIPTION = ( X64_MSI_DESCRIPTION = (
'Mercurial {version} MSI installer - x64 Windows ' '- requires admin rights' 'Mercurial {version} MSI installer - x64 Windows ' '- requires admin rights'
) )
latest = find_latest_dist(winrm_client, pattern) latest = find_latest_dist(winrm_client, pattern)
source = r'C:\hgdev\src\dist\%s' % latest source = r'C:\hgdev\src\dist\%s' % latest
dest = dest_path / latest dest = dest_path / latest
print('copying %s to %s' % (source, dest)) print('copying %s to %s' % (source, dest))
winrm_client.fetch(source, str(dest)) winrm_client.fetch(source, str(dest))
def build_inno_installer( def build_inno_installer(
winrm_client, arch: str, dest_path: pathlib.Path, version=None winrm_client,
python_version: int,
arch: str,
dest_path: pathlib.Path,
version=None,
): ):
"""Build the Inno Setup installer on a remote machine. """Build the Inno Setup installer on a remote machine.
Using a WinRM client, remote commands are executed to build Using a WinRM client, remote commands are executed to build
a Mercurial Inno Setup installer. a Mercurial Inno Setup installer.
""" """
print('building Inno Setup installer for %s' % arch) print(
'building Inno Setup installer for Python %d %s'
% (python_version, arch)
)
if python_version == 3:
# TODO fix this limitation in packaging code
if not version:
raise Exception(
"version string is required when building for Python 3"
)
if arch == "x86":
target_triple = "i686-pc-windows-msvc"
elif arch == "x64":
target_triple = "x86_64-pc-windows-msvc"
else:
raise Exception("unhandled arch: %s" % arch)
ps = BUILD_INNO_PYTHON3.format(
pyoxidizer_target=target_triple, version=version,
)
else:
extra_args = [] extra_args = []
if version: if version:
extra_args.extend(['--version', version]) extra_args.extend(['--version', version])
ps = get_vc_prefix(arch) + BUILD_INNO.format( ps = get_vc_prefix(arch) + BUILD_INNO_PYTHON2.format(
arch=arch, extra_args=' '.join(extra_args) arch=arch, extra_args=' '.join(extra_args)
) )
run_powershell(winrm_client, ps) run_powershell(winrm_client, ps)
copy_latest_dist(winrm_client, '*.exe', dest_path) copy_latest_dist(winrm_client, '*.exe', dest_path)
def build_wheel( def build_wheel(
winrm_client, python_version: str, arch: str, dest_path: pathlib.Path winrm_client, python_version: str, arch: str, dest_path: pathlib.Path
): ):
"""Build Python wheels on a remote machine. """Build Python wheels on a remote machine.
def resolve_all_artifacts(dist_path: pathlib.Path, version: str): def resolve_all_artifacts(dist_path: pathlib.Path, version: str):
return ( return (
dist_path / WHEEL_FILENAME_PYTHON27_X86.format(version=version), dist_path / WHEEL_FILENAME_PYTHON27_X86.format(version=version),
dist_path / WHEEL_FILENAME_PYTHON27_X64.format(version=version), dist_path / WHEEL_FILENAME_PYTHON27_X64.format(version=version),
dist_path / WHEEL_FILENAME_PYTHON37_X86.format(version=version), dist_path / WHEEL_FILENAME_PYTHON37_X86.format(version=version),
dist_path / WHEEL_FILENAME_PYTHON37_X64.format(version=version), dist_path / WHEEL_FILENAME_PYTHON37_X64.format(version=version),
dist_path / WHEEL_FILENAME_PYTHON38_X86.format(version=version), dist_path / WHEEL_FILENAME_PYTHON38_X86.format(version=version),
dist_path / WHEEL_FILENAME_PYTHON38_X64.format(version=version), dist_path / WHEEL_FILENAME_PYTHON38_X64.format(version=version),
dist_path / X86_EXE_FILENAME.format(version=version), dist_path / EXE_FILENAME_PYTHON2_X86.format(version=version),
dist_path / X64_EXE_FILENAME.format(version=version), dist_path / EXE_FILENAME_PYTHON2_X64.format(version=version),
dist_path / EXE_FILENAME_PYTHON3_X86.format(version=version),
dist_path / EXE_FILENAME_PYTHON3_X64.format(version=version),
dist_path / X86_MSI_FILENAME.format(version=version), dist_path / X86_MSI_FILENAME.format(version=version),
dist_path / X64_MSI_FILENAME.format(version=version), dist_path / X64_MSI_FILENAME.format(version=version),
) )
def generate_latest_dat(version: str): def generate_latest_dat(version: str):
x86_exe_filename = X86_EXE_FILENAME.format(version=version) python2_x86_exe_filename = EXE_FILENAME_PYTHON2_X86.format(version=version)
x64_exe_filename = X64_EXE_FILENAME.format(version=version) python2_x64_exe_filename = EXE_FILENAME_PYTHON2_X64.format(version=version)
python3_x86_exe_filename = EXE_FILENAME_PYTHON3_X86.format(version=version)
python3_x64_exe_filename = EXE_FILENAME_PYTHON3_X64.format(version=version)
x86_msi_filename = X86_MSI_FILENAME.format(version=version) x86_msi_filename = X86_MSI_FILENAME.format(version=version)
x64_msi_filename = X64_MSI_FILENAME.format(version=version) x64_msi_filename = X64_MSI_FILENAME.format(version=version)
entries = ( entries = (
( (
'10', '10',
version, version,
X86_USER_AGENT_PATTERN, X86_USER_AGENT_PATTERN,
'%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_exe_filename), '%s/%s' % (MERCURIAL_SCM_BASE_URL, python3_x86_exe_filename),
X86_EXE_DESCRIPTION.format(version=version), EXE_PYTHON3_X86_DESCRIPTION.format(version=version),
), ),
( (
'10', '10',
version, version,
X64_USER_AGENT_PATTERN, X64_USER_AGENT_PATTERN,
'%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_exe_filename), '%s/%s' % (MERCURIAL_SCM_BASE_URL, python3_x64_exe_filename),
X64_EXE_DESCRIPTION.format(version=version), EXE_PYTHON3_X64_DESCRIPTION.format(version=version),
),
(
'9',
version,
X86_USER_AGENT_PATTERN,
'%s/%s' % (MERCURIAL_SCM_BASE_URL, python2_x86_exe_filename),
EXE_PYTHON2_X86_DESCRIPTION.format(version=version),
),
(
'9',
version,
X64_USER_AGENT_PATTERN,
'%s/%s' % (MERCURIAL_SCM_BASE_URL, python2_x64_exe_filename),
EXE_PYTHON2_X64_DESCRIPTION.format(version=version),
), ),
( (
'10', '10',
version, version,
X86_USER_AGENT_PATTERN, X86_USER_AGENT_PATTERN,
'%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_msi_filename), '%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_msi_filename),
X86_MSI_DESCRIPTION.format(version=version), X86_MSI_DESCRIPTION.format(version=version),
), ),