diff --git a/contrib/packaging/hgpackaging/cli.py b/contrib/packaging/hgpackaging/cli.py --- a/contrib/packaging/hgpackaging/cli.py +++ b/contrib/packaging/hgpackaging/cli.py @@ -20,8 +20,11 @@ SOURCE_DIR = HERE.parent.parent.parent -def build_inno(python=None, iscc=None, version=None): - if not os.path.isabs(python): +def build_inno(pyoxidizer_target=None, python=None, iscc=None, version=None): + if not pyoxidizer_target and not python: + raise Exception("--python required unless building with PyOxidizer") + + if python and not os.path.isabs(python): raise Exception("--python arg must be an absolute path") if iscc: @@ -35,9 +38,14 @@ build_dir = SOURCE_DIR / "build" - inno.build( - SOURCE_DIR, build_dir, pathlib.Path(python), iscc, version=version, - ) + if pyoxidizer_target: + inno.build_with_pyoxidizer( + SOURCE_DIR, build_dir, pyoxidizer_target, iscc, version=version + ) + else: + inno.build_with_py2exe( + SOURCE_DIR, build_dir, pathlib.Path(python), iscc, version=version, + ) def build_wix( @@ -88,7 +96,12 @@ subparsers = parser.add_subparsers() sp = subparsers.add_parser("inno", help="Build Inno Setup installer") - sp.add_argument("--python", required=True, help="path to python.exe to use") + sp.add_argument( + "--pyoxidizer-target", + choices={"i686-pc-windows-msvc", "x86_64-pc-windows-msvc"}, + help="Build with PyOxidizer targeting this host triple", + ) + sp.add_argument("--python", help="path to python.exe to use") sp.add_argument("--iscc", help="path to iscc.exe to use") sp.add_argument( "--version", diff --git a/contrib/packaging/hgpackaging/inno.py b/contrib/packaging/hgpackaging/inno.py --- a/contrib/packaging/hgpackaging/inno.py +++ b/contrib/packaging/hgpackaging/inno.py @@ -18,8 +18,9 @@ build_py2exe, stage_install, ) +from .pyoxidizer import run_pyoxidizer from .util import ( - find_vc_runtime_files, + find_legacy_vc_runtime_files, normalize_windows_version, process_install_rules, read_version_py, @@ -41,14 +42,14 @@ } -def build( +def build_with_py2exe( source_dir: pathlib.Path, build_dir: pathlib.Path, python_exe: pathlib.Path, iscc_exe: pathlib.Path, version=None, ): - """Build the Inno installer. + """Build the Inno installer using py2exe. Build files will be placed in ``build_dir``. @@ -92,7 +93,7 @@ process_install_rules(EXTRA_INSTALL_RULES, source_dir, staging_dir) # hg.exe depends on VC9 runtime DLLs. Copy those into place. - for f in find_vc_runtime_files(vc_x64): + for f in find_legacy_vc_runtime_files(vc_x64): if f.name.endswith('.manifest'): basename = 'Microsoft.VC90.CRT.manifest' else: @@ -113,6 +114,35 @@ ) +def build_with_pyoxidizer( + source_dir: pathlib.Path, + build_dir: pathlib.Path, + target_triple: str, + iscc_exe: pathlib.Path, + version=None, +): + """Build the Inno installer using PyOxidizer.""" + if not iscc_exe.exists(): + raise Exception("%s does not exist" % iscc_exe) + + inno_build_dir = build_dir / ("inno-pyoxidizer-%s" % target_triple) + staging_dir = inno_build_dir / "stage" + + inno_build_dir.mkdir(parents=True, exist_ok=True) + run_pyoxidizer(source_dir, inno_build_dir, staging_dir, target_triple) + + process_install_rules(EXTRA_INSTALL_RULES, source_dir, staging_dir) + + build_installer( + source_dir, + inno_build_dir, + staging_dir, + iscc_exe, + version, + arch="x64" if "x86_64" in target_triple else None, + ) + + def build_installer( source_dir: pathlib.Path, inno_build_dir: pathlib.Path, diff --git a/contrib/packaging/hgpackaging/pyoxidizer.py b/contrib/packaging/hgpackaging/pyoxidizer.py new file mode 100644 --- /dev/null +++ b/contrib/packaging/hgpackaging/pyoxidizer.py @@ -0,0 +1,145 @@ +# pyoxidizer.py - Packaging support for PyOxidizer +# +# Copyright 2020 Gregory Szorc +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +# no-check-code because Python 3 native. + +import os +import pathlib +import shutil +import subprocess +import sys + +from .downloads import download_entry +from .util import ( + extract_zip_to_directory, + process_install_rules, + find_vc_runtime_dll, +) + + +STAGING_RULES_WINDOWS = [ + ('contrib/bash_completion', 'contrib/'), + ('contrib/hgk', 'contrib/hgk.tcl'), + ('contrib/hgweb.fcgi', 'contrib/'), + ('contrib/hgweb.wsgi', 'contrib/'), + ('contrib/logo-droplets.svg', 'contrib/'), + ('contrib/mercurial.el', 'contrib/'), + ('contrib/mq.el', 'contrib/'), + ('contrib/tcsh_completion', 'contrib/'), + ('contrib/tcsh_completion_build.sh', 'contrib/'), + ('contrib/vim/*', 'contrib/vim/'), + ('contrib/win32/postinstall.txt', 'ReleaseNotes.txt'), + ('contrib/win32/ReadMe.html', 'ReadMe.html'), + ('contrib/xml.rnc', 'contrib/'), + ('contrib/zsh_completion', 'contrib/'), + ('doc/*.html', 'doc/'), + ('doc/style.css', 'doc/'), + ('COPYING', 'Copying.txt'), +] + +STAGING_RULES_APP = [ + ('mercurial/helptext/**/*.txt', 'helptext/'), + ('mercurial/defaultrc/*.rc', 'defaultrc/'), + ('mercurial/locale/**/*', 'locale/'), + ('mercurial/templates/**/*', 'templates/'), +] + +STAGING_EXCLUDES_WINDOWS = [ + "doc/hg-ssh.8.html", +] + + +def run_pyoxidizer( + source_dir: pathlib.Path, + build_dir: pathlib.Path, + out_dir: pathlib.Path, + target_triple: str, +): + """Build Mercurial with PyOxidizer and copy additional files into place. + + After successful completion, ``out_dir`` contains files constituting a + Mercurial install. + """ + # We need to make gettext binaries available for compiling i18n files. + gettext_pkg, gettext_entry = download_entry('gettext', build_dir) + gettext_dep_pkg = download_entry('gettext-dep', build_dir)[0] + + gettext_root = build_dir / ('gettext-win-%s' % gettext_entry['version']) + + if not gettext_root.exists(): + extract_zip_to_directory(gettext_pkg, gettext_root) + extract_zip_to_directory(gettext_dep_pkg, gettext_root) + + env = dict(os.environ) + env["PATH"] = "%s%s%s" % ( + env["PATH"], + os.pathsep, + str(gettext_root / "bin"), + ) + + args = [ + "pyoxidizer", + "build", + "--path", + str(source_dir / "rust" / "hgcli"), + "--release", + "--target-triple", + target_triple, + ] + + subprocess.run(args, env=env, check=True) + + if "windows" in target_triple: + target = "app_windows" + else: + target = "app_posix" + + build_dir = ( + source_dir / "build" / "pyoxidizer" / target_triple / "release" / target + ) + + if out_dir.exists(): + print("purging %s" % out_dir) + shutil.rmtree(out_dir) + + # Now assemble all the files from PyOxidizer into the staging directory. + shutil.copytree(build_dir, out_dir) + + # Move some of those files around. + process_install_rules(STAGING_RULES_APP, build_dir, out_dir) + # Nuke the mercurial/* directory, as we copied resources + # to an appropriate location just above. + shutil.rmtree(out_dir / "mercurial") + + # We also need to run setup.py build_doc to produce html files, + # as they aren't built as part of ``pip install``. + # This will fail if docutils isn't installed. + subprocess.run( + [sys.executable, str(source_dir / "setup.py"), "build_doc", "--html"], + cwd=str(source_dir), + check=True, + ) + + if "windows" in target_triple: + process_install_rules(STAGING_RULES_WINDOWS, source_dir, out_dir) + + # Write out a default editor.rc file to configure notepad as the + # default editor. + with (out_dir / "defaultrc" / "editor.rc").open( + "w", encoding="utf-8" + ) as fh: + fh.write("[ui]\neditor = notepad\n") + + for f in STAGING_EXCLUDES_WINDOWS: + p = out_dir / f + if p.exists(): + print("removing %s" % p) + p.unlink() + + # Add vcruntimeXXX.dll next to executable. + vc_runtime_dll = find_vc_runtime_dll(x64="x86_64" in target_triple) + shutil.copy(vc_runtime_dll, out_dir / vc_runtime_dll.name) diff --git a/contrib/packaging/hgpackaging/util.py b/contrib/packaging/hgpackaging/util.py --- a/contrib/packaging/hgpackaging/util.py +++ b/contrib/packaging/hgpackaging/util.py @@ -29,7 +29,59 @@ zf.extractall(dest) -def find_vc_runtime_files(x64=False): +def find_vc_runtime_dll(x64=False): + """Finds Visual C++ Runtime DLL to include in distribution.""" + # We invoke vswhere to find the latest Visual Studio install. + vswhere = ( + pathlib.Path(os.environ["ProgramFiles(x86)"]) + / "Microsoft Visual Studio" + / "Installer" + / "vswhere.exe" + ) + + if not vswhere.exists(): + raise Exception( + "could not find vswhere.exe: %s does not exist" % vswhere + ) + + args = [ + str(vswhere), + # -products * is necessary to return results from Build Tools + # (as opposed to full IDE installs). + "-products", + "*", + "-requires", + "Microsoft.VisualCpp.Redist.14.Latest", + "-latest", + "-property", + "installationPath", + ] + + vs_install_path = pathlib.Path( + os.fsdecode(subprocess.check_output(args).strip()) + ) + + # This just gets us a path like + # C:\Program Files (x86)\Microsoft Visual Studio\2019\Community + # Actually vcruntime140.dll is under a path like: + # VC\Redist\MSVC\\\Microsoft.VC14.CRT\vcruntime140.dll. + + arch = "x64" if x64 else "x86" + + search_glob = ( + r"%s\VC\Redist\MSVC\*\%s\Microsoft.VC14*.CRT\vcruntime140.dll" + % (vs_install_path, arch) + ) + + candidates = glob.glob(search_glob, recursive=True) + + for candidate in reversed(candidates): + return pathlib.Path(candidate) + + raise Exception("could not find vcruntime140.dll") + + +def find_legacy_vc_runtime_files(x64=False): """Finds Visual C++ Runtime DLLs to include in distribution.""" winsxs = pathlib.Path(os.environ['SYSTEMROOT']) / 'WinSxS' diff --git a/rust/hgcli/pyoxidizer.bzl b/rust/hgcli/pyoxidizer.bzl --- a/rust/hgcli/pyoxidizer.bzl +++ b/rust/hgcli/pyoxidizer.bzl @@ -1,13 +1,24 @@ ROOT = CWD + "/../.." -def make_exe(): - dist = default_python_distribution() +# Code to run in Python interpreter. +RUN_CODE = "import hgdemandimport; hgdemandimport.enable(); from mercurial import dispatch; dispatch.run()" + + +set_build_path(ROOT + "/build/pyoxidizer") + - code = "import hgdemandimport; hgdemandimport.enable(); from mercurial import dispatch; dispatch.run()" +def make_distribution(): + return default_python_distribution() + +def make_distribution_windows(): + return default_python_distribution(flavor="standalone_dynamic") + + +def make_exe(dist): config = PythonInterpreterConfig( raw_allocator = "system", - run_eval = code, + run_eval = RUN_CODE, # We want to let the user load extensions from the file system filesystem_importer = True, # We need this to make resourceutil happy, since it looks for sys.frozen. @@ -24,30 +35,65 @@ extension_module_filter = "all", ) - exe.add_python_resources(dist.pip_install([ROOT])) + # Add Mercurial to resources. + for resource in dist.pip_install(["--verbose", ROOT]): + # This is a bit wonky and worth explaining. + # + # Various parts of Mercurial don't yet support loading package + # resources via the ResourceReader interface. Or, not having + # file-based resources would be too inconvenient for users. + # + # So, for package resources, we package them both in the + # filesystem as well as in memory. If both are defined, + # PyOxidizer will prefer the in-memory location. So even + # if the filesystem file isn't packaged in the location + # specified here, we should never encounter an errors as the + # resource will always be available in memory. + if type(resource) == "PythonPackageResource": + exe.add_filesystem_relative_python_resource(".", resource) + exe.add_in_memory_python_resource(resource) + else: + exe.add_python_resource(resource) + + # On Windows, we install extra packages for convenience. + if "windows" in BUILD_TARGET_TRIPLE: + exe.add_python_resources( + dist.pip_install(["-r", ROOT + "/contrib/packaging/requirements_win32.txt"]) + ) return exe -def make_install(exe): + +def make_manifest(dist, exe): m = FileManifest() - - # `hg` goes in root directory. m.add_python_resource(".", exe) - templates = glob( - include = [ROOT + "/mercurial/templates/**/*"], - strip_prefix = ROOT + "/mercurial/", - ) - m.add_manifest(templates) + return m - return m def make_embedded_resources(exe): return exe.to_embedded_resources() -register_target("exe", make_exe) -register_target("app", make_install, depends = ["exe"], default = True) -register_target("embedded", make_embedded_resources, depends = ["exe"], default_build_script = True) + +register_target("distribution_posix", make_distribution) +register_target("distribution_windows", make_distribution_windows) + +register_target("exe_posix", make_exe, depends = ["distribution_posix"]) +register_target("exe_windows", make_exe, depends = ["distribution_windows"]) + +register_target( + "app_posix", + make_manifest, + depends = ["distribution_posix", "exe_posix"], + default = "windows" not in BUILD_TARGET_TRIPLE, +) +register_target( + "app_windows", + make_manifest, + depends = ["distribution_windows", "exe_windows"], + default = "windows" in BUILD_TARGET_TRIPLE, +) + resolve_targets() # END OF COMMON USER-ADJUSTED SETTINGS. @@ -55,5 +101,4 @@ # Everything below this is typically managed by PyOxidizer and doesn't need # to be updated by people. -PYOXIDIZER_VERSION = "0.7.0-pre" -PYOXIDIZER_COMMIT = "c772a1379c3026314eda1c8ea244b86c0658951d" +PYOXIDIZER_VERSION = "0.7.0" diff --git a/tests/test-check-code.t b/tests/test-check-code.t --- a/tests/test-check-code.t +++ b/tests/test-check-code.t @@ -27,6 +27,7 @@ Skipping contrib/packaging/hgpackaging/downloads.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/inno.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/py2exe.py it has no-che?k-code (glob) + Skipping contrib/packaging/hgpackaging/pyoxidizer.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/util.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/wix.py it has no-che?k-code (glob) Skipping i18n/polib.py it has no-che?k-code (glob)