diff --git a/contrib/packaging/hgpackaging/downloads.py b/contrib/packaging/hgpackaging/downloads.py --- a/contrib/packaging/hgpackaging/downloads.py +++ b/contrib/packaging/hgpackaging/downloads.py @@ -25,12 +25,24 @@ 'size': 715086, 'sha256': '411f94974492fd2ecf52590cb05b1023530aec67e64154a88b1e4ebcd9c28588', }, + 'openssl': { + 'url': 'https://www.openssl.org/source/openssl-1.0.2t.tar.gz', + 'size': 5355422, + 'sha256': '14cb464efe7ac6b54799b34456bd69558a749a4931ecfd9cf9f71d7881cac7bc', + 'version': '1.0.2t', + }, 'py2exe': { 'url': 'https://versaweb.dl.sourceforge.net/project/py2exe/py2exe/0.6.9/py2exe-0.6.9.zip', 'size': 149687, 'sha256': '6bd383312e7d33eef2e43a5f236f9445e4f3e0f6b16333c6f183ed445c44ddbd', 'version': '0.6.9', }, + 'python27': { + 'url': 'https://www.python.org/ftp/python/2.7.16/Python-2.7.16.tgz', + 'size': 17431748, + 'sha256': '01da813a3600876f03f46db11cc5c408175e99f03af2ba942ef324389a83bad5', + 'version': '2.7.16', + }, # The VC9 CRT merge modules aren't readily available on most systems because # they are only installed as part of a full Visual Studio 2008 install. # While we could potentially extract them from a Visual Studio 2008 diff --git a/contrib/packaging/hgpackaging/python.py b/contrib/packaging/hgpackaging/python.py new file mode 100644 --- /dev/null +++ b/contrib/packaging/hgpackaging/python.py @@ -0,0 +1,143 @@ +# python.py - Build a functional python interpreter. +# +# Copyright 2019 Matt Harbison +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +# no-check-code because Python 3 native. + +import multiprocessing +import os +import pathlib +import subprocess + +from .downloads import ( + download_entry, +) +from .util import ( + extract_tar_to_directory, +) + +def _build_openssl(download_dir: pathlib.Path, build_dir: pathlib.Path, + common_env: dict): + """Build and locally install OpenSSL, downloading the source first if + necessary. The exact local installation path is returned. + """ + local_path, entry = download_entry('openssl', download_dir) + version = entry["version"] + + # We only need the static library, so most of the installation will be + # thrown away. + openssl_dir = build_dir / ("openssl-" + version) + distdir = openssl_dir / "dist" + openssl_lib = distdir / "usr" / "lib" / "libcrypto.a" + + if openssl_lib.exists(): + print("OpenSSL %s is already built" % version) + return distdir + + if not openssl_dir.exists(): + extract_tar_to_directory(local_path, build_dir) + + env = common_env + + subprocess.check_call(["/bin/sh", + "./Configure", + "--prefix=%s/usr" % distdir, + "--openssldir=%s/etc/openssl" % distdir, + "no-ssl2", + "zlib-dynamic", + "shared", + "enable-cms", + "darwin64-x86_64-cc", + "enable-ec_nistp_64_gcc_128"], + env=env, + cwd=str(openssl_dir)) + + subprocess.check_call(["make", "depend"], + env=env, + cwd=openssl_dir) + + subprocess.check_call(["make", "-j%d" % multiprocessing.cpu_count()], + env=env, + cwd=openssl_dir) + + subprocess.check_call(["make", "install"], + env=env, + cwd=openssl_dir) + + return distdir + +def build(source_dir: pathlib.Path, build_dir: pathlib.Path, + download_dir: pathlib.Path, common_env: dict): + """Build and locally install a python interpreter suitable for running + Mercurial, including package dependencies. + + The installation will be placed under ``build_dir``, and the exact path and + path to the python executabel is returned. + + ``source_dir`` is the path to the root of the Mercurial repository being + packaged. The environment variables provided can control the compiler and + compiler options used, and must also be used to build Mercurial itself. + """ + + local_path, entry = download_entry('python27', download_dir) + version = entry["version"] + short_version = version[:version.rindex(".")] + + prefix = pathlib.Path("usr/local/mercurial") # Should be absolute path + python_dir = build_dir / ("Python-%s" % version) + dest_dir = python_dir / "dist" + python = dest_dir / prefix / "bin" / ("python%s" % short_version) + + if not python.exists(): + openssl_dist = _build_openssl(download_dir, build_dir, common_env) + + env = { + "CFLAGS": (("-Os -pipe -fno-common -fno-strict-aliasing -fwrapv " + "-DENABLE_DTRACE -DMACOSX -DNDEBUG " + "-I{distdir}/usr/include " + "-I{sdkroot}/usr/include") + .format(distdir=openssl_dist, + sdkroot=common_env["SDKROOT"])), + "LDFLAGS": "-L{distdir}/usr/lib".format(distdir=openssl_dist), + "DESTDIR": str(dest_dir) # python "installed" here + } + + env.update(common_env) + + if not python_dir.exists(): + extract_tar_to_directory(local_path, build_dir) + + env.update(os.environ) + + subprocess.check_call(["./configure", + "--prefix=/" + str(prefix), + "--with-ensurepip", + "--enable-ipv6", + "--with-threads", + "--enable-optimizations"], + env=env, + cwd=str(python_dir)) + + subprocess.check_call(["make", "-j%d" % multiprocessing.cpu_count()], + env=env, + cwd=str(python_dir)) + + subprocess.check_call(["make", "altinstall"], + env=env, + cwd=str(python_dir)) + else: + print("Python %s is already built" % version) + env = common_env + + # Install/update python packages needed to build hg + subprocess.check_call([str(python), + "-m", "pip", "install", + "--disable-pip-version-check", + "-r", "contrib/packaging/macosx/requirements.txt"], + env=env, + cwd=str(source_dir)) + + return python, dest_dir diff --git a/contrib/packaging/macosx/build.py b/contrib/packaging/macosx/build.py new file mode 100755 --- /dev/null +++ b/contrib/packaging/macosx/build.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +# build.py - MacOS installer build script. +# +# Copyright 2019 Matt Harbison +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +# This script automates the building of the MacOS installer for Mercurial. + +# no-check-code because Python 3 native. + +import argparse +import os +import pathlib +import subprocess +import sys + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + + parser.add_argument('--python', + required=True, + help='path to python.exe to use') + parser.add_argument('--version', + help='Mercurial version string to use ' + '(detected from __version__.py if not defined') + + args = parser.parse_args() + + if not os.path.isabs(args.python): + raise Exception('--python arg must be an absolute path') + + here = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) + source_dir = here.parent.parent.parent + build_dir = source_dir / 'build' + download_dir = source_dir / 'packages' / 'downloaded' + + os.makedirs(download_dir, exist_ok=True) + + sys.path.insert(0, str(source_dir / 'contrib' / 'packaging')) + + sdkroot = subprocess.check_output(["xcrun", "--show-sdk-path"]) + sdkroot = sdkroot.decode(sys.stdout.encoding).strip() + + common_env = { + "CC": "clang", + "CXX": "clang++", + "DEVELOPER_DIR": "/Library/Developer/CommandLineTools", + "MACOSX_DEPLOYMENT_TARGET": "10.9", + "PATH": "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", + "SDKROOT": sdkroot, + } + + from hgpackaging.python import build + + python, dest_dir = build(source_dir, build_dir, download_dir, common_env) + + subprocess.check_call([python, + "setup.py", "install", "--optimize=1", + "--root=%s" % dest_dir, + "--prefix=/usr/local", + "--install-lib=/usr/local/mercurial/lib/python2.7/site-packages"], + cwd=str(source_dir), + env=common_env) + + # Generate and stage docs using external python, so the newly built + # interpreter isn't polluted with docutils junk. + subprocess.check_call(["make", + "PYTHON=%s" % args.python, + "DESTDIR=%s" % dest_dir, + "install-doc"], + cwd=str(source_dir), + env=common_env) + + # Install binaries into custom python. + subprocess.check_call(["make", + "PYTHON=%s" % python, + "DESTDIR=%s" % dest_dir, + "PREFIX=/usr/local/mercurial", + "install-bin"], + cwd=str(source_dir), + env=common_env) + + # Place a bogon .DS_Store file in the target dir so we can be + # sure it doesn't get included in the final package. + with open(dest_dir / ".DS_Store", "w+"): + pass + + # install zsh completions - this location appears to be + # searched by default as of macOS Sierra. + zsh_dest = dest_dir / "usr" / "local" / "share" / "zsh" / "site-functions" + + subprocess.check_call(["install", "-d", str(zsh_dest)], + cwd=str(source_dir), + env=common_env) + + subprocess.check_call(["install", "-m", "0644", "contrib/zsh_completion", + str(zsh_dest / "_hg")], + cwd=str(source_dir), + env=common_env) + + # install bash completions - there doesn't appear to be a + # place that's searched by default for bash, so we'll follow + # the lead of Apple's git install and just put it in a + # location of our own. + bash_dest = dest_dir / "usr" / "local" / "hg" / "contrib" + + subprocess.check_call(["install", "-d", str(bash_dest)], + cwd=str(source_dir), + env=common_env) + + subprocess.check_call(["install", "-m", "0644", "contrib/bash_completion", + str(bash_dest / "hg-completion.bash")], + cwd=str(source_dir), + env=common_env) + + subprocess.check_call(["make", "-C", "contrib/chg", + "HGPATH=/usr/local/bin/hg", + "PYTHON=%s/usr/local/mercurial/bin/python2.7" + % dest_dir, + "HGEXTDIR=/usr/local/mercurial/lib/python2.7/" + "site-packages/hgext", + "DESTDIR=%s" % dest_dir, + "PREFIX=/usr/local", + "clean", "install"], + cwd=str(source_dir), + env=common_env) + + # TODO: optionally sign all binaries here + + # TODO: consult $OUTPUTDIR? + output_dir = source_dir / "dist" + os.makedirs(output_dir, exist_ok=True) + + hgver = args.version + + if not hgver: + # In case hg isn't installed on PATH yet. `hg debuginstall` exits 1 in + # the local repo if bdiff wasn't built locally. + common_env["PATH"] = (os.environ["PATH"] + ":" + + str(dest_dir / "usr" / "local" / "bin")) + + f = ("%s/usr/local/mercurial/lib/python2.7/site-packages/" + "mercurial/__version__.py" % dest_dir) + + # TODO: Figure out how to pass off ${OSXVERSIONFLAGS}? + hgver = subprocess.check_output([args.python, + "contrib/genosxversion.py", f], + cwd=str(source_dir), + env=common_env) + hgver = hgver.decode(sys.stdout.encoding).strip() + + # XXX: There doesn't seem to be a way to get the shebang line correct + # without the python interpreter being in the correct location already. + # Since genosxversion.py needs a working hg, that's not a bad thing. But + # it needs to be replaced before packaging it. The flip side is that + # when building the installer again, the hg script isn't replaced if + # it wasn't modified in the source, which breaks the version script if hg + # wasn't subsequently installed. + hg = dest_dir / "usr" / "local" / "bin" / "hg" + with hg.open("r") as fp: + hgscript = fp.readlines() + + hgscript[0] = '#!/usr/local/mercurial/bin/python2.7\n' + + with hg.open("w") as fp: + fp.writelines(hgscript) + fp.truncate() + + osxver = common_env["MACOSX_DEPLOYMENT_TARGET"] + + subprocess.check_call(["pkgbuild", "--filter", ".DS_Store", + "--root", str(dest_dir), + "--identifier", "org.mercurial-scm.mercurial", + "--version", hgver, + "build/mercurial.pkg"], + cwd=str(source_dir), + env=common_env) + + pkg = "%s/Mercurial-%s-py2.7-macosx%s.pkg" % (output_dir, hgver, osxver) + subprocess.check_call(["productbuild", + "--distribution", + "contrib/packaging/macosx/distribution.xml", + "--package-path", "build/", + "--version", hgver, + "--resources", "contrib/packaging/macosx/", + pkg], + cwd=str(source_dir), + env=common_env) + + # TODO: optionally sign and notarize the *.pkg here + + print("Created %s" % pkg) diff --git a/contrib/packaging/macosx/requirements.txt b/contrib/packaging/macosx/requirements.txt new file mode 100644 --- /dev/null +++ b/contrib/packaging/macosx/requirements.txt @@ -0,0 +1,12 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --generate-hashes --output-file=contrib/packaging/macosx/requirements.txt contrib/packaging/macosx/requirements.txt.in +# +certifi==2019.6.16 \ + --hash=sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939 \ + --hash=sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695 +pygments==2.4.2 \ + --hash=sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127 \ + --hash=sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297 diff --git a/contrib/packaging/macosx/requirements.txt.in b/contrib/packaging/macosx/requirements.txt.in new file mode 100644 --- /dev/null +++ b/contrib/packaging/macosx/requirements.txt.in @@ -0,0 +1,2 @@ +certifi +pygments