( )⚙ D5436 rust-cpython: build via HGWITHRUSTEXT=cpython

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D5436

rust-cpython: build via HGWITHRUSTEXT=cpython
ClosedPublic

Authored by gracinet on Dec 15 2018, 6:41 AM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG462a26756f70: rust-cpython: build via HGWITHRUSTEXT=cpython
rHG9e755c16f05d: rust-cpython: build via HGWITHRUSTEXT=cpython
Summary

The existing behaviour, building the direct ffi bindings if
HGIWTHRUSTEXT is just set is unchanged, but if HGWITHRUSTEXT is
cpython, then the cpython bindings (aka mercurial/rustext.so) are
built.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

gracinet created this revision.Dec 15 2018, 6:41 AM
yuja added a subscriber: yuja.Dec 15 2018, 10:22 PM

I expect we'll soon drop the support for the old ffi binding.

Closed by commit rHG9e755c16f05d: rust-cpython: build via HGWITHRUSTEXT=cpython (authored by gracinet). · Explain WhyDec 15 2018, 10:30 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
MMakefile (1 line)
Msetup.py (90 lines)

Diff 12873

Makefile

\( -name '*.py[cdo]' -o -name '*.so' \) -exec rm -f '{}' ';' \( -name '*.py[cdo]' -o -name '*.so' \) -exec rm -f '{}' ';'
rm -f MANIFEST MANIFEST.in hgext/__index__.py tests/*.err rm -f MANIFEST MANIFEST.in hgext/__index__.py tests/*.err
rm -f mercurial/__modulepolicy__.py rm -f mercurial/__modulepolicy__.py
if test -d .hg; then rm -f mercurial/__version__.py; fi if test -d .hg; then rm -f mercurial/__version__.py; fi
rm -rf build mercurial/locale rm -rf build mercurial/locale
$(MAKE) -C doc clean $(MAKE) -C doc clean
$(MAKE) -C contrib/chg distclean $(MAKE) -C contrib/chg distclean
rm -rf rust/target rm -rf rust/target
rm -f mercurial/rustext.so
clean: cleanbutpackages clean: cleanbutpackages
rm -rf packages rm -rf packages
install: install-bin install-doc install: install-bin install-doc
install-bin: build install-bin: build
$(PYTHON) setup.py $(PURE) install --root="$(DESTDIR)/" --prefix="$(PREFIX)" --force $(PYTHON) setup.py $(PURE) install --root="$(DESTDIR)/" --prefix="$(PREFIX)" --force

setup.py

import bz2 import bz2
bz2.BZ2Compressor # silence unused import warning bz2.BZ2Compressor # silence unused import warning
except ImportError: except ImportError:
raise SystemExit( raise SystemExit(
"Couldn't import standard bz2 (incomplete Python install).") "Couldn't import standard bz2 (incomplete Python install).")
ispypy = "PyPy" in sys.version ispypy = "PyPy" in sys.version
iswithrustextensions = 'HGWITHRUSTEXT' in os.environ hgrustext = os.environ.get('HGWITHRUSTEXT')
# TODO record it for proper rebuild upon changes
# (see mercurial/__modulepolicy__.py)
if hgrustext != 'cpython' and hgrustext is not None:
hgrustext = 'direct-ffi'
import ctypes import ctypes
import stat, subprocess, time import stat, subprocess, time
import re import re
import shutil import shutil
import tempfile import tempfile
from distutils import log from distutils import log
# We have issues with setuptools on some platforms and builders. Until # We have issues with setuptools on some platforms and builders. Until
boolean_options = build_ext.boolean_options + ['zstd'] boolean_options = build_ext.boolean_options + ['zstd']
negative_opt = buildextnegops negative_opt = buildextnegops
def initialize_options(self): def initialize_options(self):
self.zstd = True self.zstd = True
return build_ext.initialize_options(self) return build_ext.initialize_options(self)
def build_extensions(self): def build_extensions(self):
ruststandalones = [e for e in self.extensions
if isinstance(e, RustStandaloneExtension)]
self.extensions = [e for e in self.extensions
if e not in ruststandalones]
# Filter out zstd if disabled via argument. # Filter out zstd if disabled via argument.
if not self.zstd: if not self.zstd:
self.extensions = [e for e in self.extensions self.extensions = [e for e in self.extensions
if e.name != 'mercurial.zstd'] if e.name != 'mercurial.zstd']
for rustext in ruststandalones:
rustext.build('' if self.inplace else self.build_lib)
return build_ext.build_extensions(self) return build_ext.build_extensions(self)
def build_extension(self, ext): def build_extension(self, ext):
if isinstance(ext, RustExtension): if isinstance(ext, RustExtension):
ext.rustbuild() ext.rustbuild()
try: try:
build_ext.build_extension(self, ext) build_ext.build_extension(self, ext)
except CCompilerError: except CCompilerError:
'mercurial/thirdparty/xdiff/xtypes.h', 'mercurial/thirdparty/xdiff/xtypes.h',
'mercurial/thirdparty/xdiff/xutils.h', 'mercurial/thirdparty/xdiff/xutils.h',
] ]
class RustCompilationError(CCompilerError): class RustCompilationError(CCompilerError):
"""Exception class for Rust compilation errors.""" """Exception class for Rust compilation errors."""
class RustExtension(Extension): class RustExtension(Extension):
"""A C Extension, conditionnally enhanced with Rust code. """Base classes for concrete Rust Extension classes.
if iswithrustextensions is False, does nothing else than plain Extension
""" """
rusttargetdir = os.path.join('rust', 'target', 'release') rusttargetdir = os.path.join('rust', 'target', 'release')
def __init__(self, mpath, sources, rustlibname, subcrate, **kw): def __init__(self, mpath, sources, rustlibname, subcrate, **kw):
Extension.__init__(self, mpath, sources, **kw) Extension.__init__(self, mpath, sources, **kw)
if not iswithrustextensions: if hgrustext is None:
return return
srcdir = self.rustsrcdir = os.path.join('rust', subcrate) srcdir = self.rustsrcdir = os.path.join('rust', subcrate)
self.libraries.append(rustlibname)
self.extra_compile_args.append('-DWITH_RUST')
# adding Rust source and control files to depends so that the extension # adding Rust source and control files to depends so that the extension
# gets rebuilt if they've changed # gets rebuilt if they've changed
self.depends.append(os.path.join(srcdir, 'Cargo.toml')) self.depends.append(os.path.join(srcdir, 'Cargo.toml'))
cargo_lock = os.path.join(srcdir, 'Cargo.lock') cargo_lock = os.path.join(srcdir, 'Cargo.lock')
if os.path.exists(cargo_lock): if os.path.exists(cargo_lock):
self.depends.append(cargo_lock) self.depends.append(cargo_lock)
for dirpath, subdir, fnames in os.walk(os.path.join(srcdir, 'src')): for dirpath, subdir, fnames in os.walk(os.path.join(srcdir, 'src')):
self.depends.extend(os.path.join(dirpath, fname) self.depends.extend(os.path.join(dirpath, fname)
for fname in fnames for fname in fnames
if os.path.splitext(fname)[1] == '.rs') if os.path.splitext(fname)[1] == '.rs')
def rustbuild(self): def rustbuild(self):
if not iswithrustextensions: if hgrustext is None:
return return
env = os.environ.copy() env = os.environ.copy()
if 'HGTEST_RESTOREENV' in env: if 'HGTEST_RESTOREENV' in env:
# Mercurial tests change HOME to a temporary directory, # Mercurial tests change HOME to a temporary directory,
# but, if installed with rustup, the Rust toolchain needs # but, if installed with rustup, the Rust toolchain needs
# HOME to be correct (otherwise the 'no default toolchain' # HOME to be correct (otherwise the 'no default toolchain'
# error message is issued and the build fails). # error message is issued and the build fails).
# This happens currently with test-hghave.t, which does # This happens currently with test-hghave.t, which does
"Cargo found, but permisssion to execute it is denied") "Cargo found, but permisssion to execute it is denied")
else: else:
raise raise
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
raise RustCompilationError( raise RustCompilationError(
"Cargo failed. Working directory: %r, " "Cargo failed. Working directory: %r, "
"command: %r, environment: %r" % (self.rustsrcdir, cmd, env)) "command: %r, environment: %r" % (self.rustsrcdir, cmd, env))
class RustEnhancedExtension(RustExtension):
"""A C Extension, conditionally enhanced with Rust code.
If the HGRUSTEXT environment variable is set to something else
than 'cpython', the Rust sources get compiled and linked within the
C target shared library object.
"""
def __init__(self, mpath, sources, rustlibname, subcrate, **kw):
RustExtension.__init__(self, mpath, sources, rustlibname, subcrate,
**kw)
if hgrustext != 'direct-ffi':
return
self.extra_compile_args.append('-DWITH_RUST')
self.libraries.append(rustlibname)
self.library_dirs.append(self.rusttargetdir)
class RustStandaloneExtension(RustExtension):
def __init__(self, pydottedname, rustcrate, dylibname, **kw):
RustExtension.__init__(self, pydottedname, [], dylibname, rustcrate,
**kw)
self.dylibname = dylibname
def build(self, target_dir):
self.rustbuild()
target = [target_dir]
target.extend(self.name.split('.'))
ext = '.so' # TODO Unix only
target[-1] += ext
shutil.copy2(os.path.join(self.rusttargetdir, self.dylibname + ext),
os.path.join(*target))
extmodules = [ extmodules = [
Extension('mercurial.cext.base85', ['mercurial/cext/base85.c'], Extension('mercurial.cext.base85', ['mercurial/cext/base85.c'],
include_dirs=common_include_dirs, include_dirs=common_include_dirs,
depends=common_depends), depends=common_depends),
Extension('mercurial.cext.bdiff', ['mercurial/bdiff.c', Extension('mercurial.cext.bdiff', ['mercurial/bdiff.c',
'mercurial/cext/bdiff.c'] + xdiff_srcs, 'mercurial/cext/bdiff.c'] + xdiff_srcs,
include_dirs=common_include_dirs, include_dirs=common_include_dirs,
depends=common_depends + ['mercurial/bdiff.h'] + xdiff_headers), depends=common_depends + ['mercurial/bdiff.h'] + xdiff_headers),
Extension('mercurial.cext.mpatch', ['mercurial/mpatch.c', Extension('mercurial.cext.mpatch', ['mercurial/mpatch.c',
'mercurial/cext/mpatch.c'], 'mercurial/cext/mpatch.c'],
include_dirs=common_include_dirs, include_dirs=common_include_dirs,
depends=common_depends), depends=common_depends),
RustExtension('mercurial.cext.parsers', ['mercurial/cext/charencode.c', RustEnhancedExtension(
'mercurial.cext.parsers', ['mercurial/cext/charencode.c',
'mercurial/cext/dirs.c', 'mercurial/cext/dirs.c',
'mercurial/cext/manifest.c', 'mercurial/cext/manifest.c',
'mercurial/cext/parsers.c', 'mercurial/cext/parsers.c',
'mercurial/cext/pathencode.c', 'mercurial/cext/pathencode.c',
'mercurial/cext/revlog.c'], 'mercurial/cext/revlog.c'],
'hgdirectffi', 'hgdirectffi',
'hg-direct-ffi', 'hg-direct-ffi',
include_dirs=common_include_dirs, include_dirs=common_include_dirs,
depends=common_depends + ['mercurial/cext/charencode.h', depends=common_depends + ['mercurial/cext/charencode.h',
'mercurial/cext/revlog.h', 'mercurial/cext/revlog.h',
'rust/hg-core/src/ancestors.rs', 'rust/hg-core/src/ancestors.rs',
'rust/hg-core/src/lib.rs']), 'rust/hg-core/src/lib.rs']),
Extension('mercurial.cext.osutil', ['mercurial/cext/osutil.c'], Extension('mercurial.cext.osutil', ['mercurial/cext/osutil.c'],
include_dirs=common_include_dirs, include_dirs=common_include_dirs,
extra_compile_args=osutil_cflags, extra_compile_args=osutil_cflags,
extra_link_args=osutil_ldflags, extra_link_args=osutil_ldflags,
depends=common_depends), depends=common_depends),
Extension( Extension(
'mercurial.thirdparty.zope.interface._zope_interface_coptimizations', [ 'mercurial.thirdparty.zope.interface._zope_interface_coptimizations', [
'mercurial/thirdparty/zope/interface/_zope_interface_coptimizations.c', 'mercurial/thirdparty/zope/interface/_zope_interface_coptimizations.c',
]), ]),
Extension('hgext.fsmonitor.pywatchman.bser', Extension('hgext.fsmonitor.pywatchman.bser',
['hgext/fsmonitor/pywatchman/bser.c']), ['hgext/fsmonitor/pywatchman/bser.c']),
] ]
if hgrustext == 'cpython':
extmodules.append(
RustStandaloneExtension('mercurial.rustext', 'hg-cpython', 'librusthg')
)
sys.path.insert(0, 'contrib/python-zstandard') sys.path.insert(0, 'contrib/python-zstandard')
import setup_zstd import setup_zstd
extmodules.append(setup_zstd.get_c_extension( extmodules.append(setup_zstd.get_c_extension(
name='mercurial.zstd', name='mercurial.zstd',
root=os.path.abspath(os.path.dirname(__file__)))) root=os.path.abspath(os.path.dirname(__file__))))
try: try:
from distutils import cygwinccompiler from distutils import cygwinccompiler