This is an archive of the discontinued Mercurial Phabricator instance.

Differential D6627

rust-parsers: move parser bindings to their own file and Python module
ClosedPublic

Authored by Alphare on Jul 10 2019, 7:46 AM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG760a7851e9ba: rust-parsers: move parser bindings to their own file and Python module
rHG83c836e8efcc: rust-parsers: move parser bindings to their own file and Python module
Summary

This tidies up the Rust side while simplifying the Python side.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Revision Contents
Changeset List

Diff 16045

mercurial/dirstate.py

pathutil, pathutil,
policy, policy,
pycompat, pycompat,
scmutil, scmutil,
txnutil, txnutil,
util, util,
) )
parsers = policy.importmod(r'parsers') orig_parsers = policy.importmod(r'parsers')
dirstatemod = policy.importrust(r'dirstate', default=parsers) parsers = policy.importrust(r'parsers', default=orig_parsers)
propertycache = util.propertycache propertycache = util.propertycache
filecache = scmutil.filecache filecache = scmutil.filecache
_rangemask = 0x7fffffff _rangemask = 0x7fffffff
dirstatetuple = parsers.dirstatetuple dirstatetuple = orig_parsers.dirstatetuple
class repocache(filecache): class repocache(filecache):
"""filecache for files in .hg/""" """filecache for files in .hg/"""
def join(self, obj, fname): def join(self, obj, fname):
return obj._opener.join(fname) return obj._opener.join(fname)
class rootcache(filecache): class rootcache(filecache):
"""filecache for files in the repository root""" """filecache for files in the repository root"""
# them as not to be tracked by the collector. However, this has no # them as not to be tracked by the collector. However, this has no
# effect on when GCs are triggered, only on what objects the GC looks # effect on when GCs are triggered, only on what objects the GC looks
# into. This means that O(number of files) GCs are unavoidable. # into. This means that O(number of files) GCs are unavoidable.
# Depending on when in the process's lifetime the dirstate is parsed, # Depending on when in the process's lifetime the dirstate is parsed,
# this can get very expensive. As a workaround, disable GC while # this can get very expensive. As a workaround, disable GC while
# parsing the dirstate. # parsing the dirstate.
# #
# (we cannot decorate the function directly since it is in a C module) # (we cannot decorate the function directly since it is in a C module)
parse_dirstate = util.nogc(dirstatemod.parse_dirstate) parse_dirstate = util.nogc(parsers.parse_dirstate)
p = parse_dirstate(self._map, self.copymap, st) p = parse_dirstate(self._map, self.copymap, st)
if not self._dirtyparents: if not self._dirtyparents:
self.setparents(*p) self.setparents(*p)
# Avoid excess attribute lookups by fast pathing certain checks # Avoid excess attribute lookups by fast pathing certain checks
self.__contains__ = self._map.__contains__ self.__contains__ = self._map.__contains__
self.__getitem__ = self._map.__getitem__ self.__getitem__ = self._map.__getitem__
self.get = self._map.get self.get = self._map.get
def write(self, st, now): def write(self, st, now):
st.write(dirstatemod.pack_dirstate(self._map, self.copymap, st.write(parsers.pack_dirstate(self._map, self.copymap,
self.parents(), now)) self.parents(), now))
st.close() st.close()
self._dirtyparents = False self._dirtyparents = False
self.nonnormalset, self.otherparentset = self.nonnormalentries() self.nonnormalset, self.otherparentset = self.nonnormalentries()
@propertycache @propertycache
def nonnormalset(self): def nonnormalset(self):
nonnorm, otherparents = self.nonnormalentries() nonnorm, otherparents = self.nonnormalentries()
self.otherparentset = otherparents self.otherparentset = otherparents

rust/hg-cpython/src/dirstate.rs

// dirstate.rs // dirstate.rs
// //
// Copyright 2019 Raphaël Gomès <rgomes@octobus.net> // Copyright 2019 Raphaël Gomès <rgomes@octobus.net>
// //
// This software may be used and distributed according to the terms of the // This software may be used and distributed according to the terms of the
// GNU General Public License version 2 or any later version. // GNU General Public License version 2 or any later version.
//! Bindings for the `hg::dirstate` module provided by the //! Bindings for the `hg::dirstate` module provided by the
//! `hg-core` package. //! `hg-core` package.
//! //!
//! From Python, this will be seen as `mercurial.rustext.dirstate` //! From Python, this will be seen as `mercurial.rustext.dirstate`
mod dirs_multiset; mod dirs_multiset;
use crate::dirstate::dirs_multiset::Dirs; use crate::dirstate::dirs_multiset::Dirs;
use cpython::{ use cpython::{
exc, PyBytes, PyDict, PyErr, PyInt, PyModule, PyObject, PyResult, PyBytes, PyDict, PyErr, PyModule, PyObject, PyResult, PySequence, Python,
PySequence, PyTuple, Python, PythonObject, ToPyObject,
};
use hg::{
pack_dirstate, parse_dirstate, CopyVecEntry, DirstateEntry,
DirstatePackError, DirstateParents, DirstateParseError, DirstateVec,
}; };
use hg::{DirstateEntry, DirstateVec};
use libc::{c_char, c_int}; use libc::{c_char, c_int};
#[cfg(feature = "python27")] #[cfg(feature = "python27")]
use python27_sys::PyCapsule_Import; use python27_sys::PyCapsule_Import;
#[cfg(feature = "python3")] #[cfg(feature = "python3")]
use python3_sys::PyCapsule_Import; use python3_sys::PyCapsule_Import;
use std::collections::HashMap;
use std::ffi::CStr; use std::ffi::CStr;
use std::mem::transmute; use std::mem::transmute;
/// C code uses a custom `dirstate_tuple` type, checks in multiple instances /// C code uses a custom `dirstate_tuple` type, checks in multiple instances
/// for this type, and raises a Python `Exception` if the check does not pass. /// for this type, and raises a Python `Exception` if the check does not pass.
/// Because this type differs only in name from the regular Python tuple, it /// Because this type differs only in name from the regular Python tuple, it
/// would be a good idea in the near future to remove it entirely to allow /// would be a good idea in the near future to remove it entirely to allow
/// for a pure Python tuple of the same effective structure to be used, /// for a pure Python tuple of the same effective structure to be used,
/// rendering this type and the capsule below useless. /// rendering this type and the capsule below useless.
type MakeDirstateTupleFn = extern "C" fn( type MakeDirstateTupleFn = extern "C" fn(
state: c_char, state: c_char,
mode: c_int, mode: c_int,
size: c_int, size: c_int,
mtime: c_int, mtime: c_int,
) -> PyObject; ) -> PyObject;
/// This is largely a copy/paste from cindex.rs, pending the merge of a /// This is largely a copy/paste from cindex.rs, pending the merge of a
/// `py_capsule_fn!` macro in the rust-cpython project: /// `py_capsule_fn!` macro in the rust-cpython project:
/// https://github.com/dgrunwald/rust-cpython/pull/169 /// https://github.com/dgrunwald/rust-cpython/pull/169
fn decapsule_make_dirstate_tuple(py: Python) -> PyResult<MakeDirstateTupleFn> { pub fn decapsule_make_dirstate_tuple(
py: Python,
) -> PyResult<MakeDirstateTupleFn> {
unsafe { unsafe {
let caps_name = CStr::from_bytes_with_nul_unchecked( let caps_name = CStr::from_bytes_with_nul_unchecked(
b"mercurial.cext.parsers.make_dirstate_tuple_CAPI\0", b"mercurial.cext.parsers.make_dirstate_tuple_CAPI\0",
); );
let from_caps = PyCapsule_Import(caps_name.as_ptr(), 0); let from_caps = PyCapsule_Import(caps_name.as_ptr(), 0);
if from_caps.is_null() { if from_caps.is_null() {
return Err(PyErr::fetch(py)); return Err(PyErr::fetch(py));
} }
Ok(transmute(from_caps)) Ok(transmute(from_caps))
} }
} }
fn parse_dirstate_wrapper( pub fn extract_dirstate_vec(
py: Python,
dmap: PyDict,
copymap: PyDict,
st: PyBytes,
) -> PyResult<PyTuple> {
match parse_dirstate(st.data(py)) {
Ok((parents, dirstate_vec, copies)) => {
for (filename, entry) in dirstate_vec {
dmap.set_item(
py,
PyBytes::new(py, &filename[..]),
decapsule_make_dirstate_tuple(py)?(
entry.state as c_char,
entry.mode,
entry.size,
entry.mtime,
),
)?;
}
for CopyVecEntry { path, copy_path } in copies {
copymap.set_item(
py,
PyBytes::new(py, path),
PyBytes::new(py, copy_path),
)?;
}
Ok((PyBytes::new(py, parents.p1), PyBytes::new(py, parents.p2))
.to_py_object(py))
}
Err(e) => Err(PyErr::new::<exc::ValueError, _>(
py,
match e {
DirstateParseError::TooLittleData => {
"too little data for parents".to_string()
}
DirstateParseError::Overflow => {
"overflow in dirstate".to_string()
}
DirstateParseError::CorruptedEntry(e) => e,
},
)),
}
}
fn extract_dirstate_vec(
py: Python, py: Python,
dmap: &PyDict, dmap: &PyDict,
) -> Result<DirstateVec, PyErr> { ) -> Result<DirstateVec, PyErr> {
dmap.items(py) dmap.items(py)
.iter() .iter()
.map(|(filename, stats)| { .map(|(filename, stats)| {
let stats = stats.extract::<PySequence>(py)?; let stats = stats.extract::<PySequence>(py)?;
let state = stats.get_item(py, 0)?.extract::<PyBytes>(py)?; let state = stats.get_item(py, 0)?.extract::<PyBytes>(py)?;
size, size,
mtime, mtime,
}, },
)) ))
}) })
.collect() .collect()
} }
fn pack_dirstate_wrapper(
py: Python,
dmap: PyDict,
copymap: PyDict,
pl: PyTuple,
now: PyInt,
) -> PyResult<PyBytes> {
let p1 = pl.get_item(py, 0).extract::<PyBytes>(py)?;
let p1: &[u8] = p1.data(py);
let p2 = pl.get_item(py, 1).extract::<PyBytes>(py)?;
let p2: &[u8] = p2.data(py);
let dirstate_vec = extract_dirstate_vec(py, &dmap)?;
let copies: Result<HashMap<Vec<u8>, Vec<u8>>, PyErr> = copymap
.items(py)
.iter()
.map(|(key, value)| {
Ok((
key.extract::<PyBytes>(py)?.data(py).to_owned(),
value.extract::<PyBytes>(py)?.data(py).to_owned(),
))
})
.collect();
match pack_dirstate(
&dirstate_vec,
&copies?,
DirstateParents { p1, p2 },
now.as_object().extract::<i32>(py)?,
) {
Ok((packed, new_dirstate_vec)) => {
for (
filename,
DirstateEntry {
state,
mode,
size,
mtime,
},
) in new_dirstate_vec
{
dmap.set_item(
py,
PyBytes::new(py, &filename[..]),
decapsule_make_dirstate_tuple(py)?(
state as c_char,
mode,
size,
mtime,
),
)?;
}
Ok(PyBytes::new(py, &packed))
}
Err(error) => Err(PyErr::new::<exc::ValueError, _>(
py,
match error {
DirstatePackError::CorruptedParent => {
"expected a 20-byte hash".to_string()
}
DirstatePackError::CorruptedEntry(e) => e,
DirstatePackError::BadSize(expected, actual) => {
format!("bad dirstate size: {} != {}", actual, expected)
}
},
)),
}
}
/// Create the module, with `__package__` given from parent /// Create the module, with `__package__` given from parent
pub fn init_module(py: Python, package: &str) -> PyResult<PyModule> { pub fn init_module(py: Python, package: &str) -> PyResult<PyModule> {
let dotted_name = &format!("{}.dirstate", package); let dotted_name = &format!("{}.dirstate", package);
let m = PyModule::new(py, dotted_name)?; let m = PyModule::new(py, dotted_name)?;
m.add(py, "__package__", package)?; m.add(py, "__package__", package)?;
m.add(py, "__doc__", "Dirstate - Rust implementation")?; m.add(py, "__doc__", "Dirstate - Rust implementation")?;
m.add(
py,
"parse_dirstate",
py_fn!(
py,
parse_dirstate_wrapper(dmap: PyDict, copymap: PyDict, st: PyBytes)
),
)?;
m.add(
py,
"pack_dirstate",
py_fn!(
py,
pack_dirstate_wrapper(
dmap: PyDict,
copymap: PyDict,
pl: PyTuple,
now: PyInt
)
),
)?;
m.add_class::<Dirs>(py)?; m.add_class::<Dirs>(py)?;
let sys = PyModule::import(py, "sys")?; let sys = PyModule::import(py, "sys")?;
let sys_modules: PyDict = sys.get(py, "modules")?.extract(py)?; let sys_modules: PyDict = sys.get(py, "modules")?.extract(py)?;
sys_modules.set_item(py, dotted_name, &m)?; sys_modules.set_item(py, dotted_name, &m)?;
Ok(m) Ok(m)
} }

rust/hg-cpython/src/lib.rs

#[macro_use] #[macro_use]
extern crate cpython; extern crate cpython;
pub mod ancestors; pub mod ancestors;
mod cindex; mod cindex;
mod conversion; mod conversion;
pub mod dagops; pub mod dagops;
pub mod dirstate; pub mod dirstate;
pub mod parsers;
pub mod discovery; pub mod discovery;
pub mod exceptions; pub mod exceptions;
pub mod filepatterns; pub mod filepatterns;
py_module_initializer!(rustext, initrustext, PyInit_rustext, |py, m| { py_module_initializer!(rustext, initrustext, PyInit_rustext, |py, m| {
m.add( m.add(
py, py,
"__doc__", "__doc__",
"Mercurial core concepts - Rust implementation", "Mercurial core concepts - Rust implementation",
)?; )?;
let dotted_name: String = m.get(py, "__name__")?.extract(py)?; let dotted_name: String = m.get(py, "__name__")?.extract(py)?;
m.add(py, "ancestor", ancestors::init_module(py, &dotted_name)?)?; m.add(py, "ancestor", ancestors::init_module(py, &dotted_name)?)?;
m.add(py, "dagop", dagops::init_module(py, &dotted_name)?)?; m.add(py, "dagop", dagops::init_module(py, &dotted_name)?)?;
m.add(py, "discovery", discovery::init_module(py, &dotted_name)?)?; m.add(py, "discovery", discovery::init_module(py, &dotted_name)?)?;
m.add(py, "dirstate", dirstate::init_module(py, &dotted_name)?)?; m.add(py, "dirstate", dirstate::init_module(py, &dotted_name)?)?;
m.add( m.add(
py, py,
"filepatterns", "filepatterns",
filepatterns::init_module(py, &dotted_name)?, filepatterns::init_module(py, &dotted_name)?,
)?; )?;
m.add(
py,
"parsers",
parsers::init_parsers_module(py, &dotted_name)?,
)?;
m.add(py, "GraphError", py.get_type::<exceptions::GraphError>())?; m.add(py, "GraphError", py.get_type::<exceptions::GraphError>())?;
m.add( m.add(
py, py,
"PatternFileError", "PatternFileError",
py.get_type::<exceptions::PatternFileError>(), py.get_type::<exceptions::PatternFileError>(),
)?; )?;
m.add( m.add(
py, py,
"PatternError", "PatternError",
py.get_type::<exceptions::PatternError>(), py.get_type::<exceptions::PatternError>(),
)?; )?;
Ok(()) Ok(())
}); });

rust/hg-cpython/src/parsers.rs

  • This file was added.
// parsers.rs
//
// Copyright 2019 Raphaël Gomès <rgomes@octobus.net>
//
// This software may be used and distributed according to the terms of the
// GNU General Public License version 2 or any later version.
//! Bindings for the `hg::dirstate::parsers` module provided by the
//! `hg-core` package.
//!
//! From Python, this will be seen as `mercurial.rustext.parsers`
//!
use cpython::{
exc, PyBytes, PyDict, PyErr, PyInt, PyModule, PyResult, PyTuple, Python,
PythonObject, ToPyObject,
};
use hg::{
pack_dirstate, parse_dirstate, CopyVecEntry, DirstateEntry,
DirstatePackError, DirstateParents, DirstateParseError,
};
use std::collections::HashMap;
use libc::c_char;
use crate::dirstate::{decapsule_make_dirstate_tuple, extract_dirstate_vec};
fn parse_dirstate_wrapper(
py: Python,
dmap: PyDict,
copymap: PyDict,
st: PyBytes,
) -> PyResult<PyTuple> {
match parse_dirstate(st.data(py)) {
Ok((parents, dirstate_vec, copies)) => {
for (filename, entry) in dirstate_vec {
dmap.set_item(
py,
PyBytes::new(py, &filename[..]),
decapsule_make_dirstate_tuple(py)?(
entry.state as c_char,
entry.mode,
entry.size,
entry.mtime,
),
)?;
}
for CopyVecEntry { path, copy_path } in copies {
copymap.set_item(
py,
PyBytes::new(py, path),
PyBytes::new(py, copy_path),
)?;
}
Ok((PyBytes::new(py, parents.p1), PyBytes::new(py, parents.p2))
.to_py_object(py))
}
Err(e) => Err(PyErr::new::<exc::ValueError, _>(
py,
match e {
DirstateParseError::TooLittleData => {
"too little data for parents".to_string()
}
DirstateParseError::Overflow => {
"overflow in dirstate".to_string()
}
DirstateParseError::CorruptedEntry(e) => e,
},
)),
}
}
fn pack_dirstate_wrapper(
py: Python,
dmap: PyDict,
copymap: PyDict,
pl: PyTuple,
now: PyInt,
) -> PyResult<PyBytes> {
let p1 = pl.get_item(py, 0).extract::<PyBytes>(py)?;
let p1: &[u8] = p1.data(py);
let p2 = pl.get_item(py, 1).extract::<PyBytes>(py)?;
let p2: &[u8] = p2.data(py);
let dirstate_vec = extract_dirstate_vec(py, &dmap)?;
let copies: Result<HashMap<Vec<u8>, Vec<u8>>, PyErr> = copymap
.items(py)
.iter()
.map(|(key, value)| {
Ok((
key.extract::<PyBytes>(py)?.data(py).to_owned(),
value.extract::<PyBytes>(py)?.data(py).to_owned(),
))
})
.collect();
match pack_dirstate(
&dirstate_vec,
&copies?,
DirstateParents { p1, p2 },
now.as_object().extract::<i32>(py)?,
) {
Ok((packed, new_dirstate_vec)) => {
for (
filename,
DirstateEntry {
state,
mode,
size,
mtime,
},
) in new_dirstate_vec
{
dmap.set_item(
py,
PyBytes::new(py, &filename[..]),
decapsule_make_dirstate_tuple(py)?(
state as c_char,
mode,
size,
mtime,
),
)?;
}
Ok(PyBytes::new(py, &packed))
}
Err(error) => Err(PyErr::new::<exc::ValueError, _>(
py,
match error {
DirstatePackError::CorruptedParent => {
"expected a 20-byte hash".to_string()
}
DirstatePackError::CorruptedEntry(e) => e,
DirstatePackError::BadSize(expected, actual) => {
format!("bad dirstate size: {} != {}", actual, expected)
}
},
)),
}
}
/// Create the module, with `__package__` given from parent
pub fn init_parsers_module(py: Python, package: &str) -> PyResult<PyModule> {
let dotted_name = &format!("{}.parsers", package);
let m = PyModule::new(py, dotted_name)?;
m.add(py, "__package__", package)?;
m.add(py, "__doc__", "Parsers - Rust implementation")?;
m.add(
py,
"parse_dirstate",
py_fn!(
py,
parse_dirstate_wrapper(dmap: PyDict, copymap: PyDict, st: PyBytes)
),
)?;
m.add(
py,
"pack_dirstate",
py_fn!(
py,
pack_dirstate_wrapper(
dmap: PyDict,
copymap: PyDict,
pl: PyTuple,
now: PyInt
)
),
)?;
let sys = PyModule::import(py, "sys")?;
let sys_modules: PyDict = sys.get(py, "modules")?.extract(py)?;
sys_modules.set_item(py, dotted_name, &m)?;
Ok(m)
}

tests/fakedirstatewritetime.py

configtable = {} configtable = {}
configitem = registrar.configitem(configtable) configitem = registrar.configitem(configtable)
configitem(b'fakedirstatewritetime', b'fakenow', configitem(b'fakedirstatewritetime', b'fakenow',
default=None, default=None,
) )
parsers = policy.importmod(r'parsers') parsers = policy.importmod(r'parsers')
rustmod = policy.importrust(r'parsers')
def pack_dirstate(fakenow, orig, dmap, copymap, pl, now): def pack_dirstate(fakenow, orig, dmap, copymap, pl, now):
# execute what original parsers.pack_dirstate should do actually # execute what original parsers.pack_dirstate should do actually
# for consistency # for consistency
actualnow = int(now) actualnow = int(now)
for f, e in dmap.items(): for f, e in dmap.items():
if e[0] == 'n' and e[3] == actualnow: if e[0] == 'n' and e[3] == actualnow:
e = parsers.dirstatetuple(e[0], e[1], e[2], -1) e = parsers.dirstatetuple(e[0], e[1], e[2], -1)
# because replacing 'parsers.pack_dirstate' is also effective # because replacing 'parsers.pack_dirstate' is also effective
# in subrepos. # in subrepos.
return func() return func()
# parsing 'fakenow' in YYYYmmddHHMM format makes comparison between # parsing 'fakenow' in YYYYmmddHHMM format makes comparison between
# 'fakenow' value and 'touch -t YYYYmmddHHMM' argument easy # 'fakenow' value and 'touch -t YYYYmmddHHMM' argument easy
fakenow = dateutil.parsedate(fakenow, [b'%Y%m%d%H%M'])[0] fakenow = dateutil.parsedate(fakenow, [b'%Y%m%d%H%M'])[0]
if rustext is not None: if rustmod is not None:
orig_module = rustext.dirstate # The Rust implementation does not use public parse/pack dirstate
orig_pack_dirstate = rustext.dirstate.pack_dirstate # to prevent conversion round-trips
else: orig_dirstatemap_write = dirstate.dirstatemap.write
orig_module = parsers wrapper = lambda self, st, now: orig_dirstatemap_write(self,
orig_pack_dirstate = parsers.pack_dirstate st,
fakenow)
dirstate.dirstatemap.write = wrapper
orig_dirstate_getfsnow = dirstate._getfsnow orig_dirstate_getfsnow = dirstate._getfsnow
wrapper = lambda *args: pack_dirstate(fakenow, orig_pack_dirstate, *args) wrapper = lambda *args: pack_dirstate(fakenow, orig_pack_dirstate, *args)
orig_module = parsers
orig_pack_dirstate = parsers.pack_dirstate
orig_module.pack_dirstate = wrapper orig_module.pack_dirstate = wrapper
dirstate._getfsnow = lambda *args: fakenow dirstate._getfsnow = lambda *args: fakenow
try: try:
return func() return func()
finally: finally:
orig_module.pack_dirstate = orig_pack_dirstate orig_module.pack_dirstate = orig_pack_dirstate
dirstate._getfsnow = orig_dirstate_getfsnow dirstate._getfsnow = orig_dirstate_getfsnow
if rustmod is not None:
dirstate.dirstatemap.write = orig_dirstatemap_write
def _poststatusfixup(orig, workingctx, status, fixup): def _poststatusfixup(orig, workingctx, status, fixup):
ui = workingctx.repo().ui ui = workingctx.repo().ui
return fakewrite(ui, lambda : orig(workingctx, status, fixup)) return fakewrite(ui, lambda : orig(workingctx, status, fixup))
def markcommitted(orig, committablectx, node): def markcommitted(orig, committablectx, node):
ui = committablectx.repo().ui ui = committablectx.repo().ui
return fakewrite(ui, lambda : orig(committablectx, node)) return fakewrite(ui, lambda : orig(committablectx, node))
def extsetup(ui): def extsetup(ui):
extensions.wrapfunction(context.workingctx, '_poststatusfixup', extensions.wrapfunction(context.workingctx, '_poststatusfixup',
_poststatusfixup) _poststatusfixup)
extensions.wrapfunction(context.workingctx, 'markcommitted', extensions.wrapfunction(context.workingctx, 'markcommitted',
markcommitted) markcommitted)