This is an archive of the discontinued Mercurial Phabricator instance.

Differential D6633

rust-dirstate: rust-cpython bridge for dirstatemap
ClosedPublic

Authored by Alphare on Jul 10 2019, 10:34 AM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG4e8f504424f3: rust-dirstate: rust-cpython bridge for dirstatemap
rHGe0f7ee53107e: rust-dirstate: rust-cpython bridge for dirstatemap
Summary

This change also showcases the limitations of the py_shared_ref! macro.

See the previous commit 'rust-dirstate: rust implementation of dirstatemap`
for an explanation for the TODOs in the code.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

Alphare created this revision.Jul 10 2019, 10:34 AM
Alphare updated this revision to Diff 16052.Jul 24 2019, 11:23 AM
Alphare added a commit: rHGe0f7ee53107e: rust-dirstate: rust-cpython bridge for dirstatemap.Aug 14 2019, 4:41 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHGe0f7ee53107e: rust-dirstate: rust-cpython bridge for dirstatemap (authored by Alphare). · Explain Why
This revision was automatically updated to reflect the committed changes.
yuja added a subscriber: yuja.Aug 17 2019, 2:49 AM

+ def getdirs(&self) -> PyResult<Dirs> {
+ TODO don't copy, share the reference
+ self.inner(py).borrow_mut().set_dirs();
+ Dirs::from_inner(
+ py,
+ DirsMultiset::new(
+ DirsIterable::Dirstate(&self.inner(py).borrow()),
+ Some(EntryState::Removed),
+ ),
+ )
+ }
+ def getalldirs(&self) -> PyResult<Dirs> {
+ TODO don't copy, share the reference
+ self.inner(py).borrow_mut().set_all_dirs();
+ Dirs::from_inner(
+ py,
+ DirsMultiset::new(
+ DirsIterable::Dirstate(&self.inner(py).borrow()),
+ None,
+ ),
+ )
+ }

How do these set_dirs/set_all_dirs work? IIUC, the return value is built
from state_map.

Revision Contents
Changeset List

Diff 15851

rust/hg-cpython/src/dirstate/copymap.rs

  • This file was added.
// copymap.rs
//
// Copyright 2019 Raphaël Gomès <rgomes@octobus.net>
//
// This software may be used and distributed according to the terms of the
// GNU General Public License version 2 or any later version.
//! Bindings for `hg::dirstate::dirstate_map::CopyMap` provided by the
//! `hg-core` package.
use cpython::{PyBytes, PyClone, PyDict, PyObject, PyResult, Python};
use std::cell::RefCell;
use std::collections::hash_map::Iter;
use dirstate::dirstate_map::{DirstateMap, DirstateMapLeakedRef};
py_class!(pub class CopyMap |py| {
data dirstate_map: DirstateMap;
def __getitem__(&self, key: PyObject) -> PyResult<PyBytes> {
(*self.dirstate_map(py)).copymapgetitem(py, key)
}
def __len__(&self) -> PyResult<usize> {
self.dirstate_map(py).copymaplen(py)
}
def __contains__(&self, key: PyObject) -> PyResult<bool> {
self.dirstate_map(py).copymapcontains(py, key)
}
def get(
&self,
key: PyObject,
default: Option<PyObject> = None
) -> PyResult<Option<PyObject>> {
self.dirstate_map(py).copymapget(py, key, default)
}
def pop(
&self,
key: PyObject,
default: Option<PyObject> = None
) -> PyResult<Option<PyObject>> {
self.dirstate_map(py).copymappop(py, key, default)
}
def __iter__(&self) -> PyResult<CopyMapKeysIterator> {
self.dirstate_map(py).copymapiter(py)
}
// Python's `dict()` builtin works with either a subclass of dict
// or an abstract mapping. Said mapping needs to implement `__getitem__`
// and `keys`.
def keys(&self) -> PyResult<CopyMapKeysIterator> {
self.dirstate_map(py).copymapiter(py)
}
def items(&self) -> PyResult<CopyMapItemsIterator> {
self.dirstate_map(py).copymapitemsiter(py)
}
def iteritems(&self) -> PyResult<CopyMapItemsIterator> {
self.dirstate_map(py).copymapitemsiter(py)
}
def __setitem__(
&self,
key: PyObject,
item: PyObject
) -> PyResult<()> {
self.dirstate_map(py).copymapsetitem(py, key, item)?;
Ok(())
}
def copy(&self) -> PyResult<PyDict> {
self.dirstate_map(py).copymapcopy(py)
}
});
impl CopyMap {
pub fn from_inner(py: Python, dm: DirstateMap) -> PyResult<Self> {
Self::create_instance(py, dm)
}
fn translate_key(
py: Python,
res: (&Vec<u8>, &Vec<u8>),
) -> PyResult<Option<PyBytes>> {
Ok(Some(PyBytes::new(py, res.0)))
}
fn translate_key_value(
py: Python,
res: (&Vec<u8>, &Vec<u8>),
) -> PyResult<Option<(PyBytes, PyBytes)>> {
let (k, v) = res;
Ok(Some((PyBytes::new(py, k), PyBytes::new(py, v))))
}
}
py_shared_mapping_iterator!(
CopyMapKeysIterator,
DirstateMapLeakedRef,
Iter,
Vec<u8>,
Vec<u8>,
CopyMap::translate_key,
Option<PyBytes>
);
py_shared_mapping_iterator!(
CopyMapItemsIterator,
DirstateMapLeakedRef,
Iter,
Vec<u8>,
Vec<u8>,
CopyMap::translate_key_value,
Option<(PyBytes, PyBytes)>
);

rust/hg-cpython/src/dirstate/dirstate_map.rs

  • This file was added.
// dirstate_map.rs
//
// Copyright 2019 Raphaël Gomès <rgomes@octobus.net>
//
// This software may be used and distributed according to the terms of the
// GNU General Public License version 2 or any later version.
//! Bindings for the `hg::dirstate::dirstate_map` file provided by the
//! `hg-core` package.
use std::cell::{RefCell, RefMut};
use std::collections::hash_map::Iter;
use std::convert::TryInto;
use std::time::Duration;
use cpython::{
exc, ObjectProtocol, PyBool, PyBytes, PyClone, PyDict, PyErr, PyObject,
PyResult, PyTuple, Python, PythonObject, ToPyObject,
};
use libc::c_char;
use dirstate::copymap::{CopyMap, CopyMapItemsIterator, CopyMapKeysIterator};
use dirstate::decapsule_make_dirstate_tuple;
use dirstate::dirs_multiset::Dirs;
use exceptions::AlreadyBorrowed;
use hg::{
DirsIterable, DirsMultiset, DirstateEntry, DirstateMap as RustDirstateMap,
DirstateParents, DirstateParseError, EntryState,
};
/// TODO
/// This object needs to share references to multiple members of its Rust
/// inner struct, namely `copy_map`, `dirs` and `all_dirs`.
/// Right now `CopyMap` is done, but it needs to have an explicit reference
/// to `RustDirstateMap` which itself needs to have an encapsulation for
/// every method in `CopyMap` (copymapcopy, etc.).
/// This is ugly and hard to maintain.
/// The same logic applies to `dirs` and `all_dirs`, however the `Dirs`
/// `py_class!` is already implemented and does not mention
/// `RustDirstateMap`, rightfully so.
/// All attributes also have to have a separate refcount data attribute for
/// leaks, with all methods that go along for reference sharing.
py_class!(pub class DirstateMap |py| {
data inner: RefCell<RustDirstateMap>;
data leak_count: RefCell<usize>;
def __new__(_cls, _root: PyObject) -> PyResult<Self> {
let inner = RustDirstateMap::default();
Self::create_instance(py, RefCell::new(inner), RefCell::new(0))
}
def clear(&self) -> PyResult<PyObject> {
self.borrow_mut(py)?.clear();
Ok(py.None())
}
def get(
&self,
key: PyObject,
default: Option<PyObject> = None
) -> PyResult<Option<PyObject>> {
let key = key.extract::<PyBytes>(py)?;
match self.inner(py).borrow().get(key.data(py)) {
Some(entry) => {
// Explicitly go through u8 first, then cast to
// platform-specific `c_char`.
let state: u8 = entry.state.into();
Ok(Some(decapsule_make_dirstate_tuple(py)?(
state as c_char,
entry.mode,
entry.size,
entry.mtime,
)))
},
None => Ok(default)
}
}
def addfile(
&self,
f: PyObject,
oldstate: PyObject,
state: PyObject,
mode: PyObject,
size: PyObject,
mtime: PyObject
) -> PyResult<PyObject> {
self.borrow_mut(py)?.add_file(
f.extract::<PyBytes>(py)?.data(py),
oldstate.extract::<PyBytes>(py)?.data(py)[0]
.try_into()
.map_err(|e: DirstateParseError| {
PyErr::new::<exc::ValueError, _>(py, e.to_string())
})?,
DirstateEntry {
state: state.extract::<PyBytes>(py)?.data(py)[0]
.try_into()
.map_err(|e: DirstateParseError| {
PyErr::new::<exc::ValueError, _>(py, e.to_string())
})?,
mode: mode.extract(py)?,
size: size.extract(py)?,
mtime: mtime.extract(py)?,
},
);
Ok(py.None())
}
def removefile(
&self,
f: PyObject,
oldstate: PyObject,
size: PyObject
) -> PyResult<PyObject> {
self.borrow_mut(py)?
.remove_file(
f.extract::<PyBytes>(py)?.data(py),
oldstate.extract::<PyBytes>(py)?.data(py)[0]
.try_into()
.map_err(|e: DirstateParseError| {
PyErr::new::<exc::ValueError, _>(py, e.to_string())
})?,
size.extract(py)?,
)
.or_else(|_| {
Err(PyErr::new::<exc::OSError, _>(
py,
"Dirstate error".to_string(),
))
})?;
Ok(py.None())
}
def dropfile(
&self,
f: PyObject,
oldstate: PyObject
) -> PyResult<PyBool> {
self.borrow_mut(py)?
.drop_file(
f.extract::<PyBytes>(py)?.data(py),
oldstate.extract::<PyBytes>(py)?.data(py)[0]
.try_into()
.map_err(|e: DirstateParseError| {
PyErr::new::<exc::ValueError, _>(py, e.to_string())
})?,
)
.and_then(|b| Ok(b.to_py_object(py)))
.or_else(|_| {
Err(PyErr::new::<exc::OSError, _>(
py,
"Dirstate error".to_string(),
))
})
}
def clearambiguoustimes(
&self,
files: PyObject,
now: PyObject
) -> PyResult<PyObject> {
let files: PyResult<Vec<Vec<u8>>> = files
.iter(py)?
.map(|filename| {
Ok(filename?.extract::<PyBytes>(py)?.data(py).to_owned())
})
.collect();
self.inner(py)
.borrow_mut()
.clear_ambiguous_times(files?, now.extract(py)?);
Ok(py.None())
}
// TODO share the reference
def nonnormalentries(&self) -> PyResult<PyObject> {
let (non_normal, other_parent) =
self.inner(py).borrow().non_normal_other_parent_entries();
let locals = PyDict::new(py);
locals.set_item(
py,
"non_normal",
non_normal
.iter()
.map(|v| PyBytes::new(py, &v))
.collect::<Vec<PyBytes>>()
.to_py_object(py),
)?;
locals.set_item(
py,
"other_parent",
other_parent
.iter()
.map(|v| PyBytes::new(py, &v))
.collect::<Vec<PyBytes>>()
.to_py_object(py),
)?;
py.eval("set(non_normal), set(other_parent)", None, Some(&locals))
}
def hastrackeddir(&self, d: PyObject) -> PyResult<PyBool> {
let d = d.extract::<PyBytes>(py)?;
Ok(self
.inner(py)
.borrow_mut()
.has_tracked_dir(d.data(py))
.to_py_object(py))
}
def hasdir(&self, d: PyObject) -> PyResult<PyBool> {
let d = d.extract::<PyBytes>(py)?;
Ok(self
.inner(py)
.borrow_mut()
.has_dir(d.data(py))
.to_py_object(py))
}
def parents(&self, st: PyObject) -> PyResult<PyTuple> {
self.inner(py)
.borrow_mut()
.parents(st.extract::<PyBytes>(py)?.data(py))
.and_then(|d| {
Ok((PyBytes::new(py, &d.p1), PyBytes::new(py, &d.p2))
.to_py_object(py))
})
.or_else(|_| {
Err(PyErr::new::<exc::OSError, _>(
py,
"Dirstate error".to_string(),
))
})
}
def setparents(&self, p1: PyObject, p2: PyObject) -> PyResult<PyObject> {
let p1 = p1.extract::<PyBytes>(py)?.data(py).to_vec();
let p2 = p2.extract::<PyBytes>(py)?.data(py).to_vec();
self.inner(py)
.borrow_mut()
.set_parents(DirstateParents { p1, p2 });
Ok(py.None())
}
def read(&self, st: PyObject) -> PyResult<Option<PyObject>> {
match self
.inner(py)
.borrow_mut()
.read(st.extract::<PyBytes>(py)?.data(py))
{
Ok(Some(parents)) => Ok(Some(
(PyBytes::new(py, &parents.p1), PyBytes::new(py, &parents.p2))
.to_py_object(py)
.into_object(),
)),
Ok(None) => Ok(Some(py.None())),
Err(_) => Err(PyErr::new::<exc::OSError, _>(
py,
"Dirstate error".to_string(),
)),
}
}
def write(
&self,
p1: PyObject,
p2: PyObject,
now: PyObject
) -> PyResult<PyBytes> {
let now = Duration::new(now.extract(py)?, 0);
let parents = DirstateParents {
p1: p1.extract::<PyBytes>(py)?.data(py).to_owned(),
p2: p2.extract::<PyBytes>(py)?.data(py).to_owned(),
};
match self.borrow_mut(py)?.pack(parents, now) {
Ok(packed) => Ok(PyBytes::new(py, &packed)),
Err(_) => Err(PyErr::new::<exc::OSError, _>(
py,
"Dirstate error".to_string(),
)),
}
}
def filefoldmapasdict(&self) -> PyResult<PyDict> {
let dict = PyDict::new(py);
for (key, value) in
self.borrow_mut(py)?.build_file_fold_map().iter()
{
dict.set_item(py, key, value)?;
}
Ok(dict)
}
def __len__(&self) -> PyResult<usize> {
Ok(self.inner(py).borrow().len())
}
def __contains__(&self, key: PyObject) -> PyResult<bool> {
let key = key.extract::<PyBytes>(py)?;
Ok(self.inner(py).borrow().contains_key(key.data(py)))
}
def __getitem__(&self, key: PyObject) -> PyResult<PyObject> {
let key = key.extract::<PyBytes>(py)?;
let key = key.data(py);
match self.inner(py).borrow().get(key) {
Some(entry) => {
// Explicitly go through u8 first, then cast to
// platform-specific `c_char`.
let state: u8 = entry.state.into();
Ok(decapsule_make_dirstate_tuple(py)?(
state as c_char,
entry.mode,
entry.size,
entry.mtime,
))
},
None => Err(PyErr::new::<exc::KeyError, _>(
py,
String::from_utf8_lossy(key),
)),
}
}
def keys(&self) -> PyResult<DirstateMapKeysIterator> {
DirstateMapKeysIterator::create_instance(
py,
RefCell::new(Some(DirstateMapLeakedRef::new(py, &self))),
RefCell::new(self.leak_immutable(py).iter()),
)
}
def items(&self) -> PyResult<DirstateMapItemsIterator> {
DirstateMapItemsIterator::create_instance(
py,
RefCell::new(Some(DirstateMapLeakedRef::new(py, &self))),
RefCell::new(self.leak_immutable(py).iter()),
)
}
def __iter__(&self) -> PyResult<DirstateMapKeysIterator> {
DirstateMapKeysIterator::create_instance(
py,
RefCell::new(Some(DirstateMapLeakedRef::new(py, &self))),
RefCell::new(self.leak_immutable(py).iter()),
)
}
def getdirs(&self) -> PyResult<Dirs> {
// TODO don't copy, share the reference
self.inner(py).borrow_mut().set_dirs();
Dirs::from_inner(
py,
DirsMultiset::new(
DirsIterable::Dirstate(&self.inner(py).borrow()),
Some(EntryState::Removed),
),
)
}
def getalldirs(&self) -> PyResult<Dirs> {
// TODO don't copy, share the reference
self.inner(py).borrow_mut().set_all_dirs();
Dirs::from_inner(
py,
DirsMultiset::new(
DirsIterable::Dirstate(&self.inner(py).borrow()),
None,
),
)
}
// TODO all copymap* methods, see docstring above
def copymapcopy(&self) -> PyResult<PyDict> {
let dict = PyDict::new(py);
for (key, value) in self.inner(py).borrow().copy_map.iter() {
dict.set_item(py, PyBytes::new(py, key), PyBytes::new(py, value))?;
}
Ok(dict)
}
def copymapgetitem(&self, key: PyObject) -> PyResult<PyBytes> {
let key = key.extract::<PyBytes>(py)?;
match self.inner(py).borrow().copy_map.get(key.data(py)) {
Some(copy) => Ok(PyBytes::new(py, copy)),
None => Err(PyErr::new::<exc::KeyError, _>(
py,
String::from_utf8_lossy(key.data(py)),
)),
}
}
def copymap(&self) -> PyResult<CopyMap> {
CopyMap::from_inner(py, self.clone_ref(py))
}
def copymaplen(&self) -> PyResult<usize> {
Ok(self.inner(py).borrow().copy_map.len())
}
def copymapcontains(&self, key: PyObject) -> PyResult<bool> {
let key = key.extract::<PyBytes>(py)?;
Ok(self.inner(py).borrow().copy_map.contains_key(key.data(py)))
}
def copymapget(
&self,
key: PyObject,
default: Option<PyObject>
) -> PyResult<Option<PyObject>> {
let key = key.extract::<PyBytes>(py)?;
match self.inner(py).borrow().copy_map.get(key.data(py)) {
Some(copy) => Ok(Some(PyBytes::new(py, copy).into_object())),
None => Ok(default),
}
}
def copymapsetitem(
&self,
key: PyObject,
value: PyObject
) -> PyResult<PyObject> {
let key = key.extract::<PyBytes>(py)?;
let value = value.extract::<PyBytes>(py)?;
self.inner(py)
.borrow_mut()
.copy_map
.insert(key.data(py).to_vec(), value.data(py).to_vec());
Ok(py.None())
}
def copymappop(
&self,
key: PyObject,
default: Option<PyObject>
) -> PyResult<Option<PyObject>> {
let key = key.extract::<PyBytes>(py)?;
match self.inner(py).borrow_mut().copy_map.remove(key.data(py)) {
Some(_) => Ok(None),
None => Ok(default),
}
}
def copymapiter(&self) -> PyResult<CopyMapKeysIterator> {
CopyMapKeysIterator::from_inner(
py,
Some(DirstateMapLeakedRef::new(py, &self)),
self.leak_immutable(py).copy_map.iter(),
)
}
def copymapitemsiter(&self) -> PyResult<CopyMapItemsIterator> {
CopyMapItemsIterator::from_inner(
py,
Some(DirstateMapLeakedRef::new(py, &self)),
self.leak_immutable(py).copy_map.iter(),
)
}
});
impl DirstateMap {
fn translate_key(
py: Python,
res: (&Vec<u8>, &DirstateEntry),
) -> PyResult<Option<PyBytes>> {
Ok(Some(PyBytes::new(py, res.0)))
}
fn translate_key_value(
py: Python,
res: (&Vec<u8>, &DirstateEntry),
) -> PyResult<Option<(PyBytes, PyObject)>> {
let (f, entry) = res;
// Explicitly go through u8 first, then cast to
// platform-specific `c_char`.
let state: u8 = entry.state.into();
Ok(Some((
PyBytes::new(py, f),
decapsule_make_dirstate_tuple(py)?(
state as c_char,
entry.mode,
entry.size,
entry.mtime,
),
)))
}
}
py_shared_ref!(DirstateMap, RustDirstateMap, inner, DirstateMapLeakedRef);
py_shared_mapping_iterator!(
DirstateMapKeysIterator,
DirstateMapLeakedRef,
Iter,
Vec<u8>,
DirstateEntry,
DirstateMap::translate_key,
Option<PyBytes>
);
py_shared_mapping_iterator!(
DirstateMapItemsIterator,
DirstateMapLeakedRef,
Iter,
Vec<u8>,
DirstateEntry,
DirstateMap::translate_key_value,
Option<(PyBytes, PyObject)>
);

rust/hg-cpython/src/dirstate/mod.rs

extern crate python27_sys as python_sys; extern crate python27_sys as python_sys;
#[cfg(feature = "python3")] #[cfg(feature = "python3")]
extern crate python3_sys as python_sys; extern crate python3_sys as python_sys;
use self::python_sys::PyCapsule_Import; use self::python_sys::PyCapsule_Import;
use libc::{c_char, c_int}; use libc::{c_char, c_int};
use std::mem::transmute; use std::mem::transmute;
mod copymap;
mod dirs_multiset; mod dirs_multiset;
mod dirstate_map;
use dirstate::dirs_multiset::Dirs; use dirstate::dirs_multiset::Dirs;
use std::convert::TryFrom; use dirstate::dirstate_map::DirstateMap;
use exceptions::AlreadyBorrowed; use exceptions::AlreadyBorrowed;
use std::convert::TryFrom;
/// C code uses a custom `dirstate_tuple` type, checks in multiple instances /// C code uses a custom `dirstate_tuple` type, checks in multiple instances
/// for this type, and raises a Python `Exception` if the check does not pass. /// for this type, and raises a Python `Exception` if the check does not pass.
/// Because this type differs only in name from the regular Python tuple, it /// Because this type differs only in name from the regular Python tuple, it
/// would be a good idea in the near future to remove it entirely to allow /// would be a good idea in the near future to remove it entirely to allow
/// for a pure Python tuple of the same effective structure to be used, /// for a pure Python tuple of the same effective structure to be used,
/// rendering this type and the capsule below useless. /// rendering this type and the capsule below useless.
type MakeDirstateTupleFn = extern "C" fn( type MakeDirstateTupleFn = extern "C" fn(
pub fn init_module(py: Python, package: &str) -> PyResult<PyModule> { pub fn init_module(py: Python, package: &str) -> PyResult<PyModule> {
let dotted_name = &format!("{}.dirstate", package); let dotted_name = &format!("{}.dirstate", package);
let m = PyModule::new(py, dotted_name)?; let m = PyModule::new(py, dotted_name)?;
m.add(py, "__package__", package)?; m.add(py, "__package__", package)?;
m.add(py, "__doc__", "Dirstate - Rust implementation")?; m.add(py, "__doc__", "Dirstate - Rust implementation")?;
m.add_class::<Dirs>(py)?; m.add_class::<Dirs>(py)?;
m.add_class::<DirstateMap>(py)?;
m.add(py, "AlreadyBorrowed", py.get_type::<AlreadyBorrowed>())?; m.add(py, "AlreadyBorrowed", py.get_type::<AlreadyBorrowed>())?;
let sys = PyModule::import(py, "sys")?; let sys = PyModule::import(py, "sys")?;
let sys_modules: PyDict = sys.get(py, "modules")?.extract(py)?; let sys_modules: PyDict = sys.get(py, "modules")?.extract(py)?;
sys_modules.set_item(py, dotted_name, &m)?; sys_modules.set_item(py, dotted_name, &m)?;
Ok(m) Ok(m)
} }