( )⚙ D10557 dirstate-tree: Make `DirstateMap` borrow from a bytes buffer

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10557

dirstate-tree: Make `DirstateMap` borrow from a bytes buffer
ClosedPublic

Authored by SimonSapin on May 3 2021, 6:26 AM.

Details

Reviewers
Alphare
Group Reviewers
hg-reviewers
Commits
rHGd8ac62374943: dirstate-tree: Make `DirstateMap` borrow from a bytes buffer
rHGe14189afbd29: dirstate-tree: Make `DirstateMap` borrow from a bytes buffer
Summary

… that has the contents of the .hg/dirstate file.
This only applies to the tree-based flavor of DirstateMap.

For now only the entire &[u8] slice is stored, so this is not useful yet.

Adding a lifetime parameter to the DirstateMap struct (in hg-core) makes
Python bindings non-trivial because we keep that struct in a Python object
that has a dynamic lifetime tied to Python’s reference-counting and GC.
As long as we keep the PyBytes that owns the borrowed bytes buffer next to
the borrowing struct, the buffer will live long enough for the borrows to stay
valid. However this relationship cannot be expressed in safe Rust code in a
way that would statisfy they borrow-checker. We use unsafe code to erase
that lifetime parameter, and encapsulate it in a safe abstraction similar to
the owning-ref crate: https://docs.rs/owning_ref/

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

SimonSapin created this revision.May 3 2021, 6:26 AM
Alphare accepted this revision.May 3 2021, 12:21 PM
Alphare added a subscriber: Alphare.

Note: Simon and I discussed this at length beforehand.

This revision is now accepted and ready to land.May 3 2021, 12:21 PM
baymax updated this revision to Diff 27633.May 6 2021, 12:04 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

Revision Contents
Changeset List

Diff 27667

rust/hg-core/src/dirstate/dirstate_map.rs

&mut self, &mut self,
directory: &HgPath, directory: &HgPath,
) -> Result<bool, DirstateMapError> { ) -> Result<bool, DirstateMapError> {
self.set_all_dirs()?; self.set_all_dirs()?;
Ok(self.all_dirs.as_ref().unwrap().contains(directory)) Ok(self.all_dirs.as_ref().unwrap().contains(directory))
} }
#[timed] #[timed]
pub fn read<'a>( pub fn read(
&mut self, &mut self,
file_contents: &'a [u8], file_contents: &[u8],
) -> Result<Option<&'a DirstateParents>, DirstateError> { ) -> Result<Option<DirstateParents>, DirstateError> {
if file_contents.is_empty() { if file_contents.is_empty() {
return Ok(None); return Ok(None);
} }
let (parents, entries, copies) = parse_dirstate(file_contents)?; let (parents, entries, copies) = parse_dirstate(file_contents)?;
self.state_map.extend( self.state_map.extend(
entries entries
.into_iter() .into_iter()
.map(|(path, entry)| (path.to_owned(), entry)), .map(|(path, entry)| (path.to_owned(), entry)),
); );
self.copy_map.extend( self.copy_map.extend(
copies copies
.into_iter() .into_iter()
.map(|(path, copy)| (path.to_owned(), copy.to_owned())), .map(|(path, copy)| (path.to_owned(), copy.to_owned())),
); );
Ok(Some(parents)) Ok(Some(parents.clone()))
} }
pub fn pack( pub fn pack(
&mut self, &mut self,
parents: DirstateParents, parents: DirstateParents,
now: Timestamp, now: Timestamp,
) -> Result<Vec<u8>, DirstateError> { ) -> Result<Vec<u8>, DirstateError> {
let packed = let packed =

rust/hg-core/src/dirstate_tree/dirstate_map.rs

use crate::DirstateStatus; use crate::DirstateStatus;
use crate::EntryState; use crate::EntryState;
use crate::FastHashMap; use crate::FastHashMap;
use crate::PatternFileWarning; use crate::PatternFileWarning;
use crate::StateMapIter; use crate::StateMapIter;
use crate::StatusError; use crate::StatusError;
use crate::StatusOptions; use crate::StatusOptions;
pub struct DirstateMap { pub struct DirstateMap<'on_disk> {
/// Contents of the `.hg/dirstate` file
on_disk: &'on_disk [u8],
pub(super) root: ChildNodes, pub(super) root: ChildNodes,
/// Number of nodes anywhere in the tree that have `.entry.is_some()`. /// Number of nodes anywhere in the tree that have `.entry.is_some()`.
nodes_with_entry_count: usize, nodes_with_entry_count: usize,
/// Number of nodes anywhere in the tree that have /// Number of nodes anywhere in the tree that have
/// `.copy_source.is_some()`. /// `.copy_source.is_some()`.
nodes_with_copy_source_count: usize, nodes_with_copy_source_count: usize,
/// `(full_path, entry, copy_source)` /// `(full_path, entry, copy_source)`
type NodeDataMut<'a> = ( type NodeDataMut<'a> = (
&'a WithBasename<HgPathBuf>, &'a WithBasename<HgPathBuf>,
&'a mut Option<DirstateEntry>, &'a mut Option<DirstateEntry>,
&'a mut Option<HgPathBuf>, &'a mut Option<HgPathBuf>,
); );
impl DirstateMap { impl<'on_disk> DirstateMap<'on_disk> {
pub fn new() -> Self { pub fn new(
Self { on_disk: &'on_disk [u8],
) -> Result<(Self, Option<DirstateParents>), DirstateError> {
let mut map = Self {
on_disk,
root: ChildNodes::default(), root: ChildNodes::default(),
nodes_with_entry_count: 0, nodes_with_entry_count: 0,
nodes_with_copy_source_count: 0, nodes_with_copy_source_count: 0,
};
let parents = map.read()?;
Ok((map, parents))
}
/// Should only be called in `new`
#[timed]
fn read(&mut self) -> Result<Option<DirstateParents>, DirstateError> {
if self.on_disk.is_empty() {
return Ok(None);
}
let parents = parse_dirstate_entries(
self.on_disk,
|path, entry, copy_source| {
let tracked = entry.state.is_tracked();
let node = Self::get_or_insert_node_tracing_ancestors(
&mut self.root,
path,
|ancestor| {
if tracked {
ancestor.tracked_descendants_count += 1
}
},
);
assert!(
node.entry.is_none(),
"duplicate dirstate entry in read"
);
assert!(
node.copy_source.is_none(),
"duplicate dirstate entry in read"
);
node.entry = Some(*entry);
node.copy_source = copy_source.map(HgPath::to_owned);
self.nodes_with_entry_count += 1;
if copy_source.is_some() {
self.nodes_with_copy_source_count += 1
} }
},
)?;
Ok(Some(parents.clone()))
} }
fn get_node(&self, path: &HgPath) -> Option<&Node> { fn get_node(&self, path: &HgPath) -> Option<&Node> {
let mut children = &self.root; let mut children = &self.root;
let mut components = path.components(); let mut components = path.components();
let mut component = let mut component =
components.next().expect("expected at least one components"); components.next().expect("expected at least one components");
loop { loop {
} else { } else {
// Reached the bottom of the stack, we’re done // Reached the bottom of the stack, we’re done
None None
} }
}) })
} }
} }
impl super::dispatch::DirstateMapMethods for DirstateMap { impl<'on_disk> super::dispatch::DirstateMapMethods for DirstateMap<'on_disk> {
fn clear(&mut self) { fn clear(&mut self) {
self.root.clear(); self.root.clear();
self.nodes_with_entry_count = 0; self.nodes_with_entry_count = 0;
self.nodes_with_copy_source_count = 0; self.nodes_with_copy_source_count = 0;
} }
fn add_file( fn add_file(
&mut self, &mut self,
// A node without a `DirstateEntry` was created to hold child // A node without a `DirstateEntry` was created to hold child
// nodes, and is therefore a directory. // nodes, and is therefore a directory.
Ok(node.entry.is_none()) Ok(node.entry.is_none())
} else { } else {
Ok(false) Ok(false)
} }
} }
#[timed]
fn read<'a>(
&mut self,
file_contents: &'a [u8],
) -> Result<Option<&'a DirstateParents>, DirstateError> {
if file_contents.is_empty() {
return Ok(None);
}
let parents = parse_dirstate_entries(
file_contents,
|path, entry, copy_source| {
let tracked = entry.state.is_tracked();
let node = Self::get_or_insert_node_tracing_ancestors(
&mut self.root,
path,
|ancestor| {
if tracked {
ancestor.tracked_descendants_count += 1
}
},
);
assert!(
node.entry.is_none(),
"duplicate dirstate entry in read"
);
assert!(
node.copy_source.is_none(),
"duplicate dirstate entry in read"
);
node.entry = Some(*entry);
node.copy_source = copy_source.map(HgPath::to_owned);
self.nodes_with_entry_count += 1;
if copy_source.is_some() {
self.nodes_with_copy_source_count += 1
}
},
)?;
Ok(Some(parents))
}
fn pack( fn pack(
&mut self, &mut self,
parents: DirstateParents, parents: DirstateParents,
now: Timestamp, now: Timestamp,
) -> Result<Vec<u8>, DirstateError> { ) -> Result<Vec<u8>, DirstateError> {
// Optizimation (to be measured?): pre-compute size to avoid `Vec` // Optizimation (to be measured?): pre-compute size to avoid `Vec`
// reallocations // reallocations
let mut size = parents.as_bytes().len(); let mut size = parents.as_bytes().len();

rust/hg-core/src/dirstate_tree/dispatch.rs

directory: &HgPath, directory: &HgPath,
) -> Result<bool, DirstateMapError>; ) -> Result<bool, DirstateMapError>;
fn has_dir( fn has_dir(
&mut self, &mut self,
directory: &HgPath, directory: &HgPath,
) -> Result<bool, DirstateMapError>; ) -> Result<bool, DirstateMapError>;
fn read<'a>(
&mut self,
file_contents: &'a [u8],
) -> Result<Option<&'a DirstateParents>, DirstateError>;
fn pack( fn pack(
&mut self, &mut self,
parents: DirstateParents, parents: DirstateParents,
now: Timestamp, now: Timestamp,
) -> Result<Vec<u8>, DirstateError>; ) -> Result<Vec<u8>, DirstateError>;
fn set_all_dirs(&mut self) -> Result<(), DirstateMapError>; fn set_all_dirs(&mut self) -> Result<(), DirstateMapError>;
fn has_dir( fn has_dir(
&mut self, &mut self,
directory: &HgPath, directory: &HgPath,
) -> Result<bool, DirstateMapError> { ) -> Result<bool, DirstateMapError> {
self.has_dir(directory) self.has_dir(directory)
} }
fn read<'a>(
&mut self,
file_contents: &'a [u8],
) -> Result<Option<&'a DirstateParents>, DirstateError> {
self.read(file_contents)
}
fn pack( fn pack(
&mut self, &mut self,
parents: DirstateParents, parents: DirstateParents,
now: Timestamp, now: Timestamp,
) -> Result<Vec<u8>, DirstateError> { ) -> Result<Vec<u8>, DirstateError> {
self.pack(parents, now) self.pack(parents, now)
} }

rust/hg-cpython/src/dirstate.rs

// dirstate.rs // dirstate.rs
// //
// Copyright 2019 Raphaël Gomès <rgomes@octobus.net> // Copyright 2019 Raphaël Gomès <rgomes@octobus.net>
// //
// This software may be used and distributed according to the terms of the // This software may be used and distributed according to the terms of the
// GNU General Public License version 2 or any later version. // GNU General Public License version 2 or any later version.
//! Bindings for the `hg::dirstate` module provided by the //! Bindings for the `hg::dirstate` module provided by the
//! `hg-core` package. //! `hg-core` package.
//! //!
//! From Python, this will be seen as `mercurial.rustext.dirstate` //! From Python, this will be seen as `mercurial.rustext.dirstate`
mod copymap; mod copymap;
mod dirs_multiset; mod dirs_multiset;
mod dirstate_map; mod dirstate_map;
mod dispatch;
mod non_normal_entries; mod non_normal_entries;
mod owning;
mod status; mod status;
use crate::{ use crate::{
dirstate::{ dirstate::{
dirs_multiset::Dirs, dirstate_map::DirstateMap, status::status_wrapper, dirs_multiset::Dirs, dirstate_map::DirstateMap, status::status_wrapper,
}, },
exceptions, exceptions,
}; };
use cpython::{ use cpython::{

rust/hg-cpython/src/dirstate/dirstate_map.rs

UnsafePyLeaked, UnsafePyLeaked,
}; };
use crate::{ use crate::{
dirstate::copymap::{CopyMap, CopyMapItemsIterator, CopyMapKeysIterator}, dirstate::copymap::{CopyMap, CopyMapItemsIterator, CopyMapKeysIterator},
dirstate::non_normal_entries::{ dirstate::non_normal_entries::{
NonNormalEntries, NonNormalEntriesIterator, NonNormalEntries, NonNormalEntriesIterator,
}, },
dirstate::owning::OwningDirstateMap,
dirstate::{dirs_multiset::Dirs, make_dirstate_tuple}, dirstate::{dirs_multiset::Dirs, make_dirstate_tuple},
parsers::dirstate_parents_to_pytuple, parsers::dirstate_parents_to_pytuple,
}; };
use hg::{ use hg::{
dirstate::parsers::Timestamp, dirstate::parsers::Timestamp,
dirstate_tree::dispatch::DirstateMapMethods, dirstate_tree::dispatch::DirstateMapMethods,
errors::HgError, errors::HgError,
revlog::Node, revlog::Node,
@shared data inner: Box<dyn DirstateMapMethods + Send>; @shared data inner: Box<dyn DirstateMapMethods + Send>;
/// Returns a `(dirstate_map, parents)` tuple /// Returns a `(dirstate_map, parents)` tuple
@staticmethod @staticmethod
def new(use_dirstate_tree: bool, on_disk: PyBytes) -> PyResult<PyObject> { def new(use_dirstate_tree: bool, on_disk: PyBytes) -> PyResult<PyObject> {
let dirstate_error = |_: DirstateError| { let dirstate_error = |_: DirstateError| {
PyErr::new::<exc::OSError, _>(py, "Dirstate error".to_string()) PyErr::new::<exc::OSError, _>(py, "Dirstate error".to_string())
}; };
let bytes = on_disk.data(py);
let (inner, parents) = if use_dirstate_tree { let (inner, parents) = if use_dirstate_tree {
let mut map = hg::dirstate_tree::dirstate_map::DirstateMap::new(); let (map, parents) =
let parents = map.read(bytes).map_err(dirstate_error)?; OwningDirstateMap::new(py, on_disk)
.map_err(dirstate_error)?;
(Box::new(map) as _, parents) (Box::new(map) as _, parents)
} else { } else {
let bytes = on_disk.data(py);
let mut map = RustDirstateMap::default(); let mut map = RustDirstateMap::default();
let parents = map.read(bytes).map_err(dirstate_error)?; let parents = map.read(bytes).map_err(dirstate_error)?;
(Box::new(map) as _, parents) (Box::new(map) as _, parents)
}; };
let map = Self::create_instance(py, inner)?; let map = Self::create_instance(py, inner)?;
let parents = parents.map(|p| dirstate_parents_to_pytuple(py, &p)); let parents = parents.map(|p| dirstate_parents_to_pytuple(py, &p));
Ok((map, parents).to_py_object(py).into_object()) Ok((map, parents).to_py_object(py).into_object())
} }

rust/hg-cpython/src/dirstate/dispatch.rs

  • This file was added.
use crate::dirstate::owning::OwningDirstateMap;
use hg::dirstate::parsers::Timestamp;
use hg::dirstate_tree::dispatch::DirstateMapMethods;
use hg::matchers::Matcher;
use hg::utils::hg_path::{HgPath, HgPathBuf};
use hg::CopyMapIter;
use hg::DirstateEntry;
use hg::DirstateError;
use hg::DirstateMapError;
use hg::DirstateParents;
use hg::DirstateStatus;
use hg::EntryState;
use hg::PatternFileWarning;
use hg::StateMapIter;
use hg::StatusError;
use hg::StatusOptions;
use std::path::PathBuf;
impl DirstateMapMethods for OwningDirstateMap {
fn clear(&mut self) {
self.get_mut().clear()
}
fn add_file(
&mut self,
filename: &HgPath,
old_state: EntryState,
entry: DirstateEntry,
) -> Result<(), DirstateMapError> {
self.get_mut().add_file(filename, old_state, entry)
}
fn remove_file(
&mut self,
filename: &HgPath,
old_state: EntryState,
size: i32,
) -> Result<(), DirstateMapError> {
self.get_mut().remove_file(filename, old_state, size)
}
fn drop_file(
&mut self,
filename: &HgPath,
old_state: EntryState,
) -> Result<bool, DirstateMapError> {
self.get_mut().drop_file(filename, old_state)
}
fn clear_ambiguous_times(&mut self, filenames: Vec<HgPathBuf>, now: i32) {
self.get_mut().clear_ambiguous_times(filenames, now)
}
fn non_normal_entries_contains(&mut self, key: &HgPath) -> bool {
self.get_mut().non_normal_entries_contains(key)
}
fn non_normal_entries_remove(&mut self, key: &HgPath) {
self.get_mut().non_normal_entries_remove(key)
}
fn non_normal_or_other_parent_paths(
&mut self,
) -> Box<dyn Iterator<Item = &HgPathBuf> + '_> {
self.get_mut().non_normal_or_other_parent_paths()
}
fn set_non_normal_other_parent_entries(&mut self, force: bool) {
self.get_mut().set_non_normal_other_parent_entries(force)
}
fn iter_non_normal_paths(
&mut self,
) -> Box<dyn Iterator<Item = &HgPathBuf> + Send + '_> {
self.get_mut().iter_non_normal_paths()
}
fn iter_non_normal_paths_panic(
&self,
) -> Box<dyn Iterator<Item = &HgPathBuf> + Send + '_> {
self.get().iter_non_normal_paths_panic()
}
fn iter_other_parent_paths(
&mut self,
) -> Box<dyn Iterator<Item = &HgPathBuf> + Send + '_> {
self.get_mut().iter_other_parent_paths()
}
fn has_tracked_dir(
&mut self,
directory: &HgPath,
) -> Result<bool, DirstateMapError> {
self.get_mut().has_tracked_dir(directory)
}
fn has_dir(
&mut self,
directory: &HgPath,
) -> Result<bool, DirstateMapError> {
self.get_mut().has_dir(directory)
}
fn pack(
&mut self,
parents: DirstateParents,
now: Timestamp,
) -> Result<Vec<u8>, DirstateError> {
self.get_mut().pack(parents, now)
}
fn set_all_dirs(&mut self) -> Result<(), DirstateMapError> {
self.get_mut().set_all_dirs()
}
fn set_dirs(&mut self) -> Result<(), DirstateMapError> {
self.get_mut().set_dirs()
}
fn status<'a>(
&'a mut self,
matcher: &'a (dyn Matcher + Sync),
root_dir: PathBuf,
ignore_files: Vec<PathBuf>,
options: StatusOptions,
) -> Result<(DirstateStatus<'a>, Vec<PatternFileWarning>), StatusError>
{
self.get_mut()
.status(matcher, root_dir, ignore_files, options)
}
fn copy_map_len(&self) -> usize {
self.get().copy_map_len()
}
fn copy_map_iter(&self) -> CopyMapIter<'_> {
self.get().copy_map_iter()
}
fn copy_map_contains_key(&self, key: &HgPath) -> bool {
self.get().copy_map_contains_key(key)
}
fn copy_map_get(&self, key: &HgPath) -> Option<&HgPathBuf> {
self.get().copy_map_get(key)
}
fn copy_map_remove(&mut self, key: &HgPath) -> Option<HgPathBuf> {
self.get_mut().copy_map_remove(key)
}
fn copy_map_insert(
&mut self,
key: HgPathBuf,
value: HgPathBuf,
) -> Option<HgPathBuf> {
self.get_mut().copy_map_insert(key, value)
}
fn len(&self) -> usize {
self.get().len()
}
fn contains_key(&self, key: &HgPath) -> bool {
self.get().contains_key(key)
}
fn get(&self, key: &HgPath) -> Option<&DirstateEntry> {
self.get().get(key)
}
fn iter(&self) -> StateMapIter<'_> {
self.get().iter()
}
}

rust/hg-cpython/src/dirstate/owning.rs

  • This file was added.
use cpython::PyBytes;
use cpython::Python;
use hg::dirstate_tree::dirstate_map::DirstateMap;
use hg::DirstateError;
use hg::DirstateParents;
/// Keep a `DirstateMap<'on_disk>` next to the `on_disk` buffer that it
/// borrows. This is similar to the owning-ref crate.
///
/// This is similar to [`OwningRef`] which is more limited because it
/// represents exactly one `&T` reference next to the value it borrows, as
/// opposed to a struct that may contain an arbitrary number of references in
/// arbitrarily-nested data structures.
///
/// [`OwningRef`]: https://docs.rs/owning_ref/0.4.1/owning_ref/struct.OwningRef.html
pub(super) struct OwningDirstateMap {
/// Owned handle to a bytes buffer with a stable address.
///
/// See <https://docs.rs/owning_ref/0.4.1/owning_ref/trait.StableAddress.html>.
on_disk: PyBytes,
/// Pointer for `Box<DirstateMap<'on_disk>>`, typed-erased because the
/// language cannot represent a lifetime referencing a sibling field.
/// This is not quite a self-referencial struct (moving this struct is not
/// a problem as it doesn’t change the address of the bytes buffer owned
/// by `PyBytes`) but touches similar borrow-checker limitations.
ptr: *mut (),
}
impl OwningDirstateMap {
pub fn new(
py: Python,
on_disk: PyBytes,
) -> Result<(Self, Option<DirstateParents>), DirstateError> {
let bytes: &'_ [u8] = on_disk.data(py);
let (map, parents) = DirstateMap::new(bytes)?;
// Like in `bytes` above, this `'_` lifetime parameter borrows from
// the bytes buffer owned by `on_disk`.
let ptr: *mut DirstateMap<'_> = Box::into_raw(Box::new(map));
// Erase the pointed type entirely in order to erase the lifetime.
let ptr: *mut () = ptr.cast();
Ok((Self { on_disk, ptr }, parents))
}
pub fn get_mut<'a>(&'a mut self) -> &'a mut DirstateMap<'a> {
// SAFETY: We cast the type-erased pointer back to the same type it had
// in `new`, except with a different lifetime parameter. This time we
// connect the lifetime to that of `self`. This cast is valid because
// `self` owns the same `PyBytes` whose buffer `DirstateMap`
// references. That buffer has a stable memory address because the byte
// string value of a `PyBytes` is immutable.
let ptr: *mut DirstateMap<'a> = self.ptr.cast();
// SAFETY: we dereference that pointer, connecting the lifetime of the
// new `&mut` to that of `self`. This is valid because the
// raw pointer is to a boxed value, and `self` owns that box.
unsafe { &mut *ptr }
}
pub fn get<'a>(&'a self) -> &'a DirstateMap<'a> {
// SAFETY: same reasoning as in `get_mut` above.
let ptr: *mut DirstateMap<'a> = self.ptr.cast();
unsafe { &*ptr }
}
}
impl Drop for OwningDirstateMap {
fn drop(&mut self) {
// Silence a "field is never read" warning, and demonstrate that this
// value is still alive.
let _ = &self.on_disk;
// SAFETY: this cast is the same as in `get_mut`, and is valid for the
// same reason. `self.on_disk` still exists at this point, drop glue
// will drop it implicitly after this `drop` method returns.
let ptr: *mut DirstateMap<'_> = self.ptr.cast();
// SAFETY: `Box::from_raw` takes ownership of the box away from `self`.
// This is fine because drop glue does nothig for `*mut ()` and we’re
// in `drop`, so `get` and `get_mut` cannot be called again.
unsafe { drop(Box::from_raw(ptr)) }
}
}
fn _static_assert_is_send<T: Send>() {}
fn _static_assert_fields_are_send() {
_static_assert_is_send::<PyBytes>();
_static_assert_is_send::<Box<DirstateMap<'_>>>();
}
// SAFETY: we don’t get this impl implicitly because `*mut (): !Send` because
// thread-safety of raw pointers is unknown in the general case. However this
// particular raw pointer represents a `Box<DirstateMap<'on_disk>>` that we
// own. Since that `Box` and `PyBytes` are both `Send` as shown in above, it
// is sound to mark this struct as `Send` too.
unsafe impl Send for OwningDirstateMap {}