( )⚙ D11098 dirstate-v2: Move fixed-size tree metadata into the docket file

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D11098

dirstate-v2: Move fixed-size tree metadata into the docket file
ClosedPublic

Authored by SimonSapin on Jul 15 2021, 5:03 PM.

Details

Reviewers
marmoute
Alphare
Group Reviewers
hg-reviewers
Commits
rHG78f7f0d490ee: dirstate-v2: Move fixed-size tree metadata into the docket file
Summary

Before this changeset, the dirstate-v2 data file contained not only nodes
and paths that may be reused when appending to an existing file,
but also some fixed-size metadata that applies to the entire tree
and was added at the end of the data file for every append.

This moves that metadata into the docket file, so that repeated "append"
operations without meaningful changes don’t actually need to grow any file.

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

SimonSapin created this revision.Jul 15 2021, 5:03 PM
baymax updated this revision to Diff 29323.Jul 16 2021, 5:39 AM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

marmoute accepted this revision.Jul 16 2021, 6:22 AM
Alphare accepted this revision.Jul 16 2021, 9:15 AM
Alphare added a subscriber: Alphare.

This change might be the right place to remind you to also put maybe 2 bytes worth of (for now unused) bitfield for forward compat in the docket?

This looks good to me otherwise.

This revision is now accepted and ready to land.Jul 16 2021, 9:15 AM
baymax updated this revision to Diff 29355.Jul 16 2021, 5:45 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

baymax updated this revision to Diff 29614.Jul 20 2021, 3:42 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

Revision Contents
Changeset List

Diff 29614

mercurial/debugcommands.py

) )
def debugdirstateignorepatternshash(ui, repo, **opts): def debugdirstateignorepatternshash(ui, repo, **opts):
"""show the hash of ignore patterns stored in dirstate if v2, """show the hash of ignore patterns stored in dirstate if v2,
or nothing for dirstate-v2 or nothing for dirstate-v2
""" """
if repo.dirstate._use_dirstate_v2: if repo.dirstate._use_dirstate_v2:
docket = repo.dirstate._map.docket docket = repo.dirstate._map.docket
hash_len = 20 # 160 bits for SHA-1 hash_len = 20 # 160 bits for SHA-1
hash_offset = docket.data_size - hash_len # hash is at the end hash_bytes = docket.tree_metadata[-hash_len:]
data_filename = docket.data_filename()
with repo.vfs(data_filename) as f:
f.seek(hash_offset)
hash_bytes = f.read(hash_len)
ui.write(binascii.hexlify(hash_bytes) + b'\n') ui.write(binascii.hexlify(hash_bytes) + b'\n')
@command( @command(
b'debugdiscovery', b'debugdiscovery',
[ [
(b'', b'old', None, _(b'use old-style discovery')), (b'', b'old', None, _(b'use old-style discovery')),
( (

mercurial/dirstatemap.py

if self._use_dirstate_v2: if self._use_dirstate_v2:
if self.docket.uuid: if self.docket.uuid:
# TODO: use mmap when possible # TODO: use mmap when possible
data = self._opener.read(self.docket.data_filename()) data = self._opener.read(self.docket.data_filename())
else: else:
data = b'' data = b''
self._rustmap = rustmod.DirstateMap.new_v2( self._rustmap = rustmod.DirstateMap.new_v2(
data, self.docket.data_size data, self.docket.data_size, self.docket.tree_metadata
) )
parents = self.docket.parents parents = self.docket.parents
else: else:
self._rustmap, parents = rustmod.DirstateMap.new_v1( self._rustmap, parents = rustmod.DirstateMap.new_v1(
self._use_dirstate_tree, self._readdirstatefile() self._use_dirstate_tree, self._readdirstatefile()
) )
if parents and not self._dirtyparents: if parents and not self._dirtyparents:
packed = self._rustmap.write_v1(p1, p2, now) packed = self._rustmap.write_v1(p1, p2, now)
st.write(packed) st.write(packed)
st.close() st.close()
self._dirtyparents = False self._dirtyparents = False
return return
# We can only append to an existing data file if there is one # We can only append to an existing data file if there is one
can_append = self.docket.uuid is not None can_append = self.docket.uuid is not None
packed, append = self._rustmap.write_v2(now, can_append) packed, meta, append = self._rustmap.write_v2(now, can_append)
if append: if append:
docket = self.docket docket = self.docket
data_filename = docket.data_filename() data_filename = docket.data_filename()
if tr: if tr:
tr.add(data_filename, docket.data_size) tr.add(data_filename, docket.data_size)
with self._opener(data_filename, b'r+b') as fp: with self._opener(data_filename, b'r+b') as fp:
fp.seek(docket.data_size) fp.seek(docket.data_size)
assert fp.tell() == docket.data_size assert fp.tell() == docket.data_size
written = fp.write(packed) written = fp.write(packed)
if written is not None: # py2 may return None if written is not None: # py2 may return None
assert written == len(packed), (written, len(packed)) assert written == len(packed), (written, len(packed))
docket.data_size += len(packed) docket.data_size += len(packed)
docket.parents = self.parents() docket.parents = self.parents()
docket.tree_metadata = meta
st.write(docket.serialize()) st.write(docket.serialize())
st.close() st.close()
else: else:
old_docket = self.docket old_docket = self.docket
new_docket = docketmod.DirstateDocket.with_new_uuid( new_docket = docketmod.DirstateDocket.with_new_uuid(
self.parents(), len(packed) self.parents(), len(packed), meta
) )
data_filename = new_docket.data_filename() data_filename = new_docket.data_filename()
if tr: if tr:
tr.add(data_filename, 0) tr.add(data_filename, 0)
self._opener.write(data_filename, packed) self._opener.write(data_filename, packed)
# Write the new docket after the new data file has been # Write the new docket after the new data file has been
# written. Because `st` was opened with `atomictemp=True`, # written. Because `st` was opened with `atomictemp=True`,
# the actual `.hg/dirstate` file is only affected on close. # the actual `.hg/dirstate` file is only affected on close.

mercurial/dirstateutils/docket.py

# dirstatedocket.py - docket file for dirstate-v2 # dirstatedocket.py - docket file for dirstate-v2
# #
# Copyright Mercurial Contributors # Copyright Mercurial Contributors
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import struct import struct
from ..revlogutils import docket as docket_mod from ..revlogutils import docket as docket_mod
V2_FORMAT_MARKER = b"dirstate-v2\n" V2_FORMAT_MARKER = b"dirstate-v2\n"
# Must match the constant of the same name in
# `rust/hg-core/src/dirstate_tree/on_disk.rs`
TREE_METADATA_SIZE = 40
# * 12 bytes: format marker # * 12 bytes: format marker
# * 32 bytes: node ID of the working directory's first parent # * 32 bytes: node ID of the working directory's first parent
# * 32 bytes: node ID of the working directory's second parent # * 32 bytes: node ID of the working directory's second parent
# * 4 bytes: big-endian used size of the data file # * 4 bytes: big-endian used size of the data file
# * {TREE_METADATA_SIZE} bytes: tree metadata, parsed separately
# * 1 byte: length of the data file's UUID # * 1 byte: length of the data file's UUID
# * variable: data file's UUID # * variable: data file's UUID
# #
# Node IDs are null-padded if shorter than 32 bytes. # Node IDs are null-padded if shorter than 32 bytes.
# A data file shorter than the specified used size is corrupted (truncated) # A data file shorter than the specified used size is corrupted (truncated)
HEADER = struct.Struct(">{}s32s32sLB".format(len(V2_FORMAT_MARKER))) HEADER = struct.Struct(
">{}s32s32sL{}sB".format(len(V2_FORMAT_MARKER), TREE_METADATA_SIZE)
)
class DirstateDocket(object): class DirstateDocket(object):
data_filename_pattern = b'dirstate.%s.d' data_filename_pattern = b'dirstate.%s.d'
def __init__(self, parents, data_size, uuid): def __init__(self, parents, data_size, tree_metadata, uuid):
self.parents = parents self.parents = parents
self.data_size = data_size self.data_size = data_size
self.tree_metadata = tree_metadata
self.uuid = uuid self.uuid = uuid
@classmethod @classmethod
def with_new_uuid(cls, parents, data): def with_new_uuid(cls, parents, data_size, tree_metadata):
return cls(parents, data, docket_mod.make_uid()) return cls(parents, data_size, tree_metadata, docket_mod.make_uid())
@classmethod @classmethod
def parse(cls, data, nodeconstants): def parse(cls, data, nodeconstants):
if not data: if not data:
parents = (nodeconstants.nullid, nodeconstants.nullid) parents = (nodeconstants.nullid, nodeconstants.nullid)
return cls(parents, 0, None) return cls(parents, 0, b'', None)
marker, p1, p2, data_size, uuid_size = HEADER.unpack_from(data) marker, p1, p2, data_size, meta, uuid_size = HEADER.unpack_from(data)
if marker != V2_FORMAT_MARKER: if marker != V2_FORMAT_MARKER:
raise ValueError("expected dirstate-v2 marker") raise ValueError("expected dirstate-v2 marker")
uuid = data[HEADER.size : HEADER.size + uuid_size] uuid = data[HEADER.size : HEADER.size + uuid_size]
p1 = p1[: nodeconstants.nodelen] p1 = p1[: nodeconstants.nodelen]
p2 = p2[: nodeconstants.nodelen] p2 = p2[: nodeconstants.nodelen]
return cls((p1, p2), data_size, uuid) return cls((p1, p2), data_size, meta, uuid)
def serialize(self): def serialize(self):
p1, p2 = self.parents p1, p2 = self.parents
header = HEADER.pack( header = HEADER.pack(
V2_FORMAT_MARKER, p1, p2, self.data_size, len(self.uuid) V2_FORMAT_MARKER,
p1,
p2,
self.data_size,
self.tree_metadata,
len(self.uuid),
) )
return header + self.uuid return header + self.uuid
def data_filename(self): def data_filename(self):
return self.data_filename_pattern % self.uuid return self.data_filename_pattern % self.uuid

rust/hg-core/src/dirstate_tree/dirstate_map.rs

unreachable_bytes: 0, unreachable_bytes: 0,
} }
} }
#[timed] #[timed]
pub fn new_v2( pub fn new_v2(
on_disk: &'on_disk [u8], on_disk: &'on_disk [u8],
data_size: usize, data_size: usize,
metadata: &[u8],
) -> Result<Self, DirstateError> { ) -> Result<Self, DirstateError> {
if let Some(data) = on_disk.get(..data_size) { if let Some(data) = on_disk.get(..data_size) {
Ok(on_disk::read(data)?) Ok(on_disk::read(data, metadata)?)
} else { } else {
Err(DirstateV2ParseError.into()) Err(DirstateV2ParseError.into())
} }
} }
#[timed] #[timed]
pub fn new_v1( pub fn new_v1(
on_disk: &'on_disk [u8], on_disk: &'on_disk [u8],
node.copy_source(self.on_disk)?, node.copy_source(self.on_disk)?,
&mut packed, &mut packed,
); );
} }
} }
Ok(packed) Ok(packed)
} }
/// Returns new data together with whether that data should be appended to /// Returns new data and metadata together with whether that data should be
/// the existing data file whose content is at `self.on_disk` (true), /// appended to the existing data file whose content is at
/// instead of written to a new data file (false). /// `self.on_disk` (true), instead of written to a new data file
/// (false).
#[timed] #[timed]
fn pack_v2( fn pack_v2(
&mut self, &mut self,
now: Timestamp, now: Timestamp,
can_append: bool, can_append: bool,
) -> Result<(Vec<u8>, bool), DirstateError> { ) -> Result<(Vec<u8>, Vec<u8>, bool), DirstateError> {
// TODO:!how do we want to handle this in 2038? // TODO:!how do we want to handle this in 2038?
let now: i32 = now.0.try_into().expect("time overflow"); let now: i32 = now.0.try_into().expect("time overflow");
let mut paths = Vec::new(); let mut paths = Vec::new();
for node in self.iter_nodes() { for node in self.iter_nodes() {
let node = node?; let node = node?;
if let Some(entry) = node.entry()? { if let Some(entry) = node.entry()? {
if entry.mtime_is_ambiguous(now) { if entry.mtime_is_ambiguous(now) {
paths.push( paths.push(

rust/hg-core/src/dirstate_tree/dispatch.rs

now: Timestamp, now: Timestamp,
) -> Result<Vec<u8>, DirstateError>; ) -> Result<Vec<u8>, DirstateError>;
/// Clear mtimes that are ambigous with `now` (similar to /// Clear mtimes that are ambigous with `now` (similar to
/// `clear_ambiguous_times` but for all files in the dirstate map), and /// `clear_ambiguous_times` but for all files in the dirstate map), and
/// serialize bytes to write a dirstate data file to disk in dirstate-v2 /// serialize bytes to write a dirstate data file to disk in dirstate-v2
/// format. /// format.
/// ///
/// Returns new data together with whether that data should be appended to /// Returns new data and metadata together with whether that data should be
/// the existing data file whose content is at `self.on_disk` (true), /// appended to the existing data file whose content is at
/// instead of written to a new data file (false). /// `self.on_disk` (true), instead of written to a new data file
/// (false).
/// ///
/// Note: this is only supported by the tree dirstate map. /// Note: this is only supported by the tree dirstate map.
fn pack_v2( fn pack_v2(
&mut self, &mut self,
now: Timestamp, now: Timestamp,
can_append: bool, can_append: bool,
) -> Result<(Vec<u8>, bool), DirstateError>; ) -> Result<(Vec<u8>, Vec<u8>, bool), DirstateError>;
/// Run the status algorithm. /// Run the status algorithm.
/// ///
/// This is not sematically a method of the dirstate map, but a different /// This is not sematically a method of the dirstate map, but a different
/// algorithm is used for the flat v.s. tree dirstate map so having it in /// algorithm is used for the flat v.s. tree dirstate map so having it in
/// this trait enables the same dynamic dispatch as with other methods. /// this trait enables the same dynamic dispatch as with other methods.
fn status<'a>( fn status<'a>(
&'a mut self, &'a mut self,
) -> Result<Vec<u8>, DirstateError> { ) -> Result<Vec<u8>, DirstateError> {
self.pack(parents, now) self.pack(parents, now)
} }
fn pack_v2( fn pack_v2(
&mut self, &mut self,
_now: Timestamp, _now: Timestamp,
_can_append: bool, _can_append: bool,
) -> Result<(Vec<u8>, bool), DirstateError> { ) -> Result<(Vec<u8>, Vec<u8>, bool), DirstateError> {
panic!( panic!(
"should have used dirstate_tree::DirstateMap to use the v2 format" "should have used dirstate_tree::DirstateMap to use the v2 format"
) )
} }
fn status<'a>( fn status<'a>(
&'a mut self, &'a mut self,
matcher: &'a (dyn Matcher + Sync), matcher: &'a (dyn Matcher + Sync),

rust/hg-core/src/dirstate_tree/on_disk.rs

const STORED_NODE_ID_BYTES: usize = 32; const STORED_NODE_ID_BYTES: usize = 32;
/// … even though only 160 bits are used for now, with SHA-1 /// … even though only 160 bits are used for now, with SHA-1
const USED_NODE_ID_BYTES: usize = 20; const USED_NODE_ID_BYTES: usize = 20;
pub(super) const IGNORE_PATTERNS_HASH_LEN: usize = 20; pub(super) const IGNORE_PATTERNS_HASH_LEN: usize = 20;
pub(super) type IgnorePatternsHash = [u8; IGNORE_PATTERNS_HASH_LEN]; pub(super) type IgnorePatternsHash = [u8; IGNORE_PATTERNS_HASH_LEN];
/// Must match the constant of the same name in
/// `mercurial/dirstateutils/docket.py`
const TREE_METADATA_SIZE: usize = 40;
/// Make sure that size-affecting changes are made knowingly
#[allow(unused)]
fn static_assert_size_of() {
let _ = std::mem::transmute::<DocketHeader, [u8; 121]>;
let _ = std::mem::transmute::<TreeMetadata, [u8; TREE_METADATA_SIZE]>;
let _ = std::mem::transmute::<Node, [u8; 43]>;
}
// Must match `HEADER` in `mercurial/dirstateutils/docket.py` // Must match `HEADER` in `mercurial/dirstateutils/docket.py`
#[derive(BytesCast)] #[derive(BytesCast)]
#[repr(C)] #[repr(C)]
struct DocketHeader { struct DocketHeader {
marker: [u8; V2_FORMAT_MARKER.len()], marker: [u8; V2_FORMAT_MARKER.len()],
parent_1: [u8; STORED_NODE_ID_BYTES], parent_1: [u8; STORED_NODE_ID_BYTES],
parent_2: [u8; STORED_NODE_ID_BYTES], parent_2: [u8; STORED_NODE_ID_BYTES],
/// Counted in bytes /// Counted in bytes
data_size: Size, data_size: Size,
metadata: TreeMetadata,
uuid_size: u8, uuid_size: u8,
} }
pub struct Docket<'on_disk> { pub struct Docket<'on_disk> {
header: &'on_disk DocketHeader, header: &'on_disk DocketHeader,
uuid: &'on_disk [u8], uuid: &'on_disk [u8],
} }
#[derive(BytesCast)] #[derive(BytesCast)]
#[repr(C)] #[repr(C)]
struct Root { struct TreeMetadata {
root_nodes: ChildNodes, root_nodes: ChildNodes,
nodes_with_entry_count: Size, nodes_with_entry_count: Size,
nodes_with_copy_source_count: Size, nodes_with_copy_source_count: Size,
/// How many bytes of this data file are not used anymore /// How many bytes of this data file are not used anymore
unreachable_bytes: Size, unreachable_bytes: Size,
/// If non-zero, a hash of ignore files that were used for some previous /// If non-zero, a hash of ignore files that were used for some previous
/// - That timestamp was already strictly in the past when observed, /// - That timestamp was already strictly in the past when observed,
/// meaning that later changes cannot happen in the same clock tick /// meaning that later changes cannot happen in the same clock tick
/// and must cause a different modification time (unless the system /// and must cause a different modification time (unless the system
/// clock jumps back and we get unlucky, which is not impossible but /// clock jumps back and we get unlucky, which is not impossible but
/// but deemed unlikely enough). /// but deemed unlikely enough).
/// - All direct children of this directory (as returned by /// - All direct children of this directory (as returned by
/// `std::fs::read_dir`) either have a corresponding dirstate node, or /// `std::fs::read_dir`) either have a corresponding dirstate node, or
/// are ignored by ignore patterns whose hash is in /// are ignored by ignore patterns whose hash is in
/// `Root::ignore_patterns_hash`. /// `TreeMetadata::ignore_patterns_hash`.
/// ///
/// This means that if `std::fs::symlink_metadata` later reports the /// This means that if `std::fs::symlink_metadata` later reports the
/// same modification time and ignored patterns haven’t changed, a run /// same modification time and ignored patterns haven’t changed, a run
/// of status that is not listing ignored files can skip calling /// of status that is not listing ignored files can skip calling
/// `std::fs::read_dir` again for this directory, iterate child /// `std::fs::read_dir` again for this directory, iterate child
/// dirstate nodes instead. /// dirstate nodes instead.
state: u8, state: u8,
data: Entry, data: Entry,
struct PathSlice { struct PathSlice {
start: Offset, start: Offset,
len: PathSize, len: PathSize,
} }
/// Either nothing if `start == 0`, or a `HgPath` of `len` bytes /// Either nothing if `start == 0`, or a `HgPath` of `len` bytes
type OptPathSlice = PathSlice; type OptPathSlice = PathSlice;
/// Make sure that size-affecting changes are made knowingly
fn _static_assert_size_of() {
let _ = std::mem::transmute::<DocketHeader, [u8; 81]>;
let _ = std::mem::transmute::<Root, [u8; 40]>;
let _ = std::mem::transmute::<Node, [u8; 43]>;
}
/// Unexpected file format found in `.hg/dirstate` with the "v2" format. /// Unexpected file format found in `.hg/dirstate` with the "v2" format.
/// ///
/// This should only happen if Mercurial is buggy or a repository is corrupted. /// This should only happen if Mercurial is buggy or a repository is corrupted.
#[derive(Debug)] #[derive(Debug)]
pub struct DirstateV2ParseError; pub struct DirstateV2ParseError;
impl From<DirstateV2ParseError> for HgError { impl From<DirstateV2ParseError> for HgError {
fn from(_: DirstateV2ParseError) -> Self { fn from(_: DirstateV2ParseError) -> Self {
.unwrap() .unwrap()
.clone(); .clone();
let p2 = Node::try_from(&self.header.parent_2[..USED_NODE_ID_BYTES]) let p2 = Node::try_from(&self.header.parent_2[..USED_NODE_ID_BYTES])
.unwrap() .unwrap()
.clone(); .clone();
DirstateParents { p1, p2 } DirstateParents { p1, p2 }
} }
pub fn tree_metadata(&self) -> &[u8] {
self.header.metadata.as_bytes()
}
pub fn data_size(&self) -> usize { pub fn data_size(&self) -> usize {
// This `unwrap` could only panic on a 16-bit CPU // This `unwrap` could only panic on a 16-bit CPU
self.header.data_size.get().try_into().unwrap() self.header.data_size.get().try_into().unwrap()
} }
pub fn data_filename(&self) -> String { pub fn data_filename(&self) -> String {
String::from_utf8(format_bytes!(b"dirstate.{}.d", self.uuid)).unwrap() String::from_utf8(format_bytes!(b"dirstate.{}.d", self.uuid)).unwrap()
} }
} }
pub fn read_docket( pub fn read_docket(
on_disk: &[u8], on_disk: &[u8],
) -> Result<Docket<'_>, DirstateV2ParseError> { ) -> Result<Docket<'_>, DirstateV2ParseError> {
let (header, uuid) = let (header, uuid) =
DocketHeader::from_bytes(on_disk).map_err(|_| DirstateV2ParseError)?; DocketHeader::from_bytes(on_disk).map_err(|_| DirstateV2ParseError)?;
let uuid_size = header.uuid_size as usize; let uuid_size = header.uuid_size as usize;
if header.marker == *V2_FORMAT_MARKER && uuid.len() == uuid_size { if header.marker == *V2_FORMAT_MARKER && uuid.len() == uuid_size {
Ok(Docket { header, uuid }) Ok(Docket { header, uuid })
} else { } else {
Err(DirstateV2ParseError) Err(DirstateV2ParseError)
} }
} }
fn read_root<'on_disk>(
on_disk: &'on_disk [u8],
) -> Result<&'on_disk Root, DirstateV2ParseError> {
// Find the `Root` at the end of the given slice
let root_offset = on_disk
.len()
.checked_sub(std::mem::size_of::<Root>())
// A non-empty slice too short is an error
.ok_or(DirstateV2ParseError)?;
let (root, _) = Root::from_bytes(&on_disk[root_offset..])
.map_err(|_| DirstateV2ParseError)?;
Ok(root)
}
pub(super) fn read<'on_disk>( pub(super) fn read<'on_disk>(
on_disk: &'on_disk [u8], on_disk: &'on_disk [u8],
metadata: &[u8],
) -> Result<DirstateMap<'on_disk>, DirstateV2ParseError> { ) -> Result<DirstateMap<'on_disk>, DirstateV2ParseError> {
if on_disk.is_empty() { if on_disk.is_empty() {
return Ok(DirstateMap::empty(on_disk)); return Ok(DirstateMap::empty(on_disk));
} }
let root = read_root(on_disk)?; let (meta, _) = TreeMetadata::from_bytes(metadata)
let mut unreachable_bytes = root.unreachable_bytes.get(); .map_err(|_| DirstateV2ParseError)?;
// Each append writes a new `Root`, so it’s never reused
unreachable_bytes += std::mem::size_of::<Root>() as u32;
let dirstate_map = DirstateMap { let dirstate_map = DirstateMap {
on_disk, on_disk,
root: dirstate_map::ChildNodes::OnDisk(read_nodes( root: dirstate_map::ChildNodes::OnDisk(read_nodes(
on_disk, on_disk,
root.root_nodes, meta.root_nodes,
)?), )?),
nodes_with_entry_count: root.nodes_with_entry_count.get(), nodes_with_entry_count: meta.nodes_with_entry_count.get(),
nodes_with_copy_source_count: root.nodes_with_copy_source_count.get(), nodes_with_copy_source_count: meta.nodes_with_copy_source_count.get(),
ignore_patterns_hash: root.ignore_patterns_hash, ignore_patterns_hash: meta.ignore_patterns_hash,
unreachable_bytes, unreachable_bytes: meta.unreachable_bytes.get(),
}; };
Ok(dirstate_map) Ok(dirstate_map)
} }
impl Node { impl Node {
pub(super) fn full_path<'on_disk>( pub(super) fn full_path<'on_disk>(
&self, &self,
on_disk: &'on_disk [u8], on_disk: &'on_disk [u8],
.get(start..) .get(start..)
.and_then(|bytes| T::slice_from_bytes(bytes, len).ok()) .and_then(|bytes| T::slice_from_bytes(bytes, len).ok())
.map(|(slice, _rest)| slice) .map(|(slice, _rest)| slice)
.ok_or_else(|| DirstateV2ParseError) .ok_or_else(|| DirstateV2ParseError)
} }
pub(crate) fn for_each_tracked_path<'on_disk>( pub(crate) fn for_each_tracked_path<'on_disk>(
on_disk: &'on_disk [u8], on_disk: &'on_disk [u8],
metadata: &[u8],
mut f: impl FnMut(&'on_disk HgPath), mut f: impl FnMut(&'on_disk HgPath),
) -> Result<(), DirstateV2ParseError> { ) -> Result<(), DirstateV2ParseError> {
let root = read_root(on_disk)?; let (meta, _) = TreeMetadata::from_bytes(metadata)
.map_err(|_| DirstateV2ParseError)?;
fn recur<'on_disk>( fn recur<'on_disk>(
on_disk: &'on_disk [u8], on_disk: &'on_disk [u8],
nodes: ChildNodes, nodes: ChildNodes,
f: &mut impl FnMut(&'on_disk HgPath), f: &mut impl FnMut(&'on_disk HgPath),
) -> Result<(), DirstateV2ParseError> { ) -> Result<(), DirstateV2ParseError> {
for node in read_nodes(on_disk, nodes)? { for node in read_nodes(on_disk, nodes)? {
if let Some(state) = node.state()? { if let Some(state) = node.state()? {
if state.is_tracked() { if state.is_tracked() {
f(node.full_path(on_disk)?) f(node.full_path(on_disk)?)
} }
} }
recur(on_disk, node.children, f)? recur(on_disk, node.children, f)?
} }
Ok(()) Ok(())
} }
recur(on_disk, root.root_nodes, &mut f) recur(on_disk, meta.root_nodes, &mut f)
} }
/// Returns new data together with whether that data should be appended to the /// Returns new data and metadata, together with whether that data should be
/// existing data file whose content is at `dirstate_map.on_disk` (true), /// appended to the existing data file whose content is at
/// instead of written to a new data file (false). /// `dirstate_map.on_disk` (true), instead of written to a new data file
/// (false).
pub(super) fn write( pub(super) fn write(
dirstate_map: &mut DirstateMap, dirstate_map: &mut DirstateMap,
can_append: bool, can_append: bool,
) -> Result<(Vec<u8>, bool), DirstateError> { ) -> Result<(Vec<u8>, Vec<u8>, bool), DirstateError> {
let append = can_append && dirstate_map.write_should_append(); let append = can_append && dirstate_map.write_should_append();
// This ignores the space for paths, and for nodes without an entry. // This ignores the space for paths, and for nodes without an entry.
// TODO: better estimate? Skip the `Vec` and write to a file directly? // TODO: better estimate? Skip the `Vec` and write to a file directly?
let size_guess = std::mem::size_of::<Root>() let size_guess = std::mem::size_of::<Node>()
+ std::mem::size_of::<Node>()
* dirstate_map.nodes_with_entry_count as usize; * dirstate_map.nodes_with_entry_count as usize;
let mut writer = Writer { let mut writer = Writer {
dirstate_map, dirstate_map,
append, append,
out: Vec::with_capacity(size_guess), out: Vec::with_capacity(size_guess),
}; };
let root_nodes = writer.write_nodes(dirstate_map.root.as_ref())?; let root_nodes = writer.write_nodes(dirstate_map.root.as_ref())?;
let root = Root { let meta = TreeMetadata {
root_nodes, root_nodes,
nodes_with_entry_count: dirstate_map.nodes_with_entry_count.into(), nodes_with_entry_count: dirstate_map.nodes_with_entry_count.into(),
nodes_with_copy_source_count: dirstate_map nodes_with_copy_source_count: dirstate_map
.nodes_with_copy_source_count .nodes_with_copy_source_count
.into(), .into(),
unreachable_bytes: dirstate_map.unreachable_bytes.into(), unreachable_bytes: dirstate_map.unreachable_bytes.into(),
ignore_patterns_hash: dirstate_map.ignore_patterns_hash, ignore_patterns_hash: dirstate_map.ignore_patterns_hash,
}; };
writer.out.extend(root.as_bytes()); Ok((writer.out, meta.as_bytes().to_vec(), append))
Ok((writer.out, append))
} }
struct Writer<'dmap, 'on_disk> { struct Writer<'dmap, 'on_disk> {
dirstate_map: &'dmap DirstateMap<'on_disk>, dirstate_map: &'dmap DirstateMap<'on_disk>,
append: bool, append: bool,
out: Vec<u8>, out: Vec<u8>,
} }

rust/hg-core/src/operations/list_tracked_files.rs

use crate::DirstateError; use crate::DirstateError;
use rayon::prelude::*; use rayon::prelude::*;
/// List files under Mercurial control in the working directory /// List files under Mercurial control in the working directory
/// by reading the dirstate /// by reading the dirstate
pub struct Dirstate { pub struct Dirstate {
/// The `dirstate` content. /// The `dirstate` content.
content: Vec<u8>, content: Vec<u8>,
dirstate_v2: bool, v2_metadata: Option<Vec<u8>>,
} }
impl Dirstate { impl Dirstate {
pub fn new(repo: &Repo) -> Result<Self, HgError> { pub fn new(repo: &Repo) -> Result<Self, HgError> {
let mut content = repo.hg_vfs().read("dirstate")?; let mut content = repo.hg_vfs().read("dirstate")?;
if repo.has_dirstate_v2() { let v2_metadata = if repo.has_dirstate_v2() {
let docket = read_docket(&content)?; let docket = read_docket(&content)?;
let meta = docket.tree_metadata().to_vec();
content = repo.hg_vfs().read(docket.data_filename())?; content = repo.hg_vfs().read(docket.data_filename())?;
} Some(meta)
} else {
None
};
Ok(Self { Ok(Self {
content, content,
dirstate_v2: repo.has_dirstate_v2(), v2_metadata,
}) })
} }
pub fn tracked_files(&self) -> Result<Vec<&HgPath>, DirstateError> { pub fn tracked_files(&self) -> Result<Vec<&HgPath>, DirstateError> {
let mut files = Vec::new(); let mut files = Vec::new();
if !self.content.is_empty() { if !self.content.is_empty() {
if self.dirstate_v2 { if let Some(meta) = &self.v2_metadata {
for_each_tracked_path(&self.content, |path| files.push(path))? for_each_tracked_path(&self.content, meta, |path| {
files.push(path)
})?
} else { } else {
let _parents = parse_dirstate_entries( let _parents = parse_dirstate_entries(
&self.content, &self.content,
|path, entry, _copy_source| { |path, entry, _copy_source| {
if entry.state.is_tracked() { if entry.state.is_tracked() {
files.push(path) files.push(path)
} }
Ok(()) Ok(())

rust/hg-cpython/src/dirstate/dirstate_map.rs

Ok((map, parents).to_py_object(py).into_object()) Ok((map, parents).to_py_object(py).into_object())
} }
/// Returns a DirstateMap /// Returns a DirstateMap
@staticmethod @staticmethod
def new_v2( def new_v2(
on_disk: PyBytes, on_disk: PyBytes,
data_size: usize, data_size: usize,
tree_metadata: PyBytes,
) -> PyResult<PyObject> { ) -> PyResult<PyObject> {
let dirstate_error = |e: DirstateError| { let dirstate_error = |e: DirstateError| {
PyErr::new::<exc::OSError, _>(py, format!("Dirstate error: {:?}", e)) PyErr::new::<exc::OSError, _>(py, format!("Dirstate error: {:?}", e))
}; };
let inner = OwningDirstateMap::new_v2(py, on_disk, data_size) let inner = OwningDirstateMap::new_v2(
.map_err(dirstate_error)?; py, on_disk, data_size, tree_metadata,
).map_err(dirstate_error)?;
let map = Self::create_instance(py, Box::new(inner))?; let map = Self::create_instance(py, Box::new(inner))?;
Ok(map.into_object()) Ok(map.into_object())
} }
def clear(&self) -> PyResult<PyObject> { def clear(&self) -> PyResult<PyObject> {
self.inner(py).borrow_mut().clear(); self.inner(py).borrow_mut().clear();
Ok(py.None()) Ok(py.None())
} }
now: PyObject, now: PyObject,
can_append: bool, can_append: bool,
) -> PyResult<PyObject> { ) -> PyResult<PyObject> {
let now = Timestamp(now.extract(py)?); let now = Timestamp(now.extract(py)?);
let mut inner = self.inner(py).borrow_mut(); let mut inner = self.inner(py).borrow_mut();
let result = inner.pack_v2(now, can_append); let result = inner.pack_v2(now, can_append);
match result { match result {
Ok((packed, append)) => { Ok((packed, tree_metadata, append)) => {
let packed = PyBytes::new(py, &packed); let packed = PyBytes::new(py, &packed);
Ok((packed, append).to_py_object(py).into_object()) let tree_metadata = PyBytes::new(py, &tree_metadata);
let tuple = (packed, tree_metadata, append);
Ok(tuple.to_py_object(py).into_object())
}, },
Err(_) => Err(PyErr::new::<exc::OSError, _>( Err(_) => Err(PyErr::new::<exc::OSError, _>(
py, py,
"Dirstate error".to_string(), "Dirstate error".to_string(),
)), )),
} }
} }

rust/hg-cpython/src/dirstate/dispatch.rs

) -> Result<Vec<u8>, DirstateError> { ) -> Result<Vec<u8>, DirstateError> {
self.get_mut().pack_v1(parents, now) self.get_mut().pack_v1(parents, now)
} }
fn pack_v2( fn pack_v2(
&mut self, &mut self,
now: Timestamp, now: Timestamp,
can_append: bool, can_append: bool,
) -> Result<(Vec<u8>, bool), DirstateError> { ) -> Result<(Vec<u8>, Vec<u8>, bool), DirstateError> {
self.get_mut().pack_v2(now, can_append) self.get_mut().pack_v2(now, can_append)
} }
fn status<'a>( fn status<'a>(
&'a mut self, &'a mut self,
matcher: &'a (dyn Matcher + Sync), matcher: &'a (dyn Matcher + Sync),
root_dir: PathBuf, root_dir: PathBuf,
ignore_files: Vec<PathBuf>, ignore_files: Vec<PathBuf>,

rust/hg-cpython/src/dirstate/owning.rs

Ok((Self { on_disk, ptr }, parents)) Ok((Self { on_disk, ptr }, parents))
} }
pub fn new_v2( pub fn new_v2(
py: Python, py: Python,
on_disk: PyBytes, on_disk: PyBytes,
data_size: usize, data_size: usize,
tree_metadata: PyBytes,
) -> Result<Self, DirstateError> { ) -> Result<Self, DirstateError> {
let bytes: &'_ [u8] = on_disk.data(py); let bytes: &'_ [u8] = on_disk.data(py);
let map = DirstateMap::new_v2(bytes, data_size)?; let map =
DirstateMap::new_v2(bytes, data_size, tree_metadata.data(py))?;
// Like in `bytes` above, this `'_` lifetime parameter borrows from // Like in `bytes` above, this `'_` lifetime parameter borrows from
// the bytes buffer owned by `on_disk`. // the bytes buffer owned by `on_disk`.
let ptr: *mut DirstateMap<'_> = Box::into_raw(Box::new(map)); let ptr: *mut DirstateMap<'_> = Box::into_raw(Box::new(map));
// Erase the pointed type entirely in order to erase the lifetime. // Erase the pointed type entirely in order to erase the lifetime.
let ptr: *mut () = ptr.cast(); let ptr: *mut () = ptr.cast();

rust/rhg/src/commands/status.rs

} else { } else {
requested requested
} }
}; };
let repo = invocation.repo?; let repo = invocation.repo?;
let dirstate_data_mmap; let dirstate_data_mmap;
let (mut dmap, parents) = if repo.has_dirstate_v2() { let (mut dmap, parents) = if repo.has_dirstate_v2() {
let docket_data =
repo.hg_vfs().read("dirstate").io_not_found_as_none()?;
let parents; let parents;
let dirstate_data; let dirstate_data;
let data_size; let data_size;
if let Some(docket_data) = let docket;
repo.hg_vfs().read("dirstate").io_not_found_as_none()? let tree_metadata;
{ if let Some(docket_data) = &docket_data {
let docket = on_disk::read_docket(&docket_data)?; docket = on_disk::read_docket(docket_data)?;
tree_metadata = docket.tree_metadata();
parents = Some(docket.parents()); parents = Some(docket.parents());
data_size = docket.data_size(); data_size = docket.data_size();
dirstate_data_mmap = repo dirstate_data_mmap = repo
.hg_vfs() .hg_vfs()
.mmap_open(docket.data_filename()) .mmap_open(docket.data_filename())
.io_not_found_as_none()?; .io_not_found_as_none()?;
dirstate_data = dirstate_data_mmap.as_deref().unwrap_or(b""); dirstate_data = dirstate_data_mmap.as_deref().unwrap_or(b"");
} else { } else {
parents = None; parents = None;
tree_metadata = b"";
data_size = 0; data_size = 0;
dirstate_data = b""; dirstate_data = b"";
} }
let dmap = DirstateMap::new_v2(dirstate_data, data_size)?; let dmap =
DirstateMap::new_v2(dirstate_data, data_size, tree_metadata)?;
(dmap, parents) (dmap, parents)
} else { } else {
dirstate_data_mmap = dirstate_data_mmap =
repo.hg_vfs().mmap_open("dirstate").io_not_found_as_none()?; repo.hg_vfs().mmap_open("dirstate").io_not_found_as_none()?;
let dirstate_data = dirstate_data_mmap.as_deref().unwrap_or(b""); let dirstate_data = dirstate_data_mmap.as_deref().unwrap_or(b"");
DirstateMap::new_v1(dirstate_data)? DirstateMap::new_v1(dirstate_data)?
}; };