| marmoute | |
| Alphare |
| hg-reviewers |
Before this changeset, the dirstate-v2 data file contained not only nodes
and paths that may be reused when appending to an existing file,
but also some fixed-size metadata that applies to the entire tree
and was added at the end of the data file for every append.
This moves that metadata into the docket file, so that repeated "append"
operations without meaningful changes don’t actually need to grow any file.
| Automatic diff as part of commit; lint not applicable. |
| Automatic diff as part of commit; unit tests not applicable. |
This change might be the right place to remind you to also put maybe 2 bytes worth of (for now unused) bitfield for forward compat in the docket?
This looks good to me otherwise.
| Path | Packages | |||
|---|---|---|---|---|
| M | mercurial/debugcommands.py (6 lines) | |||
| M | mercurial/dirstatemap.py (7 lines) | |||
| M | mercurial/dirstateutils/docket.py (29 lines) | |||
| M | rust/hg-core/src/dirstate_tree/dirstate_map.rs (12 lines) | |||
| M | rust/hg-core/src/dirstate_tree/dispatch.rs (11 lines) | |||
| M | rust/hg-core/src/dirstate_tree/on_disk.rs (85 lines) | |||
| M | rust/hg-core/src/operations/list_tracked_files.rs (18 lines) | |||
| M | rust/hg-cpython/src/dirstate/dirstate_map.rs (12 lines) | |||
| M | rust/hg-cpython/src/dirstate/dispatch.rs (2 lines) | |||
| M | rust/hg-cpython/src/dirstate/owning.rs (4 lines) | |||
| M | rust/rhg/src/commands/status.rs (15 lines) |
| Status | Author | Revision | |
|---|---|---|---|
| Closed | SimonSapin | ||
| Closed | SimonSapin | ||
| Closed | SimonSapin | ||
| Closed | SimonSapin | ||
| Closed | SimonSapin | ||
| Closed | SimonSapin | ||
| Closed | SimonSapin | ||
| Closed | SimonSapin | ||
| Closed | SimonSapin | ||
| Closed | SimonSapin | ||
| Closed | SimonSapin |
| ) | ) | ||||
| def debugdirstateignorepatternshash(ui, repo, **opts): | def debugdirstateignorepatternshash(ui, repo, **opts): | ||||
| """show the hash of ignore patterns stored in dirstate if v2, | """show the hash of ignore patterns stored in dirstate if v2, | ||||
| or nothing for dirstate-v2 | or nothing for dirstate-v2 | ||||
| """ | """ | ||||
| if repo.dirstate._use_dirstate_v2: | if repo.dirstate._use_dirstate_v2: | ||||
| docket = repo.dirstate._map.docket | docket = repo.dirstate._map.docket | ||||
| hash_len = 20 # 160 bits for SHA-1 | hash_len = 20 # 160 bits for SHA-1 | ||||
| hash_offset = docket.data_size - hash_len # hash is at the end | hash_bytes = docket.tree_metadata[-hash_len:] | ||||
| data_filename = docket.data_filename() | |||||
| with repo.vfs(data_filename) as f: | |||||
| f.seek(hash_offset) | |||||
| hash_bytes = f.read(hash_len) | |||||
| ui.write(binascii.hexlify(hash_bytes) + b'\n') | ui.write(binascii.hexlify(hash_bytes) + b'\n') | ||||
| @command( | @command( | ||||
| b'debugdiscovery', | b'debugdiscovery', | ||||
| [ | [ | ||||
| (b'', b'old', None, _(b'use old-style discovery')), | (b'', b'old', None, _(b'use old-style discovery')), | ||||
| ( | ( | ||||
| if self._use_dirstate_v2: | if self._use_dirstate_v2: | ||||
| if self.docket.uuid: | if self.docket.uuid: | ||||
| # TODO: use mmap when possible | # TODO: use mmap when possible | ||||
| data = self._opener.read(self.docket.data_filename()) | data = self._opener.read(self.docket.data_filename()) | ||||
| else: | else: | ||||
| data = b'' | data = b'' | ||||
| self._rustmap = rustmod.DirstateMap.new_v2( | self._rustmap = rustmod.DirstateMap.new_v2( | ||||
| data, self.docket.data_size | data, self.docket.data_size, self.docket.tree_metadata | ||||
| ) | ) | ||||
| parents = self.docket.parents | parents = self.docket.parents | ||||
| else: | else: | ||||
| self._rustmap, parents = rustmod.DirstateMap.new_v1( | self._rustmap, parents = rustmod.DirstateMap.new_v1( | ||||
| self._use_dirstate_tree, self._readdirstatefile() | self._use_dirstate_tree, self._readdirstatefile() | ||||
| ) | ) | ||||
| if parents and not self._dirtyparents: | if parents and not self._dirtyparents: | ||||
| packed = self._rustmap.write_v1(p1, p2, now) | packed = self._rustmap.write_v1(p1, p2, now) | ||||
| st.write(packed) | st.write(packed) | ||||
| st.close() | st.close() | ||||
| self._dirtyparents = False | self._dirtyparents = False | ||||
| return | return | ||||
| # We can only append to an existing data file if there is one | # We can only append to an existing data file if there is one | ||||
| can_append = self.docket.uuid is not None | can_append = self.docket.uuid is not None | ||||
| packed, append = self._rustmap.write_v2(now, can_append) | packed, meta, append = self._rustmap.write_v2(now, can_append) | ||||
| if append: | if append: | ||||
| docket = self.docket | docket = self.docket | ||||
| data_filename = docket.data_filename() | data_filename = docket.data_filename() | ||||
| if tr: | if tr: | ||||
| tr.add(data_filename, docket.data_size) | tr.add(data_filename, docket.data_size) | ||||
| with self._opener(data_filename, b'r+b') as fp: | with self._opener(data_filename, b'r+b') as fp: | ||||
| fp.seek(docket.data_size) | fp.seek(docket.data_size) | ||||
| assert fp.tell() == docket.data_size | assert fp.tell() == docket.data_size | ||||
| written = fp.write(packed) | written = fp.write(packed) | ||||
| if written is not None: # py2 may return None | if written is not None: # py2 may return None | ||||
| assert written == len(packed), (written, len(packed)) | assert written == len(packed), (written, len(packed)) | ||||
| docket.data_size += len(packed) | docket.data_size += len(packed) | ||||
| docket.parents = self.parents() | docket.parents = self.parents() | ||||
| docket.tree_metadata = meta | |||||
| st.write(docket.serialize()) | st.write(docket.serialize()) | ||||
| st.close() | st.close() | ||||
| else: | else: | ||||
| old_docket = self.docket | old_docket = self.docket | ||||
| new_docket = docketmod.DirstateDocket.with_new_uuid( | new_docket = docketmod.DirstateDocket.with_new_uuid( | ||||
| self.parents(), len(packed) | self.parents(), len(packed), meta | ||||
| ) | ) | ||||
| data_filename = new_docket.data_filename() | data_filename = new_docket.data_filename() | ||||
| if tr: | if tr: | ||||
| tr.add(data_filename, 0) | tr.add(data_filename, 0) | ||||
| self._opener.write(data_filename, packed) | self._opener.write(data_filename, packed) | ||||
| # Write the new docket after the new data file has been | # Write the new docket after the new data file has been | ||||
| # written. Because `st` was opened with `atomictemp=True`, | # written. Because `st` was opened with `atomictemp=True`, | ||||
| # the actual `.hg/dirstate` file is only affected on close. | # the actual `.hg/dirstate` file is only affected on close. | ||||
| # dirstatedocket.py - docket file for dirstate-v2 | # dirstatedocket.py - docket file for dirstate-v2 | ||||
| # | # | ||||
| # Copyright Mercurial Contributors | # Copyright Mercurial Contributors | ||||
| # | # | ||||
| # This software may be used and distributed according to the terms of the | # This software may be used and distributed according to the terms of the | ||||
| # GNU General Public License version 2 or any later version. | # GNU General Public License version 2 or any later version. | ||||
| from __future__ import absolute_import | from __future__ import absolute_import | ||||
| import struct | import struct | ||||
| from ..revlogutils import docket as docket_mod | from ..revlogutils import docket as docket_mod | ||||
| V2_FORMAT_MARKER = b"dirstate-v2\n" | V2_FORMAT_MARKER = b"dirstate-v2\n" | ||||
| # Must match the constant of the same name in | |||||
| # `rust/hg-core/src/dirstate_tree/on_disk.rs` | |||||
| TREE_METADATA_SIZE = 40 | |||||
| # * 12 bytes: format marker | # * 12 bytes: format marker | ||||
| # * 32 bytes: node ID of the working directory's first parent | # * 32 bytes: node ID of the working directory's first parent | ||||
| # * 32 bytes: node ID of the working directory's second parent | # * 32 bytes: node ID of the working directory's second parent | ||||
| # * 4 bytes: big-endian used size of the data file | # * 4 bytes: big-endian used size of the data file | ||||
| # * {TREE_METADATA_SIZE} bytes: tree metadata, parsed separately | |||||
| # * 1 byte: length of the data file's UUID | # * 1 byte: length of the data file's UUID | ||||
| # * variable: data file's UUID | # * variable: data file's UUID | ||||
| # | # | ||||
| # Node IDs are null-padded if shorter than 32 bytes. | # Node IDs are null-padded if shorter than 32 bytes. | ||||
| # A data file shorter than the specified used size is corrupted (truncated) | # A data file shorter than the specified used size is corrupted (truncated) | ||||
| HEADER = struct.Struct(">{}s32s32sLB".format(len(V2_FORMAT_MARKER))) | HEADER = struct.Struct( | ||||
| ">{}s32s32sL{}sB".format(len(V2_FORMAT_MARKER), TREE_METADATA_SIZE) | |||||
| ) | |||||
| class DirstateDocket(object): | class DirstateDocket(object): | ||||
| data_filename_pattern = b'dirstate.%s.d' | data_filename_pattern = b'dirstate.%s.d' | ||||
| def __init__(self, parents, data_size, uuid): | def __init__(self, parents, data_size, tree_metadata, uuid): | ||||
| self.parents = parents | self.parents = parents | ||||
| self.data_size = data_size | self.data_size = data_size | ||||
| self.tree_metadata = tree_metadata | |||||
| self.uuid = uuid | self.uuid = uuid | ||||
| @classmethod | @classmethod | ||||
| def with_new_uuid(cls, parents, data): | def with_new_uuid(cls, parents, data_size, tree_metadata): | ||||
| return cls(parents, data, docket_mod.make_uid()) | return cls(parents, data_size, tree_metadata, docket_mod.make_uid()) | ||||
| @classmethod | @classmethod | ||||
| def parse(cls, data, nodeconstants): | def parse(cls, data, nodeconstants): | ||||
| if not data: | if not data: | ||||
| parents = (nodeconstants.nullid, nodeconstants.nullid) | parents = (nodeconstants.nullid, nodeconstants.nullid) | ||||
| return cls(parents, 0, None) | return cls(parents, 0, b'', None) | ||||
| marker, p1, p2, data_size, uuid_size = HEADER.unpack_from(data) | marker, p1, p2, data_size, meta, uuid_size = HEADER.unpack_from(data) | ||||
| if marker != V2_FORMAT_MARKER: | if marker != V2_FORMAT_MARKER: | ||||
| raise ValueError("expected dirstate-v2 marker") | raise ValueError("expected dirstate-v2 marker") | ||||
| uuid = data[HEADER.size : HEADER.size + uuid_size] | uuid = data[HEADER.size : HEADER.size + uuid_size] | ||||
| p1 = p1[: nodeconstants.nodelen] | p1 = p1[: nodeconstants.nodelen] | ||||
| p2 = p2[: nodeconstants.nodelen] | p2 = p2[: nodeconstants.nodelen] | ||||
| return cls((p1, p2), data_size, uuid) | return cls((p1, p2), data_size, meta, uuid) | ||||
| def serialize(self): | def serialize(self): | ||||
| p1, p2 = self.parents | p1, p2 = self.parents | ||||
| header = HEADER.pack( | header = HEADER.pack( | ||||
| V2_FORMAT_MARKER, p1, p2, self.data_size, len(self.uuid) | V2_FORMAT_MARKER, | ||||
| p1, | |||||
| p2, | |||||
| self.data_size, | |||||
| self.tree_metadata, | |||||
| len(self.uuid), | |||||
| ) | ) | ||||
| return header + self.uuid | return header + self.uuid | ||||
| def data_filename(self): | def data_filename(self): | ||||
| return self.data_filename_pattern % self.uuid | return self.data_filename_pattern % self.uuid | ||||
| unreachable_bytes: 0, | unreachable_bytes: 0, | ||||
| } | } | ||||
| } | } | ||||
| #[timed] | #[timed] | ||||
| pub fn new_v2( | pub fn new_v2( | ||||
| on_disk: &'on_disk [u8], | on_disk: &'on_disk [u8], | ||||
| data_size: usize, | data_size: usize, | ||||
| metadata: &[u8], | |||||
| ) -> Result<Self, DirstateError> { | ) -> Result<Self, DirstateError> { | ||||
| if let Some(data) = on_disk.get(..data_size) { | if let Some(data) = on_disk.get(..data_size) { | ||||
| Ok(on_disk::read(data)?) | Ok(on_disk::read(data, metadata)?) | ||||
| } else { | } else { | ||||
| Err(DirstateV2ParseError.into()) | Err(DirstateV2ParseError.into()) | ||||
| } | } | ||||
| } | } | ||||
| #[timed] | #[timed] | ||||
| pub fn new_v1( | pub fn new_v1( | ||||
| on_disk: &'on_disk [u8], | on_disk: &'on_disk [u8], | ||||
| node.copy_source(self.on_disk)?, | node.copy_source(self.on_disk)?, | ||||
| &mut packed, | &mut packed, | ||||
| ); | ); | ||||
| } | } | ||||
| } | } | ||||
| Ok(packed) | Ok(packed) | ||||
| } | } | ||||
| /// Returns new data together with whether that data should be appended to | /// Returns new data and metadata together with whether that data should be | ||||
| /// the existing data file whose content is at `self.on_disk` (true), | /// appended to the existing data file whose content is at | ||||
| /// instead of written to a new data file (false). | /// `self.on_disk` (true), instead of written to a new data file | ||||
| /// (false). | |||||
| #[timed] | #[timed] | ||||
| fn pack_v2( | fn pack_v2( | ||||
| &mut self, | &mut self, | ||||
| now: Timestamp, | now: Timestamp, | ||||
| can_append: bool, | can_append: bool, | ||||
| ) -> Result<(Vec<u8>, bool), DirstateError> { | ) -> Result<(Vec<u8>, Vec<u8>, bool), DirstateError> { | ||||
| // TODO:!how do we want to handle this in 2038? | // TODO:!how do we want to handle this in 2038? | ||||
| let now: i32 = now.0.try_into().expect("time overflow"); | let now: i32 = now.0.try_into().expect("time overflow"); | ||||
| let mut paths = Vec::new(); | let mut paths = Vec::new(); | ||||
| for node in self.iter_nodes() { | for node in self.iter_nodes() { | ||||
| let node = node?; | let node = node?; | ||||
| if let Some(entry) = node.entry()? { | if let Some(entry) = node.entry()? { | ||||
| if entry.mtime_is_ambiguous(now) { | if entry.mtime_is_ambiguous(now) { | ||||
| paths.push( | paths.push( | ||||
| now: Timestamp, | now: Timestamp, | ||||
| ) -> Result<Vec<u8>, DirstateError>; | ) -> Result<Vec<u8>, DirstateError>; | ||||
| /// Clear mtimes that are ambigous with `now` (similar to | /// Clear mtimes that are ambigous with `now` (similar to | ||||
| /// `clear_ambiguous_times` but for all files in the dirstate map), and | /// `clear_ambiguous_times` but for all files in the dirstate map), and | ||||
| /// serialize bytes to write a dirstate data file to disk in dirstate-v2 | /// serialize bytes to write a dirstate data file to disk in dirstate-v2 | ||||
| /// format. | /// format. | ||||
| /// | /// | ||||
| /// Returns new data together with whether that data should be appended to | /// Returns new data and metadata together with whether that data should be | ||||
| /// the existing data file whose content is at `self.on_disk` (true), | /// appended to the existing data file whose content is at | ||||
| /// instead of written to a new data file (false). | /// `self.on_disk` (true), instead of written to a new data file | ||||
| /// (false). | |||||
| /// | /// | ||||
| /// Note: this is only supported by the tree dirstate map. | /// Note: this is only supported by the tree dirstate map. | ||||
| fn pack_v2( | fn pack_v2( | ||||
| &mut self, | &mut self, | ||||
| now: Timestamp, | now: Timestamp, | ||||
| can_append: bool, | can_append: bool, | ||||
| ) -> Result<(Vec<u8>, bool), DirstateError>; | ) -> Result<(Vec<u8>, Vec<u8>, bool), DirstateError>; | ||||
| /// Run the status algorithm. | /// Run the status algorithm. | ||||
| /// | /// | ||||
| /// This is not sematically a method of the dirstate map, but a different | /// This is not sematically a method of the dirstate map, but a different | ||||
| /// algorithm is used for the flat v.s. tree dirstate map so having it in | /// algorithm is used for the flat v.s. tree dirstate map so having it in | ||||
| /// this trait enables the same dynamic dispatch as with other methods. | /// this trait enables the same dynamic dispatch as with other methods. | ||||
| fn status<'a>( | fn status<'a>( | ||||
| &'a mut self, | &'a mut self, | ||||
| ) -> Result<Vec<u8>, DirstateError> { | ) -> Result<Vec<u8>, DirstateError> { | ||||
| self.pack(parents, now) | self.pack(parents, now) | ||||
| } | } | ||||
| fn pack_v2( | fn pack_v2( | ||||
| &mut self, | &mut self, | ||||
| _now: Timestamp, | _now: Timestamp, | ||||
| _can_append: bool, | _can_append: bool, | ||||
| ) -> Result<(Vec<u8>, bool), DirstateError> { | ) -> Result<(Vec<u8>, Vec<u8>, bool), DirstateError> { | ||||
| panic!( | panic!( | ||||
| "should have used dirstate_tree::DirstateMap to use the v2 format" | "should have used dirstate_tree::DirstateMap to use the v2 format" | ||||
| ) | ) | ||||
| } | } | ||||
| fn status<'a>( | fn status<'a>( | ||||
| &'a mut self, | &'a mut self, | ||||
| matcher: &'a (dyn Matcher + Sync), | matcher: &'a (dyn Matcher + Sync), | ||||
| const STORED_NODE_ID_BYTES: usize = 32; | const STORED_NODE_ID_BYTES: usize = 32; | ||||
| /// … even though only 160 bits are used for now, with SHA-1 | /// … even though only 160 bits are used for now, with SHA-1 | ||||
| const USED_NODE_ID_BYTES: usize = 20; | const USED_NODE_ID_BYTES: usize = 20; | ||||
| pub(super) const IGNORE_PATTERNS_HASH_LEN: usize = 20; | pub(super) const IGNORE_PATTERNS_HASH_LEN: usize = 20; | ||||
| pub(super) type IgnorePatternsHash = [u8; IGNORE_PATTERNS_HASH_LEN]; | pub(super) type IgnorePatternsHash = [u8; IGNORE_PATTERNS_HASH_LEN]; | ||||
| /// Must match the constant of the same name in | |||||
| /// `mercurial/dirstateutils/docket.py` | |||||
| const TREE_METADATA_SIZE: usize = 40; | |||||
| /// Make sure that size-affecting changes are made knowingly | |||||
| #[allow(unused)] | |||||
| fn static_assert_size_of() { | |||||
| let _ = std::mem::transmute::<DocketHeader, [u8; 121]>; | |||||
| let _ = std::mem::transmute::<TreeMetadata, [u8; TREE_METADATA_SIZE]>; | |||||
| let _ = std::mem::transmute::<Node, [u8; 43]>; | |||||
| } | |||||
| // Must match `HEADER` in `mercurial/dirstateutils/docket.py` | // Must match `HEADER` in `mercurial/dirstateutils/docket.py` | ||||
| #[derive(BytesCast)] | #[derive(BytesCast)] | ||||
| #[repr(C)] | #[repr(C)] | ||||
| struct DocketHeader { | struct DocketHeader { | ||||
| marker: [u8; V2_FORMAT_MARKER.len()], | marker: [u8; V2_FORMAT_MARKER.len()], | ||||
| parent_1: [u8; STORED_NODE_ID_BYTES], | parent_1: [u8; STORED_NODE_ID_BYTES], | ||||
| parent_2: [u8; STORED_NODE_ID_BYTES], | parent_2: [u8; STORED_NODE_ID_BYTES], | ||||
| /// Counted in bytes | /// Counted in bytes | ||||
| data_size: Size, | data_size: Size, | ||||
| metadata: TreeMetadata, | |||||
| uuid_size: u8, | uuid_size: u8, | ||||
| } | } | ||||
| pub struct Docket<'on_disk> { | pub struct Docket<'on_disk> { | ||||
| header: &'on_disk DocketHeader, | header: &'on_disk DocketHeader, | ||||
| uuid: &'on_disk [u8], | uuid: &'on_disk [u8], | ||||
| } | } | ||||
| #[derive(BytesCast)] | #[derive(BytesCast)] | ||||
| #[repr(C)] | #[repr(C)] | ||||
| struct Root { | struct TreeMetadata { | ||||
| root_nodes: ChildNodes, | root_nodes: ChildNodes, | ||||
| nodes_with_entry_count: Size, | nodes_with_entry_count: Size, | ||||
| nodes_with_copy_source_count: Size, | nodes_with_copy_source_count: Size, | ||||
| /// How many bytes of this data file are not used anymore | /// How many bytes of this data file are not used anymore | ||||
| unreachable_bytes: Size, | unreachable_bytes: Size, | ||||
| /// If non-zero, a hash of ignore files that were used for some previous | /// If non-zero, a hash of ignore files that were used for some previous | ||||
| /// - That timestamp was already strictly in the past when observed, | /// - That timestamp was already strictly in the past when observed, | ||||
| /// meaning that later changes cannot happen in the same clock tick | /// meaning that later changes cannot happen in the same clock tick | ||||
| /// and must cause a different modification time (unless the system | /// and must cause a different modification time (unless the system | ||||
| /// clock jumps back and we get unlucky, which is not impossible but | /// clock jumps back and we get unlucky, which is not impossible but | ||||
| /// but deemed unlikely enough). | /// but deemed unlikely enough). | ||||
| /// - All direct children of this directory (as returned by | /// - All direct children of this directory (as returned by | ||||
| /// `std::fs::read_dir`) either have a corresponding dirstate node, or | /// `std::fs::read_dir`) either have a corresponding dirstate node, or | ||||
| /// are ignored by ignore patterns whose hash is in | /// are ignored by ignore patterns whose hash is in | ||||
| /// `Root::ignore_patterns_hash`. | /// `TreeMetadata::ignore_patterns_hash`. | ||||
| /// | /// | ||||
| /// This means that if `std::fs::symlink_metadata` later reports the | /// This means that if `std::fs::symlink_metadata` later reports the | ||||
| /// same modification time and ignored patterns haven’t changed, a run | /// same modification time and ignored patterns haven’t changed, a run | ||||
| /// of status that is not listing ignored files can skip calling | /// of status that is not listing ignored files can skip calling | ||||
| /// `std::fs::read_dir` again for this directory, iterate child | /// `std::fs::read_dir` again for this directory, iterate child | ||||
| /// dirstate nodes instead. | /// dirstate nodes instead. | ||||
| state: u8, | state: u8, | ||||
| data: Entry, | data: Entry, | ||||
| struct PathSlice { | struct PathSlice { | ||||
| start: Offset, | start: Offset, | ||||
| len: PathSize, | len: PathSize, | ||||
| } | } | ||||
| /// Either nothing if `start == 0`, or a `HgPath` of `len` bytes | /// Either nothing if `start == 0`, or a `HgPath` of `len` bytes | ||||
| type OptPathSlice = PathSlice; | type OptPathSlice = PathSlice; | ||||
| /// Make sure that size-affecting changes are made knowingly | |||||
| fn _static_assert_size_of() { | |||||
| let _ = std::mem::transmute::<DocketHeader, [u8; 81]>; | |||||
| let _ = std::mem::transmute::<Root, [u8; 40]>; | |||||
| let _ = std::mem::transmute::<Node, [u8; 43]>; | |||||
| } | |||||
| /// Unexpected file format found in `.hg/dirstate` with the "v2" format. | /// Unexpected file format found in `.hg/dirstate` with the "v2" format. | ||||
| /// | /// | ||||
| /// This should only happen if Mercurial is buggy or a repository is corrupted. | /// This should only happen if Mercurial is buggy or a repository is corrupted. | ||||
| #[derive(Debug)] | #[derive(Debug)] | ||||
| pub struct DirstateV2ParseError; | pub struct DirstateV2ParseError; | ||||
| impl From<DirstateV2ParseError> for HgError { | impl From<DirstateV2ParseError> for HgError { | ||||
| fn from(_: DirstateV2ParseError) -> Self { | fn from(_: DirstateV2ParseError) -> Self { | ||||
| .unwrap() | .unwrap() | ||||
| .clone(); | .clone(); | ||||
| let p2 = Node::try_from(&self.header.parent_2[..USED_NODE_ID_BYTES]) | let p2 = Node::try_from(&self.header.parent_2[..USED_NODE_ID_BYTES]) | ||||
| .unwrap() | .unwrap() | ||||
| .clone(); | .clone(); | ||||
| DirstateParents { p1, p2 } | DirstateParents { p1, p2 } | ||||
| } | } | ||||
| pub fn tree_metadata(&self) -> &[u8] { | |||||
| self.header.metadata.as_bytes() | |||||
| } | |||||
| pub fn data_size(&self) -> usize { | pub fn data_size(&self) -> usize { | ||||
| // This `unwrap` could only panic on a 16-bit CPU | // This `unwrap` could only panic on a 16-bit CPU | ||||
| self.header.data_size.get().try_into().unwrap() | self.header.data_size.get().try_into().unwrap() | ||||
| } | } | ||||
| pub fn data_filename(&self) -> String { | pub fn data_filename(&self) -> String { | ||||
| String::from_utf8(format_bytes!(b"dirstate.{}.d", self.uuid)).unwrap() | String::from_utf8(format_bytes!(b"dirstate.{}.d", self.uuid)).unwrap() | ||||
| } | } | ||||
| } | } | ||||
| pub fn read_docket( | pub fn read_docket( | ||||
| on_disk: &[u8], | on_disk: &[u8], | ||||
| ) -> Result<Docket<'_>, DirstateV2ParseError> { | ) -> Result<Docket<'_>, DirstateV2ParseError> { | ||||
| let (header, uuid) = | let (header, uuid) = | ||||
| DocketHeader::from_bytes(on_disk).map_err(|_| DirstateV2ParseError)?; | DocketHeader::from_bytes(on_disk).map_err(|_| DirstateV2ParseError)?; | ||||
| let uuid_size = header.uuid_size as usize; | let uuid_size = header.uuid_size as usize; | ||||
| if header.marker == *V2_FORMAT_MARKER && uuid.len() == uuid_size { | if header.marker == *V2_FORMAT_MARKER && uuid.len() == uuid_size { | ||||
| Ok(Docket { header, uuid }) | Ok(Docket { header, uuid }) | ||||
| } else { | } else { | ||||
| Err(DirstateV2ParseError) | Err(DirstateV2ParseError) | ||||
| } | } | ||||
| } | } | ||||
| fn read_root<'on_disk>( | |||||
| on_disk: &'on_disk [u8], | |||||
| ) -> Result<&'on_disk Root, DirstateV2ParseError> { | |||||
| // Find the `Root` at the end of the given slice | |||||
| let root_offset = on_disk | |||||
| .len() | |||||
| .checked_sub(std::mem::size_of::<Root>()) | |||||
| // A non-empty slice too short is an error | |||||
| .ok_or(DirstateV2ParseError)?; | |||||
| let (root, _) = Root::from_bytes(&on_disk[root_offset..]) | |||||
| .map_err(|_| DirstateV2ParseError)?; | |||||
| Ok(root) | |||||
| } | |||||
| pub(super) fn read<'on_disk>( | pub(super) fn read<'on_disk>( | ||||
| on_disk: &'on_disk [u8], | on_disk: &'on_disk [u8], | ||||
| metadata: &[u8], | |||||
| ) -> Result<DirstateMap<'on_disk>, DirstateV2ParseError> { | ) -> Result<DirstateMap<'on_disk>, DirstateV2ParseError> { | ||||
| if on_disk.is_empty() { | if on_disk.is_empty() { | ||||
| return Ok(DirstateMap::empty(on_disk)); | return Ok(DirstateMap::empty(on_disk)); | ||||
| } | } | ||||
| let root = read_root(on_disk)?; | let (meta, _) = TreeMetadata::from_bytes(metadata) | ||||
| let mut unreachable_bytes = root.unreachable_bytes.get(); | .map_err(|_| DirstateV2ParseError)?; | ||||
| // Each append writes a new `Root`, so it’s never reused | |||||
| unreachable_bytes += std::mem::size_of::<Root>() as u32; | |||||
| let dirstate_map = DirstateMap { | let dirstate_map = DirstateMap { | ||||
| on_disk, | on_disk, | ||||
| root: dirstate_map::ChildNodes::OnDisk(read_nodes( | root: dirstate_map::ChildNodes::OnDisk(read_nodes( | ||||
| on_disk, | on_disk, | ||||
| root.root_nodes, | meta.root_nodes, | ||||
| )?), | )?), | ||||
| nodes_with_entry_count: root.nodes_with_entry_count.get(), | nodes_with_entry_count: meta.nodes_with_entry_count.get(), | ||||
| nodes_with_copy_source_count: root.nodes_with_copy_source_count.get(), | nodes_with_copy_source_count: meta.nodes_with_copy_source_count.get(), | ||||
| ignore_patterns_hash: root.ignore_patterns_hash, | ignore_patterns_hash: meta.ignore_patterns_hash, | ||||
| unreachable_bytes, | unreachable_bytes: meta.unreachable_bytes.get(), | ||||
| }; | }; | ||||
| Ok(dirstate_map) | Ok(dirstate_map) | ||||
| } | } | ||||
| impl Node { | impl Node { | ||||
| pub(super) fn full_path<'on_disk>( | pub(super) fn full_path<'on_disk>( | ||||
| &self, | &self, | ||||
| on_disk: &'on_disk [u8], | on_disk: &'on_disk [u8], | ||||
| .get(start..) | .get(start..) | ||||
| .and_then(|bytes| T::slice_from_bytes(bytes, len).ok()) | .and_then(|bytes| T::slice_from_bytes(bytes, len).ok()) | ||||
| .map(|(slice, _rest)| slice) | .map(|(slice, _rest)| slice) | ||||
| .ok_or_else(|| DirstateV2ParseError) | .ok_or_else(|| DirstateV2ParseError) | ||||
| } | } | ||||
| pub(crate) fn for_each_tracked_path<'on_disk>( | pub(crate) fn for_each_tracked_path<'on_disk>( | ||||
| on_disk: &'on_disk [u8], | on_disk: &'on_disk [u8], | ||||
| metadata: &[u8], | |||||
| mut f: impl FnMut(&'on_disk HgPath), | mut f: impl FnMut(&'on_disk HgPath), | ||||
| ) -> Result<(), DirstateV2ParseError> { | ) -> Result<(), DirstateV2ParseError> { | ||||
| let root = read_root(on_disk)?; | let (meta, _) = TreeMetadata::from_bytes(metadata) | ||||
| .map_err(|_| DirstateV2ParseError)?; | |||||
| fn recur<'on_disk>( | fn recur<'on_disk>( | ||||
| on_disk: &'on_disk [u8], | on_disk: &'on_disk [u8], | ||||
| nodes: ChildNodes, | nodes: ChildNodes, | ||||
| f: &mut impl FnMut(&'on_disk HgPath), | f: &mut impl FnMut(&'on_disk HgPath), | ||||
| ) -> Result<(), DirstateV2ParseError> { | ) -> Result<(), DirstateV2ParseError> { | ||||
| for node in read_nodes(on_disk, nodes)? { | for node in read_nodes(on_disk, nodes)? { | ||||
| if let Some(state) = node.state()? { | if let Some(state) = node.state()? { | ||||
| if state.is_tracked() { | if state.is_tracked() { | ||||
| f(node.full_path(on_disk)?) | f(node.full_path(on_disk)?) | ||||
| } | } | ||||
| } | } | ||||
| recur(on_disk, node.children, f)? | recur(on_disk, node.children, f)? | ||||
| } | } | ||||
| Ok(()) | Ok(()) | ||||
| } | } | ||||
| recur(on_disk, root.root_nodes, &mut f) | recur(on_disk, meta.root_nodes, &mut f) | ||||
| } | } | ||||
| /// Returns new data together with whether that data should be appended to the | /// Returns new data and metadata, together with whether that data should be | ||||
| /// existing data file whose content is at `dirstate_map.on_disk` (true), | /// appended to the existing data file whose content is at | ||||
| /// instead of written to a new data file (false). | /// `dirstate_map.on_disk` (true), instead of written to a new data file | ||||
| /// (false). | |||||
| pub(super) fn write( | pub(super) fn write( | ||||
| dirstate_map: &mut DirstateMap, | dirstate_map: &mut DirstateMap, | ||||
| can_append: bool, | can_append: bool, | ||||
| ) -> Result<(Vec<u8>, bool), DirstateError> { | ) -> Result<(Vec<u8>, Vec<u8>, bool), DirstateError> { | ||||
| let append = can_append && dirstate_map.write_should_append(); | let append = can_append && dirstate_map.write_should_append(); | ||||
| // This ignores the space for paths, and for nodes without an entry. | // This ignores the space for paths, and for nodes without an entry. | ||||
| // TODO: better estimate? Skip the `Vec` and write to a file directly? | // TODO: better estimate? Skip the `Vec` and write to a file directly? | ||||
| let size_guess = std::mem::size_of::<Root>() | let size_guess = std::mem::size_of::<Node>() | ||||
| + std::mem::size_of::<Node>() | |||||
| * dirstate_map.nodes_with_entry_count as usize; | * dirstate_map.nodes_with_entry_count as usize; | ||||
| let mut writer = Writer { | let mut writer = Writer { | ||||
| dirstate_map, | dirstate_map, | ||||
| append, | append, | ||||
| out: Vec::with_capacity(size_guess), | out: Vec::with_capacity(size_guess), | ||||
| }; | }; | ||||
| let root_nodes = writer.write_nodes(dirstate_map.root.as_ref())?; | let root_nodes = writer.write_nodes(dirstate_map.root.as_ref())?; | ||||
| let root = Root { | let meta = TreeMetadata { | ||||
| root_nodes, | root_nodes, | ||||
| nodes_with_entry_count: dirstate_map.nodes_with_entry_count.into(), | nodes_with_entry_count: dirstate_map.nodes_with_entry_count.into(), | ||||
| nodes_with_copy_source_count: dirstate_map | nodes_with_copy_source_count: dirstate_map | ||||
| .nodes_with_copy_source_count | .nodes_with_copy_source_count | ||||
| .into(), | .into(), | ||||
| unreachable_bytes: dirstate_map.unreachable_bytes.into(), | unreachable_bytes: dirstate_map.unreachable_bytes.into(), | ||||
| ignore_patterns_hash: dirstate_map.ignore_patterns_hash, | ignore_patterns_hash: dirstate_map.ignore_patterns_hash, | ||||
| }; | }; | ||||
| writer.out.extend(root.as_bytes()); | Ok((writer.out, meta.as_bytes().to_vec(), append)) | ||||
| Ok((writer.out, append)) | |||||
| } | } | ||||
| struct Writer<'dmap, 'on_disk> { | struct Writer<'dmap, 'on_disk> { | ||||
| dirstate_map: &'dmap DirstateMap<'on_disk>, | dirstate_map: &'dmap DirstateMap<'on_disk>, | ||||
| append: bool, | append: bool, | ||||
| out: Vec<u8>, | out: Vec<u8>, | ||||
| } | } | ||||
| use crate::DirstateError; | use crate::DirstateError; | ||||
| use rayon::prelude::*; | use rayon::prelude::*; | ||||
| /// List files under Mercurial control in the working directory | /// List files under Mercurial control in the working directory | ||||
| /// by reading the dirstate | /// by reading the dirstate | ||||
| pub struct Dirstate { | pub struct Dirstate { | ||||
| /// The `dirstate` content. | /// The `dirstate` content. | ||||
| content: Vec<u8>, | content: Vec<u8>, | ||||
| dirstate_v2: bool, | v2_metadata: Option<Vec<u8>>, | ||||
| } | } | ||||
| impl Dirstate { | impl Dirstate { | ||||
| pub fn new(repo: &Repo) -> Result<Self, HgError> { | pub fn new(repo: &Repo) -> Result<Self, HgError> { | ||||
| let mut content = repo.hg_vfs().read("dirstate")?; | let mut content = repo.hg_vfs().read("dirstate")?; | ||||
| if repo.has_dirstate_v2() { | let v2_metadata = if repo.has_dirstate_v2() { | ||||
| let docket = read_docket(&content)?; | let docket = read_docket(&content)?; | ||||
| let meta = docket.tree_metadata().to_vec(); | |||||
| content = repo.hg_vfs().read(docket.data_filename())?; | content = repo.hg_vfs().read(docket.data_filename())?; | ||||
| } | Some(meta) | ||||
| } else { | |||||
| None | |||||
| }; | |||||
| Ok(Self { | Ok(Self { | ||||
| content, | content, | ||||
| dirstate_v2: repo.has_dirstate_v2(), | v2_metadata, | ||||
| }) | }) | ||||
| } | } | ||||
| pub fn tracked_files(&self) -> Result<Vec<&HgPath>, DirstateError> { | pub fn tracked_files(&self) -> Result<Vec<&HgPath>, DirstateError> { | ||||
| let mut files = Vec::new(); | let mut files = Vec::new(); | ||||
| if !self.content.is_empty() { | if !self.content.is_empty() { | ||||
| if self.dirstate_v2 { | if let Some(meta) = &self.v2_metadata { | ||||
| for_each_tracked_path(&self.content, |path| files.push(path))? | for_each_tracked_path(&self.content, meta, |path| { | ||||
| files.push(path) | |||||
| })? | |||||
| } else { | } else { | ||||
| let _parents = parse_dirstate_entries( | let _parents = parse_dirstate_entries( | ||||
| &self.content, | &self.content, | ||||
| |path, entry, _copy_source| { | |path, entry, _copy_source| { | ||||
| if entry.state.is_tracked() { | if entry.state.is_tracked() { | ||||
| files.push(path) | files.push(path) | ||||
| } | } | ||||
| Ok(()) | Ok(()) | ||||
| Ok((map, parents).to_py_object(py).into_object()) | Ok((map, parents).to_py_object(py).into_object()) | ||||
| } | } | ||||
| /// Returns a DirstateMap | /// Returns a DirstateMap | ||||
| @staticmethod | @staticmethod | ||||
| def new_v2( | def new_v2( | ||||
| on_disk: PyBytes, | on_disk: PyBytes, | ||||
| data_size: usize, | data_size: usize, | ||||
| tree_metadata: PyBytes, | |||||
| ) -> PyResult<PyObject> { | ) -> PyResult<PyObject> { | ||||
| let dirstate_error = |e: DirstateError| { | let dirstate_error = |e: DirstateError| { | ||||
| PyErr::new::<exc::OSError, _>(py, format!("Dirstate error: {:?}", e)) | PyErr::new::<exc::OSError, _>(py, format!("Dirstate error: {:?}", e)) | ||||
| }; | }; | ||||
| let inner = OwningDirstateMap::new_v2(py, on_disk, data_size) | let inner = OwningDirstateMap::new_v2( | ||||
| .map_err(dirstate_error)?; | py, on_disk, data_size, tree_metadata, | ||||
| ).map_err(dirstate_error)?; | |||||
| let map = Self::create_instance(py, Box::new(inner))?; | let map = Self::create_instance(py, Box::new(inner))?; | ||||
| Ok(map.into_object()) | Ok(map.into_object()) | ||||
| } | } | ||||
| def clear(&self) -> PyResult<PyObject> { | def clear(&self) -> PyResult<PyObject> { | ||||
| self.inner(py).borrow_mut().clear(); | self.inner(py).borrow_mut().clear(); | ||||
| Ok(py.None()) | Ok(py.None()) | ||||
| } | } | ||||
| now: PyObject, | now: PyObject, | ||||
| can_append: bool, | can_append: bool, | ||||
| ) -> PyResult<PyObject> { | ) -> PyResult<PyObject> { | ||||
| let now = Timestamp(now.extract(py)?); | let now = Timestamp(now.extract(py)?); | ||||
| let mut inner = self.inner(py).borrow_mut(); | let mut inner = self.inner(py).borrow_mut(); | ||||
| let result = inner.pack_v2(now, can_append); | let result = inner.pack_v2(now, can_append); | ||||
| match result { | match result { | ||||
| Ok((packed, append)) => { | Ok((packed, tree_metadata, append)) => { | ||||
| let packed = PyBytes::new(py, &packed); | let packed = PyBytes::new(py, &packed); | ||||
| Ok((packed, append).to_py_object(py).into_object()) | let tree_metadata = PyBytes::new(py, &tree_metadata); | ||||
| let tuple = (packed, tree_metadata, append); | |||||
| Ok(tuple.to_py_object(py).into_object()) | |||||
| }, | }, | ||||
| Err(_) => Err(PyErr::new::<exc::OSError, _>( | Err(_) => Err(PyErr::new::<exc::OSError, _>( | ||||
| py, | py, | ||||
| "Dirstate error".to_string(), | "Dirstate error".to_string(), | ||||
| )), | )), | ||||
| } | } | ||||
| } | } | ||||
| ) -> Result<Vec<u8>, DirstateError> { | ) -> Result<Vec<u8>, DirstateError> { | ||||
| self.get_mut().pack_v1(parents, now) | self.get_mut().pack_v1(parents, now) | ||||
| } | } | ||||
| fn pack_v2( | fn pack_v2( | ||||
| &mut self, | &mut self, | ||||
| now: Timestamp, | now: Timestamp, | ||||
| can_append: bool, | can_append: bool, | ||||
| ) -> Result<(Vec<u8>, bool), DirstateError> { | ) -> Result<(Vec<u8>, Vec<u8>, bool), DirstateError> { | ||||
| self.get_mut().pack_v2(now, can_append) | self.get_mut().pack_v2(now, can_append) | ||||
| } | } | ||||
| fn status<'a>( | fn status<'a>( | ||||
| &'a mut self, | &'a mut self, | ||||
| matcher: &'a (dyn Matcher + Sync), | matcher: &'a (dyn Matcher + Sync), | ||||
| root_dir: PathBuf, | root_dir: PathBuf, | ||||
| ignore_files: Vec<PathBuf>, | ignore_files: Vec<PathBuf>, | ||||
| Ok((Self { on_disk, ptr }, parents)) | Ok((Self { on_disk, ptr }, parents)) | ||||
| } | } | ||||
| pub fn new_v2( | pub fn new_v2( | ||||
| py: Python, | py: Python, | ||||
| on_disk: PyBytes, | on_disk: PyBytes, | ||||
| data_size: usize, | data_size: usize, | ||||
| tree_metadata: PyBytes, | |||||
| ) -> Result<Self, DirstateError> { | ) -> Result<Self, DirstateError> { | ||||
| let bytes: &'_ [u8] = on_disk.data(py); | let bytes: &'_ [u8] = on_disk.data(py); | ||||
| let map = DirstateMap::new_v2(bytes, data_size)?; | let map = | ||||
| DirstateMap::new_v2(bytes, data_size, tree_metadata.data(py))?; | |||||
| // Like in `bytes` above, this `'_` lifetime parameter borrows from | // Like in `bytes` above, this `'_` lifetime parameter borrows from | ||||
| // the bytes buffer owned by `on_disk`. | // the bytes buffer owned by `on_disk`. | ||||
| let ptr: *mut DirstateMap<'_> = Box::into_raw(Box::new(map)); | let ptr: *mut DirstateMap<'_> = Box::into_raw(Box::new(map)); | ||||
| // Erase the pointed type entirely in order to erase the lifetime. | // Erase the pointed type entirely in order to erase the lifetime. | ||||
| let ptr: *mut () = ptr.cast(); | let ptr: *mut () = ptr.cast(); | ||||
| } else { | } else { | ||||
| requested | requested | ||||
| } | } | ||||
| }; | }; | ||||
| let repo = invocation.repo?; | let repo = invocation.repo?; | ||||
| let dirstate_data_mmap; | let dirstate_data_mmap; | ||||
| let (mut dmap, parents) = if repo.has_dirstate_v2() { | let (mut dmap, parents) = if repo.has_dirstate_v2() { | ||||
| let docket_data = | |||||
| repo.hg_vfs().read("dirstate").io_not_found_as_none()?; | |||||
| let parents; | let parents; | ||||
| let dirstate_data; | let dirstate_data; | ||||
| let data_size; | let data_size; | ||||
| if let Some(docket_data) = | let docket; | ||||
| repo.hg_vfs().read("dirstate").io_not_found_as_none()? | let tree_metadata; | ||||
| { | if let Some(docket_data) = &docket_data { | ||||
| let docket = on_disk::read_docket(&docket_data)?; | docket = on_disk::read_docket(docket_data)?; | ||||
| tree_metadata = docket.tree_metadata(); | |||||
| parents = Some(docket.parents()); | parents = Some(docket.parents()); | ||||
| data_size = docket.data_size(); | data_size = docket.data_size(); | ||||
| dirstate_data_mmap = repo | dirstate_data_mmap = repo | ||||
| .hg_vfs() | .hg_vfs() | ||||
| .mmap_open(docket.data_filename()) | .mmap_open(docket.data_filename()) | ||||
| .io_not_found_as_none()?; | .io_not_found_as_none()?; | ||||
| dirstate_data = dirstate_data_mmap.as_deref().unwrap_or(b""); | dirstate_data = dirstate_data_mmap.as_deref().unwrap_or(b""); | ||||
| } else { | } else { | ||||
| parents = None; | parents = None; | ||||
| tree_metadata = b""; | |||||
| data_size = 0; | data_size = 0; | ||||
| dirstate_data = b""; | dirstate_data = b""; | ||||
| } | } | ||||
| let dmap = DirstateMap::new_v2(dirstate_data, data_size)?; | let dmap = | ||||
| DirstateMap::new_v2(dirstate_data, data_size, tree_metadata)?; | |||||
| (dmap, parents) | (dmap, parents) | ||||
| } else { | } else { | ||||
| dirstate_data_mmap = | dirstate_data_mmap = | ||||
| repo.hg_vfs().mmap_open("dirstate").io_not_found_as_none()?; | repo.hg_vfs().mmap_open("dirstate").io_not_found_as_none()?; | ||||
| let dirstate_data = dirstate_data_mmap.as_deref().unwrap_or(b""); | let dirstate_data = dirstate_data_mmap.as_deref().unwrap_or(b""); | ||||
| DirstateMap::new_v1(dirstate_data)? | DirstateMap::new_v1(dirstate_data)? | ||||
| }; | }; | ||||