This is an archive of the discontinued Mercurial Phabricator instance.

Differential D11682

dirstate-v2: adds two flag to track the presence of some unrecorded files
ClosedPublic

Authored by marmoute on Oct 18 2021, 2:18 AM.

Details

Reviewers
Alphare
Group Reviewers
hg-reviewers
Commits
rHGdfc5a505ddc5: dirstate-v2: adds two flag to track the presence of some unrecorded files
Summary

Right now, we don't record ignored or unknown files in the dirstate. However
the structure would allow it. So we introduce two flags that can be used to
clarify whether all unknown/ignored children are recorded or not. This will
allow for more information to be stored in the future if this end up being
relevant.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

marmoute created this revision.Oct 18 2021, 2:18 AM
Alphare requested changes to this revision.Oct 18 2021, 12:00 PM
Alphare added a subscriber: Alphare.

After discussing this in a VC, we'll flip the logic to avoid double negatives

This revision now requires changes to proceed.Oct 18 2021, 12:00 PM
Alphare requested changes to this revision.Oct 19 2021, 3:58 AM

Aside from my one comment, the commit message should say "whether unknown/ignored children are recorded or not"

mercurial/cext/util.h
44

This should be dirstate_flag_all_ignored_recorded

This revision now requires changes to proceed.Oct 19 2021, 3:58 AM
marmoute edited the summary of this revision. (Show Details)Oct 19 2021, 4:35 AM
marmoute updated this revision to Diff 30888.
Alphare accepted this revision.Oct 19 2021, 8:08 AM
This revision is now accepted and ready to land.Oct 19 2021, 8:08 AM

Revision Contents
Changeset List

Diff 30900

mercurial/cext/util.h

static const int dirstate_flag_p1_tracked = 1 << 1; static const int dirstate_flag_p1_tracked = 1 << 1;
static const int dirstate_flag_p2_info = 1 << 2; static const int dirstate_flag_p2_info = 1 << 2;
static const int dirstate_flag_has_meaningful_data = 1 << 3; static const int dirstate_flag_has_meaningful_data = 1 << 3;
static const int dirstate_flag_has_file_mtime = 1 << 4; static const int dirstate_flag_has_file_mtime = 1 << 4;
static const int dirstate_flag_has_directory_mtime = 1 << 5; static const int dirstate_flag_has_directory_mtime = 1 << 5;
static const int dirstate_flag_mode_exec_perm = 1 << 6; static const int dirstate_flag_mode_exec_perm = 1 << 6;
static const int dirstate_flag_mode_is_symlink = 1 << 7; static const int dirstate_flag_mode_is_symlink = 1 << 7;
static const int dirstate_flag_expected_state_is_modified = 1 << 8; static const int dirstate_flag_expected_state_is_modified = 1 << 8;
static const int dirstate_flag_all_unknown_recorded = 1 << 9;
static const int dirstate_flag_all_ignored_recorded = 1 << 10;
AlphareUnsubmitted
Not Done

This should be dirstate_flag_all_ignored_recorded

Alphare: This should be `dirstate_flag_all_ignored_recorded`
extern PyTypeObject dirstateItemType; extern PyTypeObject dirstateItemType;
#define dirstate_tuple_check(op) (Py_TYPE(op) == &dirstateItemType) #define dirstate_tuple_check(op) (Py_TYPE(op) == &dirstateItemType)
#ifndef MIN #ifndef MIN
#define MIN(a, b) (((a) < (b)) ? (a) : (b)) #define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif #endif
/* VC9 doesn't include bool and lacks stdbool.h based on my searching */ /* VC9 doesn't include bool and lacks stdbool.h based on my searching */

mercurial/dirstateutils/v2.py

) = TREE_METADATA.unpack(tree_metadata) ) = TREE_METADATA.unpack(tree_metadata)
parse_nodes(map, copy_map, data, root_nodes_start, root_nodes_len) parse_nodes(map, copy_map, data, root_nodes_start, root_nodes_len)
def parse_nodes(map, copy_map, data, start, len): def parse_nodes(map, copy_map, data, start, len):
"""parse <len> nodes from <data> starting at offset <start> """parse <len> nodes from <data> starting at offset <start>
This is used by parse_dirstate to recursively fill `map` and `copy_map`. This is used by parse_dirstate to recursively fill `map` and `copy_map`.
All directory specific information is ignored and do not need any
processing (HAS_DIRECTORY_MTIME, ALL_UNKNOWN_RECORDED, ALL_IGNORED_RECORDED)
""" """
for i in range(len): for i in range(len):
node_start = start + NODE_SIZE * i node_start = start + NODE_SIZE * i
node_bytes = slice_with_len(data, node_start, NODE_SIZE) node_bytes = slice_with_len(data, node_start, NODE_SIZE)
( (
path_start, path_start,
path_len, path_len,
_basename_start, _basename_start,

mercurial/helptext/internals/dirstate-v2.txt

P1_TRACKED = 1 << 1 P1_TRACKED = 1 << 1
P2_INFO = 1 << 2 P2_INFO = 1 << 2
HAS_MODE_AND_SIZE = 1 << 3 HAS_MODE_AND_SIZE = 1 << 3
HAS_FILE_MTIME = 1 << 4 HAS_FILE_MTIME = 1 << 4
HAS_DIRECTORY_MTIME = 1 << 5 HAS_DIRECTORY_MTIME = 1 << 5
MODE_EXEC_PERM = 1 << 6 MODE_EXEC_PERM = 1 << 6
MODE_IS_SYMLINK = 1 << 7 MODE_IS_SYMLINK = 1 << 7
EXPECTED_STATE_IS_MODIFIED = 1 << 8 EXPECTED_STATE_IS_MODIFIED = 1 << 8
ALL_UNKNOWN_RECORDED = 1 << 9
ALL_IGNORED_RECORDED = 1 << 10
The meaning of each bit is described below. The meaning of each bit is described below.
Other bits are unset. Other bits are unset.
They may be assigned meaning if the future, They may be assigned meaning if the future,
with the limitation that Mercurial versions that pre-date such meaning with the limitation that Mercurial versions that pre-date such meaning
will always reset those bits to unset when writing nodes. will always reset those bits to unset when writing nodes.
(A new node is written for any mutation in its subtree, (A new node is written for any mutation in its subtree,
If set, the status is modified. If unset, it is clean. If set, the status is modified. If unset, it is clean.
In cases where `hg status` needs to read the contents of a file In cases where `hg status` needs to read the contents of a file
because metadata is ambiguous, this bit lets it record the result because metadata is ambiguous, this bit lets it record the result
if the result is modified so that a future run of `hg status` if the result is modified so that a future run of `hg status`
does not need to do the same again. does not need to do the same again.
It is valid to never set this bit, It is valid to never set this bit,
and consider expected metadata ambiguous if it is set. and consider expected metadata ambiguous if it is set.
`ALL_UNKNOWN_RECORDED`
If set, all "unknown" children existing on disk (at the time of the last
status) have been recorded and the `mtime` associated with
`HAS_DIRECTORY_MTIME` can be used for optimization even when "unknown" file
are listed.
Note that the amount recorded "unknown" children can still be zero if None
where present.
Also note that having this flag unset does not imply that no "unknown"
children have been recorded. Some might be present, but there is no garantee
that is will be all of them.
`ALL_IGNORED_RECORDED`
If set, all "ignored" children existing on disk (at the time of the last
status) have been recorded and the `mtime` associated with
`HAS_DIRECTORY_MTIME` can be used for optimization even when "ignored" file
are listed.
Note that the amount recorded "ignored" children can still be zero if None
where present.
Also note that having this flag unset does not imply that no "ignored"
children have been recorded. Some might be present, but there is no garantee
that is will be all of them.

mercurial/pure/parsers.py

DIRSTATE_V2_P1_TRACKED = 1 << 1 DIRSTATE_V2_P1_TRACKED = 1 << 1
DIRSTATE_V2_P2_INFO = 1 << 2 DIRSTATE_V2_P2_INFO = 1 << 2
DIRSTATE_V2_HAS_MODE_AND_SIZE = 1 << 3 DIRSTATE_V2_HAS_MODE_AND_SIZE = 1 << 3
DIRSTATE_V2_HAS_FILE_MTIME = 1 << 4 DIRSTATE_V2_HAS_FILE_MTIME = 1 << 4
_DIRSTATE_V2_HAS_DIRCTORY_MTIME = 1 << 5 # Unused when Rust is not available _DIRSTATE_V2_HAS_DIRCTORY_MTIME = 1 << 5 # Unused when Rust is not available
DIRSTATE_V2_MODE_EXEC_PERM = 1 << 6 DIRSTATE_V2_MODE_EXEC_PERM = 1 << 6
DIRSTATE_V2_MODE_IS_SYMLINK = 1 << 7 DIRSTATE_V2_MODE_IS_SYMLINK = 1 << 7
DIRSTATE_V2_EXPECTED_STATE_IS_MODIFIED = 1 << 8 DIRSTATE_V2_EXPECTED_STATE_IS_MODIFIED = 1 << 8
DIRSTATE_V2_ALL_UNKNOWN_RECORDED = 1 << 9
DIRSTATE_V2_ALL_IGNORED_RECORDED = 1 << 10
@attr.s(slots=True, init=False) @attr.s(slots=True, init=False)
class DirstateItem(object): class DirstateItem(object):
"""represent a dirstate entry """represent a dirstate entry
It hold multiple attributes It hold multiple attributes
if self._mode is not None and self._size is not None: if self._mode is not None and self._size is not None:
flags |= DIRSTATE_V2_HAS_MODE_AND_SIZE flags |= DIRSTATE_V2_HAS_MODE_AND_SIZE
if self.mode & stat.S_IXUSR: if self.mode & stat.S_IXUSR:
flags |= DIRSTATE_V2_MODE_EXEC_PERM flags |= DIRSTATE_V2_MODE_EXEC_PERM
if stat.S_ISLNK(self.mode): if stat.S_ISLNK(self.mode):
flags |= DIRSTATE_V2_MODE_IS_SYMLINK flags |= DIRSTATE_V2_MODE_IS_SYMLINK
if self._mtime is not None: if self._mtime is not None:
flags |= DIRSTATE_V2_HAS_FILE_MTIME flags |= DIRSTATE_V2_HAS_FILE_MTIME
# Note: we do not need to do anything regarding
# DIRSTATE_V2_ALL_UNKNOWN_RECORDED and DIRSTATE_V2_ALL_IGNORED_RECORDED
# since we never set _DIRSTATE_V2_HAS_DIRCTORY_MTIME
return (flags, self._size or 0, self._mtime or 0) return (flags, self._size or 0, self._mtime or 0)
def v1_state(self): def v1_state(self):
"""return a "state" suitable for v1 serialization""" """return a "state" suitable for v1 serialization"""
if not self.any_tracked: if not self.any_tracked:
# the object has no state to record, this is -currently- # the object has no state to record, this is -currently-
# unsupported # unsupported
raise RuntimeError('untracked item') raise RuntimeError('untracked item')

rust/hg-core/src/dirstate_tree/on_disk.rs

const P1_TRACKED = 1 << 1; const P1_TRACKED = 1 << 1;
const P2_INFO = 1 << 2; const P2_INFO = 1 << 2;
const HAS_MODE_AND_SIZE = 1 << 3; const HAS_MODE_AND_SIZE = 1 << 3;
const HAS_FILE_MTIME = 1 << 4; const HAS_FILE_MTIME = 1 << 4;
const HAS_DIRECTORY_MTIME = 1 << 5; const HAS_DIRECTORY_MTIME = 1 << 5;
const MODE_EXEC_PERM = 1 << 6; const MODE_EXEC_PERM = 1 << 6;
const MODE_IS_SYMLINK = 1 << 7; const MODE_IS_SYMLINK = 1 << 7;
const EXPECTED_STATE_IS_MODIFIED = 1 << 8; const EXPECTED_STATE_IS_MODIFIED = 1 << 8;
const ALL_UNKNOWN_RECORDED = 1 << 9;
const ALL_IGNORED_RECORDED = 1 << 10;
} }
} }
/// Duration since the Unix epoch /// Duration since the Unix epoch
#[derive(BytesCast, Copy, Clone)] #[derive(BytesCast, Copy, Clone)]
#[repr(C)] #[repr(C)]
struct PackedTruncatedTimestamp { struct PackedTruncatedTimestamp {
truncated_seconds: U32Be, truncated_seconds: U32Be,
} else { } else {
Ok(dirstate_map::NodeData::None) Ok(dirstate_map::NodeData::None)
} }
} }
pub(super) fn cached_directory_mtime( pub(super) fn cached_directory_mtime(
&self, &self,
) -> Result<Option<TruncatedTimestamp>, DirstateV2ParseError> { ) -> Result<Option<TruncatedTimestamp>, DirstateV2ParseError> {
if self.flags().contains(Flags::HAS_DIRECTORY_MTIME) { // For now we do not have code to handle ALL_UNKNOWN_RECORDED, so we
// ignore the mtime if the flag is set.
if self.flags().contains(Flags::HAS_DIRECTORY_MTIME)
&& self.flags().contains(Flags::ALL_UNKNOWN_RECORDED)
{
if self.flags().contains(Flags::HAS_FILE_MTIME) { if self.flags().contains(Flags::HAS_FILE_MTIME) {
Err(DirstateV2ParseError) Err(DirstateV2ParseError)
} else { } else {
Ok(Some(self.mtime.try_into()?)) Ok(Some(self.mtime.try_into()?))
} }
} else { } else {
Ok(None) Ok(None)
} }
}; };
on_disk_nodes.push(match node { on_disk_nodes.push(match node {
NodeRef::InMemory(path, node) => { NodeRef::InMemory(path, node) => {
let (flags, size, mtime) = match &node.data { let (flags, size, mtime) = match &node.data {
dirstate_map::NodeData::Entry(entry) => { dirstate_map::NodeData::Entry(entry) => {
Node::from_dirstate_entry(entry) Node::from_dirstate_entry(entry)
} }
dirstate_map::NodeData::CachedDirectory { mtime } => ( dirstate_map::NodeData::CachedDirectory { mtime } => (
Flags::HAS_DIRECTORY_MTIME, // we currently never set a mtime if unknown file
// are present.
// So if we have a mtime for a directory, we know
// they are no unknown
// files and we
// blindly set ALL_UNKNOWN_RECORDED.
//
// We never set ALL_IGNORED_RECORDED since we
// don't track that case
// currently.
Flags::HAS_DIRECTORY_MTIME
| Flags::ALL_UNKNOWN_RECORDED,
0.into(), 0.into(),
(*mtime).into(), (*mtime).into(),
), ),
dirstate_map::NodeData::None => ( dirstate_map::NodeData::None => (
Flags::empty(), Flags::empty(),
0.into(), 0.into(),
PackedTruncatedTimestamp::null(), PackedTruncatedTimestamp::null(),
), ),