( )⚙ D11681 dirstate-v2: adds a flag to mark a file as modified

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D11681

dirstate-v2: adds a flag to mark a file as modified
ClosedPublic

Authored by marmoute on Oct 18 2021, 2:18 AM.

Details

Reviewers
Alphare
Group Reviewers
hg-reviewers
Commits
rHG1730b2fceaa1: dirstate-v2: adds a flag to mark a file as modified
Summary

Right now, a files with a file system state that requires a lookup (same size,
different mtime) will requires a lookup. If the result of that lookup is a
modified files, it will remains ambiguous, requiring a lookup on the next status
run too.

To fix this, we introduce a dedicated flag in the new format. Such flag will
allow to record such file as "known modified" avoiding an extra lookup later.

As None of the associate code currently exist in the status code, we do the
minimal implementation: if we read a dirstate entry with this flag set, we make
it as "ambiguous" so that the next status code has to look it up. The same as it
would have to without this flag existing anyway.

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Revision Contents
Changeset List

Diff 30848

mercurial/cext/parsers.c

static inline bool dirstate_item_c_tracked(dirstateItemObject *self) static inline bool dirstate_item_c_tracked(dirstateItemObject *self)
{ {
return (self->flags & dirstate_flag_wc_tracked); return (self->flags & dirstate_flag_wc_tracked);
} }
static inline bool dirstate_item_c_any_tracked(dirstateItemObject *self) static inline bool dirstate_item_c_any_tracked(dirstateItemObject *self)
{ {
const unsigned char mask = dirstate_flag_wc_tracked | const int mask = dirstate_flag_wc_tracked | dirstate_flag_p1_tracked |
dirstate_flag_p1_tracked |
dirstate_flag_p2_info; dirstate_flag_p2_info;
return (self->flags & mask); return (self->flags & mask);
} }
static inline bool dirstate_item_c_added(dirstateItemObject *self) static inline bool dirstate_item_c_added(dirstateItemObject *self)
{ {
const unsigned char mask = const int mask = (dirstate_flag_wc_tracked | dirstate_flag_p1_tracked |
(dirstate_flag_wc_tracked | dirstate_flag_p1_tracked |
dirstate_flag_p2_info); dirstate_flag_p2_info);
const unsigned char target = dirstate_flag_wc_tracked; const int target = dirstate_flag_wc_tracked;
return (self->flags & mask) == target; return (self->flags & mask) == target;
} }
static inline bool dirstate_item_c_removed(dirstateItemObject *self) static inline bool dirstate_item_c_removed(dirstateItemObject *self)
{ {
if (self->flags & dirstate_flag_wc_tracked) { if (self->flags & dirstate_flag_wc_tracked) {
return false; return false;
} }
return ambiguous_time; return ambiguous_time;
} else { } else {
return self->mtime; return self->mtime;
} }
} }
static PyObject *dirstate_item_v2_data(dirstateItemObject *self) static PyObject *dirstate_item_v2_data(dirstateItemObject *self)
{ {
unsigned char flags = self->flags; int flags = self->flags;
int mode = dirstate_item_c_v1_mode(self); int mode = dirstate_item_c_v1_mode(self);
if ((mode & S_IXUSR) != 0) { if ((mode & S_IXUSR) != 0) {
flags |= dirstate_flag_mode_exec_perm; flags |= dirstate_flag_mode_exec_perm;
} else { } else {
flags &= ~dirstate_flag_mode_exec_perm; flags &= ~dirstate_flag_mode_exec_perm;
} }
if (S_ISLNK(mode)) { if (S_ISLNK(mode)) {
flags |= dirstate_flag_mode_is_symlink; flags |= dirstate_flag_mode_is_symlink;
} else { } else {
flags &= ~dirstate_flag_mode_is_symlink; flags &= ~dirstate_flag_mode_is_symlink;
} }
return Py_BuildValue("Bii", flags, self->size, self->mtime); return Py_BuildValue("iii", flags, self->size, self->mtime);
}; };
static PyObject *dirstate_item_v1_state(dirstateItemObject *self) static PyObject *dirstate_item_v1_state(dirstateItemObject *self)
{ {
char state = dirstate_item_c_v1_state(self); char state = dirstate_item_c_v1_state(self);
return PyBytes_FromStringAndSize(&state, 1); return PyBytes_FromStringAndSize(&state, 1);
}; };
static PyObject *dirstate_item_from_v2_meth(PyTypeObject *subtype, static PyObject *dirstate_item_from_v2_meth(PyTypeObject *subtype,
PyObject *args) PyObject *args)
{ {
dirstateItemObject *t = dirstateItemObject *t =
PyObject_New(dirstateItemObject, &dirstateItemType); PyObject_New(dirstateItemObject, &dirstateItemType);
if (!t) { if (!t) {
return NULL; return NULL;
} }
if (!PyArg_ParseTuple(args, "bii", &t->flags, &t->size, &t->mtime)) { if (!PyArg_ParseTuple(args, "iii", &t->flags, &t->size, &t->mtime)) {
return NULL; return NULL;
} }
if (t->flags & dirstate_flag_expected_state_is_modified) {
t->flags &= ~(dirstate_flag_expected_state_is_modified |
dirstate_flag_has_meaningful_data |
dirstate_flag_has_file_mtime);
}
t->mode = 0; t->mode = 0;
if (t->flags & dirstate_flag_has_meaningful_data) { if (t->flags & dirstate_flag_has_meaningful_data) {
if (t->flags & dirstate_flag_mode_exec_perm) { if (t->flags & dirstate_flag_mode_exec_perm) {
t->mode = 0755; t->mode = 0755;
} else { } else {
t->mode = 0644; t->mode = 0644;
} }
if (t->flags & dirstate_flag_mode_is_symlink) { if (t->flags & dirstate_flag_mode_is_symlink) {

mercurial/cext/util.h

#define PY23(py2, py3) py3 #define PY23(py2, py3) py3
#else #else
#define PY23(py2, py3) py2 #define PY23(py2, py3) py2
#endif #endif
/* clang-format off */ /* clang-format off */
typedef struct { typedef struct {
PyObject_HEAD PyObject_HEAD
unsigned char flags; int flags;
int mode; int mode;
int size; int size;
int mtime; int mtime;
} dirstateItemObject; } dirstateItemObject;
/* clang-format on */ /* clang-format on */
static const unsigned char dirstate_flag_wc_tracked = 1; static const int dirstate_flag_wc_tracked = 1;
static const unsigned char dirstate_flag_p1_tracked = 1 << 1; static const int dirstate_flag_p1_tracked = 1 << 1;
static const unsigned char dirstate_flag_p2_info = 1 << 2; static const int dirstate_flag_p2_info = 1 << 2;
static const unsigned char dirstate_flag_has_meaningful_data = 1 << 3; static const int dirstate_flag_has_meaningful_data = 1 << 3;
static const unsigned char dirstate_flag_has_file_mtime = 1 << 4; static const int dirstate_flag_has_file_mtime = 1 << 4;
static const unsigned char dirstate_flag_has_directory_mtime = 1 << 5; static const int dirstate_flag_has_directory_mtime = 1 << 5;
static const unsigned char dirstate_flag_mode_exec_perm = 1 << 6; static const int dirstate_flag_mode_exec_perm = 1 << 6;
static const unsigned char dirstate_flag_mode_is_symlink = 1 << 7; static const int dirstate_flag_mode_is_symlink = 1 << 7;
static const int dirstate_flag_expected_state_is_modified = 1 << 8;
extern PyTypeObject dirstateItemType; extern PyTypeObject dirstateItemType;
#define dirstate_tuple_check(op) (Py_TYPE(op) == &dirstateItemType) #define dirstate_tuple_check(op) (Py_TYPE(op) == &dirstateItemType)
#ifndef MIN #ifndef MIN
#define MIN(a, b) (((a) < (b)) ? (a) : (b)) #define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif #endif
/* VC9 doesn't include bool and lacks stdbool.h based on my searching */ /* VC9 doesn't include bool and lacks stdbool.h based on my searching */

mercurial/helptext/internals/dirstate-v2.txt

WDIR_TRACKED = 1 << 0 WDIR_TRACKED = 1 << 0
P1_TRACKED = 1 << 1 P1_TRACKED = 1 << 1
P2_INFO = 1 << 2 P2_INFO = 1 << 2
HAS_MODE_AND_SIZE = 1 << 3 HAS_MODE_AND_SIZE = 1 << 3
HAS_FILE_MTIME = 1 << 4 HAS_FILE_MTIME = 1 << 4
HAS_DIRECTORY_MTIME = 1 << 5 HAS_DIRECTORY_MTIME = 1 << 5
MODE_EXEC_PERM = 1 << 6 MODE_EXEC_PERM = 1 << 6
MODE_IS_SYMLINK = 1 << 7 MODE_IS_SYMLINK = 1 << 7
EXPECTED_STATE_IS_MODIFIED = 1 << 8
The meaning of each bit is described below. The meaning of each bit is described below.
Other bits are unset. Other bits are unset.
They may be assigned meaning if the future, They may be assigned meaning if the future,
with the limitation that Mercurial versions that pre-date such meaning with the limitation that Mercurial versions that pre-date such meaning
will always reset those bits to unset when writing nodes. will always reset those bits to unset when writing nodes.
(A new node is written for any mutation in its subtree, (A new node is written for any mutation in its subtree,
add nodes for untracked files or/and ignored files in the working directory add nodes for untracked files or/and ignored files in the working directory
in order to optimize `hg status` in order to optimize `hg status`
by enabling it to skip `readdir` in more cases. by enabling it to skip `readdir` in more cases.
`HAS_MODE_AND_SIZE` `HAS_MODE_AND_SIZE`
Must be unset for untracked nodes. Must be unset for untracked nodes.
For files tracked anywhere, if this is set: For files tracked anywhere, if this is set:
- The `size` field is the expected file size, - The `size` field is the expected file size,
in bytes truncated its lower to 31 bits, in bytes truncated its lower to 31 bits.
for the file to be clean.
- The expected execute permission for the file’s owner - The expected execute permission for the file’s owner
is given by `MODE_EXEC_PERM` is given by `MODE_EXEC_PERM`
- The expected file type is given by `MODE_IS_SIMLINK`: - The expected file type is given by `MODE_IS_SIMLINK`:
a symbolic link if set, or a normal file if unset. a symbolic link if set, or a normal file if unset.
If this is unset the expected size, permission, and file type are unknown. If this is unset the expected size, permission, and file type are unknown.
The `size` field is unused (set to zero). The `size` field is unused (set to zero).
`HAS_FILE_MTIME` `HAS_FILE_MTIME`
Must be unset for untracked nodes. Must be unset for untracked nodes.
If this and `HAS_DIRECTORY_MTIME` are both unset, If this and `HAS_DIRECTORY_MTIME` are both unset,
the `mtime` field is unused (set to zero). the `mtime` field is unused (set to zero).
If this is set, `mtime` is the modification time If this is set, `mtime` is the expected modification time.
expected for the file to be considered clean.
`HAS_DIRECTORY_MTIME` `HAS_DIRECTORY_MTIME`
Must be unset for file tracked anywhere. Must be unset for file tracked anywhere.
If this and `HAS_DIRECTORY_MTIME` are both unset, If this and `HAS_DIRECTORY_MTIME` are both unset,
the `mtime` field is unused (set to zero). the `mtime` field is unused (set to zero).
If this is set, at some point, If this is set, at some point,
this path in the working directory was observed: this path in the working directory was observed:
this indicates whether the file’s own is expected this indicates whether the file’s own is expected
to have execute permission. to have execute permission.
`MODE_IS_SYMLINK` `MODE_IS_SYMLINK`
Must be unset if `HAS_MODE_AND_SIZE` is unset. Must be unset if `HAS_MODE_AND_SIZE` is unset.
If `HAS_MODE_AND_SIZE` is set, If `HAS_MODE_AND_SIZE` is set,
this indicates whether the file is expected to be a symlink this indicates whether the file is expected to be a symlink
as opposed to a normal file. as opposed to a normal file.
`EXPECTED_STATE_IS_MODIFIED`
Must be unset for untracked nodes.
For:
- a file tracked anywhere
- that has expected metadata (`HAS_MODE_AND_SIZE` and `HAS_FILE_MTIME`)
- if that metadata matches
metadata found in the working directory with `stat`
This bit indicates the status of the file.
If set, the status is modified. If unset, it is clean.
In cases where `hg status` needs to read the contents of a file
because metadata is ambiguous, this bit lets it record the result
if the result is modified so that a future run of `hg status`
does not need to do the same again.
It is valid to never set this bit,
and consider expected metadata ambiguous if it is set.

mercurial/pure/parsers.py

DIRSTATE_V2_WDIR_TRACKED = 1 << 0 DIRSTATE_V2_WDIR_TRACKED = 1 << 0
DIRSTATE_V2_P1_TRACKED = 1 << 1 DIRSTATE_V2_P1_TRACKED = 1 << 1
DIRSTATE_V2_P2_INFO = 1 << 2 DIRSTATE_V2_P2_INFO = 1 << 2
DIRSTATE_V2_HAS_MODE_AND_SIZE = 1 << 3 DIRSTATE_V2_HAS_MODE_AND_SIZE = 1 << 3
DIRSTATE_V2_HAS_FILE_MTIME = 1 << 4 DIRSTATE_V2_HAS_FILE_MTIME = 1 << 4
_DIRSTATE_V2_HAS_DIRCTORY_MTIME = 1 << 5 # Unused when Rust is not available _DIRSTATE_V2_HAS_DIRCTORY_MTIME = 1 << 5 # Unused when Rust is not available
DIRSTATE_V2_MODE_EXEC_PERM = 1 << 6 DIRSTATE_V2_MODE_EXEC_PERM = 1 << 6
DIRSTATE_V2_MODE_IS_SYMLINK = 1 << 7 DIRSTATE_V2_MODE_IS_SYMLINK = 1 << 7
DIRSTATE_V2_EXPECTED_STATE_IS_MODIFIED = 1 << 8
@attr.s(slots=True, init=False) @attr.s(slots=True, init=False)
class DirstateItem(object): class DirstateItem(object):
"""represent a dirstate entry """represent a dirstate entry
It hold multiple attributes It hold multiple attributes
self._size = parentfiledata[1] self._size = parentfiledata[1]
if has_meaningful_mtime: if has_meaningful_mtime:
self._mtime = parentfiledata[2] self._mtime = parentfiledata[2]
@classmethod @classmethod
def from_v2_data(cls, flags, size, mtime): def from_v2_data(cls, flags, size, mtime):
"""Build a new DirstateItem object from V2 data""" """Build a new DirstateItem object from V2 data"""
has_mode_size = bool(flags & DIRSTATE_V2_HAS_MODE_AND_SIZE) has_mode_size = bool(flags & DIRSTATE_V2_HAS_MODE_AND_SIZE)
has_meaningful_mtime = bool(flags & DIRSTATE_V2_HAS_FILE_MTIME)
mode = None mode = None
if flags & +DIRSTATE_V2_EXPECTED_STATE_IS_MODIFIED:
# we do not have support for this flag in the code yet,
# force a lookup for this file.
has_mode_size = False
has_meaningful_mtime = False
if has_mode_size: if has_mode_size:
assert stat.S_IXUSR == 0o100 assert stat.S_IXUSR == 0o100
if flags & DIRSTATE_V2_MODE_EXEC_PERM: if flags & DIRSTATE_V2_MODE_EXEC_PERM:
mode = 0o755 mode = 0o755
else: else:
mode = 0o644 mode = 0o644
if flags & DIRSTATE_V2_MODE_IS_SYMLINK: if flags & DIRSTATE_V2_MODE_IS_SYMLINK:
mode |= stat.S_IFLNK mode |= stat.S_IFLNK
else: else:
mode |= stat.S_IFREG mode |= stat.S_IFREG
return cls( return cls(
wc_tracked=bool(flags & DIRSTATE_V2_WDIR_TRACKED), wc_tracked=bool(flags & DIRSTATE_V2_WDIR_TRACKED),
p1_tracked=bool(flags & DIRSTATE_V2_P1_TRACKED), p1_tracked=bool(flags & DIRSTATE_V2_P1_TRACKED),
p2_info=bool(flags & DIRSTATE_V2_P2_INFO), p2_info=bool(flags & DIRSTATE_V2_P2_INFO),
has_meaningful_data=has_mode_size, has_meaningful_data=has_mode_size,
has_meaningful_mtime=bool(flags & DIRSTATE_V2_HAS_FILE_MTIME), has_meaningful_mtime=has_meaningful_mtime,
parentfiledata=(mode, size, mtime), parentfiledata=(mode, size, mtime),
) )
@classmethod @classmethod
def from_v1_data(cls, state, mode, size, mtime): def from_v1_data(cls, state, mode, size, mtime):
"""Build a new DirstateItem object from V1 data """Build a new DirstateItem object from V1 data
Since the dirstate-v1 format is frozen, the signature of this function Since the dirstate-v1 format is frozen, the signature of this function

rust/hg-core/src/dirstate_tree/on_disk.rs

const WDIR_TRACKED = 1 << 0; const WDIR_TRACKED = 1 << 0;
const P1_TRACKED = 1 << 1; const P1_TRACKED = 1 << 1;
const P2_INFO = 1 << 2; const P2_INFO = 1 << 2;
const HAS_MODE_AND_SIZE = 1 << 3; const HAS_MODE_AND_SIZE = 1 << 3;
const HAS_FILE_MTIME = 1 << 4; const HAS_FILE_MTIME = 1 << 4;
const HAS_DIRECTORY_MTIME = 1 << 5; const HAS_DIRECTORY_MTIME = 1 << 5;
const MODE_EXEC_PERM = 1 << 6; const MODE_EXEC_PERM = 1 << 6;
const MODE_IS_SYMLINK = 1 << 7; const MODE_IS_SYMLINK = 1 << 7;
const EXPECTED_STATE_IS_MODIFIED = 1 << 8;
} }
} }
/// Duration since the Unix epoch /// Duration since the Unix epoch
#[derive(BytesCast, Copy, Clone)] #[derive(BytesCast, Copy, Clone)]
#[repr(C)] #[repr(C)]
struct PackedTruncatedTimestamp { struct PackedTruncatedTimestamp {
truncated_seconds: U32Be, truncated_seconds: U32Be,
file_type | permisions file_type | permisions
} }
fn assume_entry(&self) -> DirstateEntry { fn assume_entry(&self) -> DirstateEntry {
// TODO: convert through raw bits instead? // TODO: convert through raw bits instead?
let wdir_tracked = self.flags().contains(Flags::WDIR_TRACKED); let wdir_tracked = self.flags().contains(Flags::WDIR_TRACKED);
let p1_tracked = self.flags().contains(Flags::P1_TRACKED); let p1_tracked = self.flags().contains(Flags::P1_TRACKED);
let p2_info = self.flags().contains(Flags::P2_INFO); let p2_info = self.flags().contains(Flags::P2_INFO);
let mode_size = if self.flags().contains(Flags::HAS_MODE_AND_SIZE) { let mode_size = if self.flags().contains(Flags::HAS_MODE_AND_SIZE)
&& !self.flags().contains(Flags::EXPECTED_STATE_IS_MODIFIED)
{
Some((self.synthesize_unix_mode(), self.size.into())) Some((self.synthesize_unix_mode(), self.size.into()))
} else { } else {
None None
}; };
let mtime = if self.flags().contains(Flags::HAS_FILE_MTIME) { let mtime = if self.flags().contains(Flags::HAS_FILE_MTIME)
&& !self.flags().contains(Flags::EXPECTED_STATE_IS_MODIFIED)
{
Some(self.mtime.truncated_seconds.into()) Some(self.mtime.truncated_seconds.into())
} else { } else {
None None
}; };
DirstateEntry::from_v2_data( DirstateEntry::from_v2_data(
wdir_tracked, wdir_tracked,
p1_tracked, p1_tracked,
p2_info, p2_info,