( )⚙ D11796 dirstate: remove need_delay logic

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D11796

dirstate: remove need_delay logic
ClosedPublic

Authored by marmoute on Nov 24 2021, 6:13 AM.

Details

Reviewers
Alphare
Group Reviewers
hg-reviewers
Commits
rHG434de12918fd: dirstate: remove need_delay logic
rHG2b5d161861da: dirstate: remove need_delay logic
Summary

Now that all¹ stored mtime are non ambiguous, we no longer need to apply the need_delay step.
The need delay logic was not great are mtime gathered during longer operation
could be ambiguous but younger than the dirstate.write call time.

So, we don't need that logic anymore and can drop it

This make the code much simpler. The code related to the test extension faking
the dirstate write is now obsolete and associated test will be migrated as
follow up. They currently do not break.

[1] except the ones from hg update, but need_delay no longer help for them
either.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

marmoute created this revision.Nov 24 2021, 6:13 AM
Alphare accepted this revision.Nov 24 2021, 8:51 AM
Alphare added a subscriber: Alphare.
Alphare added inline comments.
tests/test-merge1.t
352

"scénarion" is very fancy

This revision is now accepted and ready to land.Nov 24 2021, 8:51 AM
marmoute edited the summary of this revision. (Show Details)Nov 30 2021, 6:54 PM
marmoute updated this revision to Diff 31231.
marmoute updated this revision to Diff 31237.Dec 1 2021, 8:22 AM
Alphare added a commit: rHG2b5d161861da: dirstate: remove need_delay logic.Dec 1 2021, 10:24 AM
Closed by commit rHG2b5d161861da: dirstate: remove need_delay logic (authored by Alphare). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 31258

mercurial/cext/parsers.c

return PyInt_FromLong(dirstate_item_c_v1_size(self)); return PyInt_FromLong(dirstate_item_c_v1_size(self));
}; };
static PyObject *dirstate_item_v1_mtime(dirstateItemObject *self) static PyObject *dirstate_item_v1_mtime(dirstateItemObject *self)
{ {
return PyInt_FromLong(dirstate_item_c_v1_mtime(self)); return PyInt_FromLong(dirstate_item_c_v1_mtime(self));
}; };
static PyObject *dirstate_item_need_delay(dirstateItemObject *self,
PyObject *now)
{
int now_s;
int now_ns;
if (!PyArg_ParseTuple(now, "ii", &now_s, &now_ns)) {
return NULL;
}
if (dirstate_item_c_v1_state(self) == 'n' && self->mtime_s == now_s) {
Py_RETURN_TRUE;
} else {
Py_RETURN_FALSE;
}
};
static PyObject *dirstate_item_mtime_likely_equal_to(dirstateItemObject *self, static PyObject *dirstate_item_mtime_likely_equal_to(dirstateItemObject *self,
PyObject *other) PyObject *other)
{ {
int other_s; int other_s;
int other_ns; int other_ns;
if (!PyArg_ParseTuple(other, "ii", &other_s, &other_ns)) { if (!PyArg_ParseTuple(other, "ii", &other_s, &other_ns)) {
return NULL; return NULL;
} }
{"v1_state", (PyCFunction)dirstate_item_v1_state, METH_NOARGS, {"v1_state", (PyCFunction)dirstate_item_v1_state, METH_NOARGS,
"return a \"state\" suitable for v1 serialization"}, "return a \"state\" suitable for v1 serialization"},
{"v1_mode", (PyCFunction)dirstate_item_v1_mode, METH_NOARGS, {"v1_mode", (PyCFunction)dirstate_item_v1_mode, METH_NOARGS,
"return a \"mode\" suitable for v1 serialization"}, "return a \"mode\" suitable for v1 serialization"},
{"v1_size", (PyCFunction)dirstate_item_v1_size, METH_NOARGS, {"v1_size", (PyCFunction)dirstate_item_v1_size, METH_NOARGS,
"return a \"size\" suitable for v1 serialization"}, "return a \"size\" suitable for v1 serialization"},
{"v1_mtime", (PyCFunction)dirstate_item_v1_mtime, METH_NOARGS, {"v1_mtime", (PyCFunction)dirstate_item_v1_mtime, METH_NOARGS,
"return a \"mtime\" suitable for v1 serialization"}, "return a \"mtime\" suitable for v1 serialization"},
{"need_delay", (PyCFunction)dirstate_item_need_delay, METH_O,
"True if the stored mtime would be ambiguous with the current time"},
{"mtime_likely_equal_to", (PyCFunction)dirstate_item_mtime_likely_equal_to, {"mtime_likely_equal_to", (PyCFunction)dirstate_item_mtime_likely_equal_to,
METH_O, "True if the stored mtime is likely equal to the given mtime"}, METH_O, "True if the stored mtime is likely equal to the given mtime"},
{"from_v1_data", (PyCFunction)dirstate_item_from_v1_meth, {"from_v1_data", (PyCFunction)dirstate_item_from_v1_meth,
METH_VARARGS | METH_CLASS, "build a new DirstateItem object from V1 data"}, METH_VARARGS | METH_CLASS, "build a new DirstateItem object from V1 data"},
{"from_v2_data", (PyCFunction)dirstate_item_from_v2_meth, {"from_v2_data", (PyCFunction)dirstate_item_from_v2_meth,
METH_VARARGS | METH_CLASS, "build a new DirstateItem object from V2 data"}, METH_VARARGS | METH_CLASS, "build a new DirstateItem object from V2 data"},
{"set_possibly_dirty", (PyCFunction)dirstate_item_set_possibly_dirty, {"set_possibly_dirty", (PyCFunction)dirstate_item_set_possibly_dirty,
METH_NOARGS, "mark a file as \"possibly dirty\""}, METH_NOARGS, "mark a file as \"possibly dirty\""},
*/ */
static PyObject *pack_dirstate(PyObject *self, PyObject *args) static PyObject *pack_dirstate(PyObject *self, PyObject *args)
{ {
PyObject *packobj = NULL; PyObject *packobj = NULL;
PyObject *map, *copymap, *pl, *mtime_unset = NULL; PyObject *map, *copymap, *pl, *mtime_unset = NULL;
Py_ssize_t nbytes, pos, l; Py_ssize_t nbytes, pos, l;
PyObject *k, *v = NULL, *pn; PyObject *k, *v = NULL, *pn;
char *p, *s; char *p, *s;
int now_s;
int now_ns;
if (!PyArg_ParseTuple(args, "O!O!O!(ii):pack_dirstate", &PyDict_Type, if (!PyArg_ParseTuple(args, "O!O!O!:pack_dirstate", &PyDict_Type, &map,
&map, &PyDict_Type, &copymap, &PyTuple_Type, &pl, &PyDict_Type, &copymap, &PyTuple_Type, &pl)) {
&now_s, &now_ns)) {
return NULL; return NULL;
} }
if (PyTuple_Size(pl) != 2) { if (PyTuple_Size(pl) != 2) {
PyErr_SetString(PyExc_TypeError, "expected 2-element tuple"); PyErr_SetString(PyExc_TypeError, "expected 2-element tuple");
return NULL; return NULL;
} }
goto bail; goto bail;
} }
tuple = (dirstateItemObject *)v; tuple = (dirstateItemObject *)v;
state = dirstate_item_c_v1_state(tuple); state = dirstate_item_c_v1_state(tuple);
mode = dirstate_item_c_v1_mode(tuple); mode = dirstate_item_c_v1_mode(tuple);
size = dirstate_item_c_v1_size(tuple); size = dirstate_item_c_v1_size(tuple);
mtime = dirstate_item_c_v1_mtime(tuple); mtime = dirstate_item_c_v1_mtime(tuple);
if (state == 'n' && tuple->mtime_s == now_s) {
/* See pure/parsers.py:pack_dirstate for why we do
* this. */
mtime = -1;
mtime_unset = (PyObject *)dirstate_item_from_v1_data(
state, mode, size, mtime);
if (!mtime_unset) {
goto bail;
}
if (PyDict_SetItem(map, k, mtime_unset) == -1) {
goto bail;
}
Py_DECREF(mtime_unset);
mtime_unset = NULL;
}
*p++ = state; *p++ = state;
putbe32((uint32_t)mode, p); putbe32((uint32_t)mode, p);
putbe32((uint32_t)size, p + 4); putbe32((uint32_t)size, p + 4);
putbe32((uint32_t)mtime, p + 8); putbe32((uint32_t)mtime, p + 8);
t = p + 12; t = p + 12;
p += 16; p += 16;
len = PyBytes_GET_SIZE(k); len = PyBytes_GET_SIZE(k);
memcpy(p, PyBytes_AS_STRING(k), len); memcpy(p, PyBytes_AS_STRING(k), len);

mercurial/dirstate.py

callback(self, self._origpl, self._pl) callback(self, self._origpl, self._pl)
self._origpl = None self._origpl = None
if now is None: if now is None:
# use the modification time of the newly created temporary file as the # use the modification time of the newly created temporary file as the
# filesystem's notion of 'now' # filesystem's notion of 'now'
now = timestamp.mtime_of(util.fstat(st)) now = timestamp.mtime_of(util.fstat(st))
# enough 'delaywrite' prevents 'pack_dirstate' from dropping
# timestamp of each entries in dirstate, because of 'now > mtime'
delaywrite = self._ui.configint(b'debug', b'dirstate.delaywrite')
if delaywrite > 0:
# do we have any files to delay for?
for f, e in pycompat.iteritems(self._map):
if e.need_delay(now):
import time # to avoid useless import
# rather than sleep n seconds, sleep until the next
# multiple of n seconds
clock = time.time()
start = int(clock) - (int(clock) % delaywrite)
end = start + delaywrite
time.sleep(end - clock)
# trust our estimate that the end is near now
now = timestamp.timestamp((end, 0))
break
self._map.write(tr, st, now) self._map.write(tr, st, now)
self._dirty = False self._dirty = False
def _dirignore(self, f): def _dirignore(self, f):
if self._ignore(f): if self._ignore(f):
return True return True
for p in pathutil.finddirs(f): for p in pathutil.finddirs(f):
if self._ignore(p): if self._ignore(p):

mercurial/dirstatemap.py

# Avoid excess attribute lookups by fast pathing certain checks # Avoid excess attribute lookups by fast pathing certain checks
self.__contains__ = self._map.__contains__ self.__contains__ = self._map.__contains__
self.__getitem__ = self._map.__getitem__ self.__getitem__ = self._map.__getitem__
self.get = self._map.get self.get = self._map.get
def write(self, tr, st, now): def write(self, tr, st, now):
if self._use_dirstate_v2: if self._use_dirstate_v2:
packed, meta = v2.pack_dirstate(self._map, self.copymap, now) packed, meta = v2.pack_dirstate(self._map, self.copymap)
self.write_v2_no_append(tr, st, meta, packed) self.write_v2_no_append(tr, st, meta, packed)
else: else:
packed = parsers.pack_dirstate( packed = parsers.pack_dirstate(
self._map, self.copymap, self.parents(), now self._map, self.copymap, self.parents()
) )
st.write(packed) st.write(packed)
st.close() st.close()
self._dirtyparents = False self._dirtyparents = False
@propertycache @propertycache
def identity(self): def identity(self):
self._map self._map
@propertycache @propertycache
def identity(self): def identity(self):
self._map self._map
return self.identity return self.identity
def write(self, tr, st, now): def write(self, tr, st, now):
if not self._use_dirstate_v2: if not self._use_dirstate_v2:
p1, p2 = self.parents() p1, p2 = self.parents()
packed = self._map.write_v1(p1, p2, now) packed = self._map.write_v1(p1, p2)
st.write(packed) st.write(packed)
st.close() st.close()
self._dirtyparents = False self._dirtyparents = False
return return
# We can only append to an existing data file if there is one # We can only append to an existing data file if there is one
can_append = self.docket.uuid is not None can_append = self.docket.uuid is not None
packed, meta, append = self._map.write_v2(now, can_append) packed, meta, append = self._map.write_v2(can_append)
if append: if append:
docket = self.docket docket = self.docket
data_filename = docket.data_filename() data_filename = docket.data_filename()
if tr: if tr:
tr.add(data_filename, docket.data_size) tr.add(data_filename, docket.data_size)
with self._opener(data_filename, b'r+b') as fp: with self._opener(data_filename, b'r+b') as fp:
fp.seek(docket.data_size) fp.seek(docket.data_size)
assert fp.tell() == docket.data_size assert fp.tell() == docket.data_size

mercurial/dirstateutils/v2.py

self.tracked_descendants, self.tracked_descendants,
flags, flags,
size, size,
mtime_s, mtime_s,
mtime_ns, mtime_ns,
) )
def pack_dirstate(map, copy_map, now): def pack_dirstate(map, copy_map):
""" """
Pack `map` and `copy_map` into the dirstate v2 binary format and return Pack `map` and `copy_map` into the dirstate v2 binary format and return
the bytearray. the bytearray.
`now` is a timestamp of the current filesystem time used to detect race
conditions in writing the dirstate to disk, see inline comment.
The on-disk format expects a tree-like structure where the leaves are The on-disk format expects a tree-like structure where the leaves are
written first (and sorted per-directory), going up levels until the root written first (and sorted per-directory), going up levels until the root
node and writing that one to the docket. See more details on the on-disk node and writing that one to the docket. See more details on the on-disk
format in `mercurial/helptext/internals/dirstate-v2`. format in `mercurial/helptext/internals/dirstate-v2`.
Since both `map` and `copy_map` are flat dicts we need to figure out the Since both `map` and `copy_map` are flat dicts we need to figure out the
hierarchy. This algorithm does so without having to build the entire tree hierarchy. This algorithm does so without having to build the entire tree
# Use a stack to not have to only remember the nodes we currently need # Use a stack to not have to only remember the nodes we currently need
# instead of building the entire tree in memory # instead of building the entire tree in memory
stack = [] stack = []
current_node = Node(b"", None) current_node = Node(b"", None)
stack.append(current_node) stack.append(current_node)
for index, (path, entry) in enumerate(sorted_map, 1): for index, (path, entry) in enumerate(sorted_map, 1):
if entry.need_delay(now):
# The file was last modified "simultaneously" with the current
# write to dirstate (i.e. within the same second for file-
# systems with a granularity of 1 sec). This commonly happens
# for at least a couple of files on 'update'.
# The user could change the file without changing its size
# within the same second. Invalidate the file's mtime in
# dirstate, forcing future 'status' calls to compare the
# contents of the file if the size is the same. This prevents
# mistakenly treating such files as clean.
entry.set_possibly_dirty()
nodes_with_entry_count += 1 nodes_with_entry_count += 1
if path in copy_map: if path in copy_map:
nodes_with_copy_source_count += 1 nodes_with_copy_source_count += 1
current_folder = get_folder(path) current_folder = get_folder(path)
current_node = move_to_correct_node_in_tree( current_node = move_to_correct_node_in_tree(
current_folder, current_node, stack current_folder, current_node, stack
) )

mercurial/pure/parsers.py

return AMBIGUOUS_TIME return AMBIGUOUS_TIME
elif self._p2_info: elif self._p2_info:
return AMBIGUOUS_TIME return AMBIGUOUS_TIME
elif not self._p1_tracked: elif not self._p1_tracked:
return AMBIGUOUS_TIME return AMBIGUOUS_TIME
else: else:
return self._mtime_s return self._mtime_s
def need_delay(self, now):
"""True if the stored mtime would be ambiguous with the current time"""
return self.v1_state() == b'n' and self._mtime_s == now[0]
def gettype(q): def gettype(q):
return int(q & 0xFFFF) return int(q & 0xFFFF)
class BaseIndexObject(object): class BaseIndexObject(object):
# Can I be passed to an algorithme implemented in Rust ? # Can I be passed to an algorithme implemented in Rust ?
rust_ext_compat = 0 rust_ext_compat = 0
f = st[pos2:pos1] f = st[pos2:pos1]
if b'\0' in f: if b'\0' in f:
f, c = f.split(b'\0') f, c = f.split(b'\0')
copymap[f] = c copymap[f] = c
dmap[f] = DirstateItem.from_v1_data(*e[:4]) dmap[f] = DirstateItem.from_v1_data(*e[:4])
return parents return parents
def pack_dirstate(dmap, copymap, pl, now): def pack_dirstate(dmap, copymap, pl):
cs = stringio() cs = stringio()
write = cs.write write = cs.write
write(b"".join(pl)) write(b"".join(pl))
for f, e in pycompat.iteritems(dmap): for f, e in pycompat.iteritems(dmap):
if e.need_delay(now):
# The file was last modified "simultaneously" with the current
# write to dirstate (i.e. within the same second for file-
# systems with a granularity of 1 sec). This commonly happens
# for at least a couple of files on 'update'.
# The user could change the file without changing its size
# within the same second. Invalidate the file's mtime in
# dirstate, forcing future 'status' calls to compare the
# contents of the file if the size is the same. This prevents
# mistakenly treating such files as clean.
e.set_possibly_dirty()
if f in copymap: if f in copymap:
f = b"%s\0%s" % (f, copymap[f]) f = b"%s\0%s" % (f, copymap[f])
e = _pack( e = _pack(
b">cllll", b">cllll",
e.v1_state(), e.v1_state(),
e.v1_mode(), e.v1_mode(),
e.v1_size(), e.v1_size(),
e.v1_mtime(), e.v1_mtime(),
len(f), len(f),
) )
write(e) write(e)
write(f) write(f)
return cs.getvalue() return cs.getvalue()

rust/hg-core/src/dirstate/entry.rs

dirstate_exec_bit != fs_exec_bit dirstate_exec_bit != fs_exec_bit
} }
/// Returns a `(state, mode, size, mtime)` tuple as for /// Returns a `(state, mode, size, mtime)` tuple as for
/// `DirstateMapMethods::debug_iter`. /// `DirstateMapMethods::debug_iter`.
pub fn debug_tuple(&self) -> (u8, i32, i32, i32) { pub fn debug_tuple(&self) -> (u8, i32, i32, i32) {
(self.state().into(), self.mode(), self.size(), self.mtime()) (self.state().into(), self.mode(), self.size(), self.mtime())
} }
/// True if the stored mtime would be ambiguous with the current time
pub fn need_delay(&self, now: TruncatedTimestamp) -> bool {
if let Some(mtime) = self.mtime {
self.state() == EntryState::Normal
&& mtime.truncated_seconds() == now.truncated_seconds()
} else {
false
}
}
} }
impl EntryState { impl EntryState {
pub fn is_tracked(self) -> bool { pub fn is_tracked(self) -> bool {
use EntryState::*; use EntryState::*;
match self { match self {
Normal | Added | Merged => true, Normal | Added | Merged => true,
Removed => false, Removed => false,

rust/hg-core/src/dirstate_tree/dirstate_map.rs

Some(Ok(child_node)) Some(Ok(child_node))
} else { } else {
// Reached the bottom of the stack, we’re done // Reached the bottom of the stack, we’re done
None None
} }
}) })
} }
fn clear_known_ambiguous_mtimes(
&mut self,
paths: &[impl AsRef<HgPath>],
) -> Result<(), DirstateV2ParseError> {
for path in paths {
if let Some(node) = Self::get_node_mut(
self.on_disk,
&mut self.unreachable_bytes,
&mut self.root,
path.as_ref(),
)? {
if let NodeData::Entry(entry) = &mut node.data {
entry.set_possibly_dirty();
}
}
}
Ok(())
}
fn count_dropped_path(unreachable_bytes: &mut u32, path: &Cow<HgPath>) { fn count_dropped_path(unreachable_bytes: &mut u32, path: &Cow<HgPath>) {
if let Cow::Borrowed(path) = path { if let Cow::Borrowed(path) = path {
*unreachable_bytes += path.len() as u32 *unreachable_bytes += path.len() as u32
} }
} }
} }
/// Like `Iterator::filter_map`, but over a fallible iterator of `Result`s. /// Like `Iterator::filter_map`, but over a fallible iterator of `Result`s.
Ok(false) Ok(false)
} }
} }
#[timed] #[timed]
pub fn pack_v1( pub fn pack_v1(
&mut self, &mut self,
parents: DirstateParents, parents: DirstateParents,
now: TruncatedTimestamp,
) -> Result<Vec<u8>, DirstateError> { ) -> Result<Vec<u8>, DirstateError> {
let map = self.get_map_mut(); let map = self.get_map_mut();
let mut ambiguous_mtimes = Vec::new();
// Optizimation (to be measured?): pre-compute size to avoid `Vec` // Optizimation (to be measured?): pre-compute size to avoid `Vec`
// reallocations // reallocations
let mut size = parents.as_bytes().len(); let mut size = parents.as_bytes().len();
for node in map.iter_nodes() { for node in map.iter_nodes() {
let node = node?; let node = node?;
if let Some(entry) = node.entry()? { if node.entry()?.is_some() {
size += packed_entry_size( size += packed_entry_size(
node.full_path(map.on_disk)?, node.full_path(map.on_disk)?,
node.copy_source(map.on_disk)?, node.copy_source(map.on_disk)?,
); );
if entry.need_delay(now) {
ambiguous_mtimes.push(
node.full_path_borrowed(map.on_disk)?
.detach_from_tree(),
)
}
} }
} }
map.clear_known_ambiguous_mtimes(&ambiguous_mtimes)?;
let mut packed = Vec::with_capacity(size); let mut packed = Vec::with_capacity(size);
packed.extend(parents.as_bytes()); packed.extend(parents.as_bytes());
for node in map.iter_nodes() { for node in map.iter_nodes() {
let node = node?; let node = node?;
if let Some(entry) = node.entry()? { if let Some(entry) = node.entry()? {
pack_entry( pack_entry(
/// Returns new data and metadata together with whether that data should be /// Returns new data and metadata together with whether that data should be
/// appended to the existing data file whose content is at /// appended to the existing data file whose content is at
/// `map.on_disk` (true), instead of written to a new data file /// `map.on_disk` (true), instead of written to a new data file
/// (false). /// (false).
#[timed] #[timed]
pub fn pack_v2( pub fn pack_v2(
&mut self, &mut self,
now: TruncatedTimestamp,
can_append: bool, can_append: bool,
) -> Result<(Vec<u8>, Vec<u8>, bool), DirstateError> { ) -> Result<(Vec<u8>, Vec<u8>, bool), DirstateError> {
let map = self.get_map_mut(); let map = self.get_map_mut();
let mut paths = Vec::new();
for node in map.iter_nodes() {
let node = node?;
if let Some(entry) = node.entry()? {
if entry.need_delay(now) {
paths.push(
node.full_path_borrowed(map.on_disk)?
.detach_from_tree(),
)
}
}
}
// Borrow of `self` ends here since we collect cloned paths
map.clear_known_ambiguous_mtimes(&paths)?;
on_disk::write(map, can_append) on_disk::write(map, can_append)
} }
pub fn status<'a>( pub fn status<'a>(
&'a mut self, &'a mut self,
matcher: &'a (dyn Matcher + Sync), matcher: &'a (dyn Matcher + Sync),
root_dir: PathBuf, root_dir: PathBuf,
ignore_files: Vec<PathBuf>, ignore_files: Vec<PathBuf>,

rust/hg-cpython/src/dirstate/dirstate_map.rs

use cpython::{ use cpython::{
exc, PyBool, PyBytes, PyClone, PyDict, PyErr, PyList, PyNone, PyObject, exc, PyBool, PyBytes, PyClone, PyDict, PyErr, PyList, PyNone, PyObject,
PyResult, Python, PythonObject, ToPyObject, UnsafePyLeaked, PyResult, Python, PythonObject, ToPyObject, UnsafePyLeaked,
}; };
use crate::{ use crate::{
dirstate::copymap::{CopyMap, CopyMapItemsIterator, CopyMapKeysIterator}, dirstate::copymap::{CopyMap, CopyMapItemsIterator, CopyMapKeysIterator},
dirstate::item::{timestamp, DirstateItem}, dirstate::item::DirstateItem,
pybytes_deref::PyBytesDeref, pybytes_deref::PyBytesDeref,
}; };
use hg::{ use hg::{
dirstate::StateMapIter, dirstate::StateMapIter,
dirstate_tree::dirstate_map::DirstateMap as TreeDirstateMap, dirstate_tree::dirstate_map::DirstateMap as TreeDirstateMap,
dirstate_tree::on_disk::DirstateV2ParseError, dirstate_tree::on_disk::DirstateV2ParseError,
dirstate_tree::owning::OwningDirstateMap, dirstate_tree::owning::OwningDirstateMap,
revlog::Node, revlog::Node,
})? })?
.to_py_object(py)) .to_py_object(py))
} }
def write_v1( def write_v1(
&self, &self,
p1: PyObject, p1: PyObject,
p2: PyObject, p2: PyObject,
now: (u32, u32)
) -> PyResult<PyBytes> { ) -> PyResult<PyBytes> {
let now = timestamp(py, now)?;
let mut inner = self.inner(py).borrow_mut(); let mut inner = self.inner(py).borrow_mut();
let parents = DirstateParents { let parents = DirstateParents {
p1: extract_node_id(py, &p1)?, p1: extract_node_id(py, &p1)?,
p2: extract_node_id(py, &p2)?, p2: extract_node_id(py, &p2)?,
}; };
let result = inner.pack_v1(parents, now); let result = inner.pack_v1(parents);
match result { match result {
Ok(packed) => Ok(PyBytes::new(py, &packed)), Ok(packed) => Ok(PyBytes::new(py, &packed)),
Err(_) => Err(PyErr::new::<exc::OSError, _>( Err(_) => Err(PyErr::new::<exc::OSError, _>(
py, py,
"Dirstate error".to_string(), "Dirstate error".to_string(),
)), )),
} }
} }
/// Returns new data together with whether that data should be appended to /// Returns new data together with whether that data should be appended to
/// the existing data file whose content is at `self.on_disk` (True), /// the existing data file whose content is at `self.on_disk` (True),
/// instead of written to a new data file (False). /// instead of written to a new data file (False).
def write_v2( def write_v2(
&self, &self,
now: (u32, u32),
can_append: bool, can_append: bool,
) -> PyResult<PyObject> { ) -> PyResult<PyObject> {
let now = timestamp(py, now)?;
let mut inner = self.inner(py).borrow_mut(); let mut inner = self.inner(py).borrow_mut();
let result = inner.pack_v2(now, can_append); let result = inner.pack_v2(can_append);
match result { match result {
Ok((packed, tree_metadata, append)) => { Ok((packed, tree_metadata, append)) => {
let packed = PyBytes::new(py, &packed); let packed = PyBytes::new(py, &packed);
let tree_metadata = PyBytes::new(py, &tree_metadata); let tree_metadata = PyBytes::new(py, &tree_metadata);
let tuple = (packed, tree_metadata, append); let tuple = (packed, tree_metadata, append);
Ok(tuple.to_py_object(py).into_object()) Ok(tuple.to_py_object(py).into_object())
}, },
Err(_) => Err(PyErr::new::<exc::OSError, _>( Err(_) => Err(PyErr::new::<exc::OSError, _>(

rust/hg-cpython/src/dirstate/item.rs

Ok(size) Ok(size)
} }
def v1_mtime(&self) -> PyResult<i32> { def v1_mtime(&self) -> PyResult<i32> {
let (_state, _mode, _size, mtime) = self.entry(py).get().v1_data(); let (_state, _mode, _size, mtime) = self.entry(py).get().v1_data();
Ok(mtime) Ok(mtime)
} }
def need_delay(&self, now: (u32, u32)) -> PyResult<bool> {
let now = timestamp(py, now)?;
Ok(self.entry(py).get().need_delay(now))
}
def mtime_likely_equal_to(&self, other: (u32, u32)) -> PyResult<bool> { def mtime_likely_equal_to(&self, other: (u32, u32)) -> PyResult<bool> {
if let Some(mtime) = self.entry(py).get().truncated_mtime() { if let Some(mtime) = self.entry(py).get().truncated_mtime() {
Ok(mtime.likely_equal(timestamp(py, other)?)) Ok(mtime.likely_equal(timestamp(py, other)?))
} else { } else {
Ok(false) Ok(false)
} }
} }

tests/fakedirstatewritetime.py

b'fakenow', b'fakenow',
default=None, default=None,
) )
parsers = policy.importmod('parsers') parsers = policy.importmod('parsers')
has_rust_dirstate = policy.importrust('dirstate') is not None has_rust_dirstate = policy.importrust('dirstate') is not None
def pack_dirstate(fakenow, orig, dmap, copymap, pl, now): def pack_dirstate(orig, dmap, copymap, pl):
# execute what original parsers.pack_dirstate should do actually return orig(dmap, copymap, pl)
# for consistency
for f, e in dmap.items():
if e.need_delay(now):
e.set_possibly_dirty()
return orig(dmap, copymap, pl, fakenow)
def fakewrite(ui, func): def fakewrite(ui, func):
# fake "now" of 'pack_dirstate' only if it is invoked while 'func' # fake "now" of 'pack_dirstate' only if it is invoked while 'func'
fakenow = ui.config(b'fakedirstatewritetime', b'fakenow') fakenow = ui.config(b'fakedirstatewritetime', b'fakenow')
if not fakenow: if not fakenow:
# Execute original one, if fakenow isn't configured. This is # Execute original one, if fakenow isn't configured. This is
# useful to prevent subrepos from executing replaced one, # useful to prevent subrepos from executing replaced one,
# because replacing 'parsers.pack_dirstate' is also effective # because replacing 'parsers.pack_dirstate' is also effective
# in subrepos. # in subrepos.
return func() return func()
# parsing 'fakenow' in YYYYmmddHHMM format makes comparison between # parsing 'fakenow' in YYYYmmddHHMM format makes comparison between
# 'fakenow' value and 'touch -t YYYYmmddHHMM' argument easy # 'fakenow' value and 'touch -t YYYYmmddHHMM' argument easy
fakenow = dateutil.parsedate(fakenow, [b'%Y%m%d%H%M'])[0] fakenow = dateutil.parsedate(fakenow, [b'%Y%m%d%H%M'])[0]
fakenow = timestamp.timestamp((fakenow, 0)) fakenow = timestamp.timestamp((fakenow, 0))
if has_rust_dirstate: if has_rust_dirstate:
# The Rust implementation does not use public parse/pack dirstate # The Rust implementation does not use public parse/pack dirstate
# to prevent conversion round-trips # to prevent conversion round-trips
orig_dirstatemap_write = dirstatemapmod.dirstatemap.write orig_dirstatemap_write = dirstatemapmod.dirstatemap.write
wrapper = lambda self, tr, st, now: orig_dirstatemap_write( wrapper = lambda self, tr, st: orig_dirstatemap_write(self, tr, st)
self, tr, st, fakenow
)
dirstatemapmod.dirstatemap.write = wrapper dirstatemapmod.dirstatemap.write = wrapper
orig_get_fs_now = timestamp.get_fs_now orig_get_fs_now = timestamp.get_fs_now
wrapper = lambda *args: pack_dirstate(fakenow, orig_pack_dirstate, *args) wrapper = lambda *args: pack_dirstate(orig_pack_dirstate, *args)
orig_module = parsers orig_module = parsers
orig_pack_dirstate = parsers.pack_dirstate orig_pack_dirstate = parsers.pack_dirstate
orig_module.pack_dirstate = wrapper orig_module.pack_dirstate = wrapper
timestamp.get_fs_now = lambda *args: fakenow timestamp.get_fs_now = (
lambda *args: fakenow
) # XXX useless for this purpose now
try: try:
return func() return func()
finally: finally:
orig_module.pack_dirstate = orig_pack_dirstate orig_module.pack_dirstate = orig_pack_dirstate
timestamp.get_fs_now = orig_get_fs_now timestamp.get_fs_now = orig_get_fs_now
if has_rust_dirstate: if has_rust_dirstate:
dirstatemapmod.dirstatemap.write = orig_dirstatemap_write dirstatemapmod.dirstatemap.write = orig_dirstatemap_write

tests/test-merge1.t

$ hg revert -r -2 b $ hg revert -r -2 b
$ hg up -q -- -2 $ hg up -q -- -2
Test that updated files are treated as "modified", when Test that updated files are treated as "modified", when
'merge.update()' is aborted before 'merge.recordupdates()' (= parents 'merge.update()' is aborted before 'merge.recordupdates()' (= parents
aren't changed), even if none of mode, size and timestamp of them aren't changed), even if none of mode, size and timestamp of them
isn't changed on the filesystem (see also issue4583). isn't changed on the filesystem (see also issue4583).
This test is now "best effort" as the mechanism to prevent such race are
AlphareUnsubmitted
Not Done

"scénarion" is very fancy

Alphare: "scénarion" is very fancy
getting better, it get more complicated to test a specific scenario that would
trigger it. If you see flakyness here, there is a race.
$ cat > $TESTTMP/abort.py <<EOF $ cat > $TESTTMP/abort.py <<EOF
> from __future__ import absolute_import > from __future__ import absolute_import
> # emulate aborting before "recordupdates()". in this case, files > # emulate aborting before "recordupdates()". in this case, files
> # are changed without updating dirstate > # are changed without updating dirstate
> from mercurial import ( > from mercurial import (
> error, > error,
> extensions, > extensions,
> merge, > merge,
> ) > )
> def applyupdates(orig, *args, **kwargs): > def applyupdates(orig, *args, **kwargs):
> orig(*args, **kwargs) > orig(*args, **kwargs)
> raise error.Abort(b'intentional aborting') > raise error.Abort(b'intentional aborting')
> def extsetup(ui): > def extsetup(ui):
> extensions.wrapfunction(merge, "applyupdates", applyupdates) > extensions.wrapfunction(merge, "applyupdates", applyupdates)
> EOF > EOF
$ cat >> .hg/hgrc <<EOF
> [fakedirstatewritetime]
> # emulate invoking dirstate.write() via repo.status()
> # at 2000-01-01 00:00
> fakenow = 200001010000
> EOF
(file gotten from other revision) (file gotten from other revision)
$ hg update -q -C 2 $ hg update -q -C 2
$ echo 'THIS IS FILE B5' > b $ echo 'THIS IS FILE B5' > b
$ hg commit -m 'commit #5' $ hg commit -m 'commit #5'
$ hg update -q -C 3 $ hg update -q -C 3
$ cat b $ cat b
This is file b1 This is file b1
$ touch -t 200001010000 b
$ hg debugrebuildstate
$ cat >> .hg/hgrc <<EOF $ cat >> .hg/hgrc <<EOF
> [extensions] > [extensions]
> fakedirstatewritetime = $TESTDIR/fakedirstatewritetime.py
> abort = $TESTTMP/abort.py > abort = $TESTTMP/abort.py
> EOF > EOF
$ hg merge 5 $ hg merge 5
abort: intentional aborting abort: intentional aborting
[255] [255]
$ cat >> .hg/hgrc <<EOF $ cat >> .hg/hgrc <<EOF
> [extensions] > [extensions]
> fakedirstatewritetime = !
> abort = ! > abort = !
> EOF > EOF
$ cat b $ cat b
THIS IS FILE B5 THIS IS FILE B5
$ touch -t 200001010000 b
$ hg status -A b $ hg status -A b
M b M b
(file merged from other revision) (file merged from other revision)
$ hg update -q -C 3 $ hg update -q -C 3
$ echo 'this is file b6' > b $ echo 'this is file b6' > b
$ hg commit -m 'commit #6' $ hg commit -m 'commit #6'
created new head created new head
$ cat b $ cat b
this is file b6 this is file b6
$ touch -t 200001010000 b $ hg status
$ hg debugrebuildstate
$ cat >> .hg/hgrc <<EOF $ cat >> .hg/hgrc <<EOF
> [extensions] > [extensions]
> fakedirstatewritetime = $TESTDIR/fakedirstatewritetime.py
> abort = $TESTTMP/abort.py > abort = $TESTTMP/abort.py
> EOF > EOF
$ hg merge --tool internal:other 5 $ hg merge --tool internal:other 5
abort: intentional aborting abort: intentional aborting
[255] [255]
$ cat >> .hg/hgrc <<EOF $ cat >> .hg/hgrc <<EOF
> [extensions] > [extensions]
> fakedirstatewritetime = !
> abort = ! > abort = !
> EOF > EOF
$ cat b $ cat b
THIS IS FILE B5 THIS IS FILE B5
$ touch -t 200001010000 b
$ hg status -A b $ hg status -A b
M b M b
$ cd .. $ cd ..

tests/test-subrepo.t

abort: push creates new remote head cc505f09a8b2 (in subrepository "s") abort: push creates new remote head cc505f09a8b2 (in subrepository "s")
(merge or see 'hg help push' for details about pushing new heads) (merge or see 'hg help push' for details about pushing new heads)
[255] [255]
$ hg -R repo update $ hg -R repo update
0 files updated, 0 files merged, 0 files removed, 0 files unresolved 0 files updated, 0 files merged, 0 files removed, 0 files unresolved
test if untracked file is not overwritten test if untracked file is not overwritten
(this also tests that updated .hgsubstate is treated as "modified", (this tests also has a change to update .hgsubstate and merge it within the
when 'merge.update()' is aborted before 'merge.recordupdates()', even same second. It should mark is are modified , even if none of mode, size and
if none of mode, size and timestamp of it isn't changed on the timestamp of it isn't changed on the filesystem (see also issue4583))
filesystem (see also issue4583))
$ echo issue3276_ok > repo/s/b $ echo issue3276_ok > repo/s/b
$ hg -R repo2 push -f -q $ hg -R repo2 push -f -q
$ touch -t 200001010000 repo/.hgsubstate
$ cat >> repo/.hg/hgrc <<EOF
> [fakedirstatewritetime]
> # emulate invoking dirstate.write() via repo.status()
> # at 2000-01-01 00:00
> fakenow = 200001010000
>
> [extensions]
> fakedirstatewritetime = $TESTDIR/fakedirstatewritetime.py
> EOF
$ hg -R repo update $ hg -R repo update
b: untracked file differs b: untracked file differs
abort: untracked files in working directory differ from files in requested revision (in subrepository "s") abort: untracked files in working directory differ from files in requested revision (in subrepository "s")
[255] [255]
$ cat >> repo/.hg/hgrc <<EOF
> [extensions]
> fakedirstatewritetime = !
> EOF
$ cat repo/s/b $ cat repo/s/b
issue3276_ok issue3276_ok
$ rm repo/s/b $ rm repo/s/b
$ touch -t 200001010000 repo/.hgsubstate
$ hg -R repo revert --all $ hg -R repo revert --all
reverting repo/.hgsubstate reverting repo/.hgsubstate
reverting subrepo s reverting subrepo s
$ hg -R repo update $ hg -R repo update
1 files updated, 0 files merged, 0 files removed, 0 files unresolved 1 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ cat repo/s/b $ cat repo/s/b
2 2
$ rm -rf repo2 repo $ rm -rf repo2 repo