( )⚙ D11795 dirstate: remove `lastnormaltime` mechanism

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D11795

dirstate: remove `lastnormaltime` mechanism
ClosedPublic

Authored by marmoute on Nov 24 2021, 6:12 AM.

Details

Reviewers
Alphare
Group Reviewers
hg-reviewers
Commits
rHGb80e5e75d51e: dirstate: remove `lastnormaltime` mechanism
rHGa19d12250e05: dirstate: remove `lastnormaltime` mechanism
Summary

This is now redundant with the new, simpler mtime_boundary one.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

Diff 31257

mercurial/dirstate.py

self._opener = opener self._opener = opener
self._validate = validate self._validate = validate
self._root = root self._root = root
self._sparsematchfn = sparsematchfn self._sparsematchfn = sparsematchfn
# ntpath.join(root, '') of Python 2.7.9 does not add sep if root is # ntpath.join(root, '') of Python 2.7.9 does not add sep if root is
# UNC path pointing to root share (issue4557) # UNC path pointing to root share (issue4557)
self._rootdir = pathutil.normasprefix(root) self._rootdir = pathutil.normasprefix(root)
self._dirty = False self._dirty = False
self._lastnormaltime = timestamp.zero()
self._ui = ui self._ui = ui
self._filecache = {} self._filecache = {}
self._parentwriters = 0 self._parentwriters = 0
self._filename = b'dirstate' self._filename = b'dirstate'
self._pendingfilename = b'%s.pending' % self._filename self._pendingfilename = b'%s.pending' % self._filename
self._plchangecallbacks = {} self._plchangecallbacks = {}
self._origpl = None self._origpl = None
self._mapcls = dirstatemap.dirstatemap self._mapcls = dirstatemap.dirstatemap
This is different from localrepo.invalidatedirstate() because it always This is different from localrepo.invalidatedirstate() because it always
rereads the dirstate. Use localrepo.invalidatedirstate() if you want to rereads the dirstate. Use localrepo.invalidatedirstate() if you want to
check whether the dirstate has changed before rereading it.""" check whether the dirstate has changed before rereading it."""
for a in ("_map", "_branch", "_ignore"): for a in ("_map", "_branch", "_ignore"):
if a in self.__dict__: if a in self.__dict__:
delattr(self, a) delattr(self, a)
self._lastnormaltime = timestamp.zero()
self._dirty = False self._dirty = False
self._parentwriters = 0 self._parentwriters = 0
self._origpl = None self._origpl = None
def copy(self, source, dest): def copy(self, source, dest):
"""Mark dest as a copy of source. Unmark dest if source is None.""" """Mark dest as a copy of source. Unmark dest if source is None."""
if source == dest: if source == dest:
return return
@requires_no_parents_change @requires_no_parents_change
def set_clean(self, filename, parentfiledata): def set_clean(self, filename, parentfiledata):
"""record that the current state of the file on disk is known to be clean""" """record that the current state of the file on disk is known to be clean"""
self._dirty = True self._dirty = True
if not self._map[filename].tracked: if not self._map[filename].tracked:
self._check_new_tracked_filename(filename) self._check_new_tracked_filename(filename)
(mode, size, mtime) = parentfiledata (mode, size, mtime) = parentfiledata
self._map.set_clean(filename, mode, size, mtime) self._map.set_clean(filename, mode, size, mtime)
if mtime > self._lastnormaltime:
# Remember the most recent modification timeslot for status(),
# to make sure we won't miss future size-preserving file content
# modifications that happen within the same timeslot.
self._lastnormaltime = mtime
@requires_no_parents_change @requires_no_parents_change
def set_possibly_dirty(self, filename): def set_possibly_dirty(self, filename):
"""record that the current state of the file on disk is unknown""" """record that the current state of the file on disk is unknown"""
self._dirty = True self._dirty = True
self._map.set_possibly_dirty(filename) self._map.set_possibly_dirty(filename)
@requires_parents_change @requires_parents_change
self._map.reset_state( self._map.reset_state(
filename, filename,
wc_tracked, wc_tracked,
p1_tracked, p1_tracked,
p2_info=p2_info, p2_info=p2_info,
has_meaningful_mtime=not possibly_dirty, has_meaningful_mtime=not possibly_dirty,
parentfiledata=parentfiledata, parentfiledata=parentfiledata,
) )
if (
parentfiledata is not None
and parentfiledata[2] is not None
and parentfiledata[2] > self._lastnormaltime
):
# Remember the most recent modification timeslot for status(),
# to make sure we won't miss future size-preserving file content
# modifications that happen within the same timeslot.
self._lastnormaltime = parentfiledata[2]
def _check_new_tracked_filename(self, filename): def _check_new_tracked_filename(self, filename):
scmutil.checkfilename(filename) scmutil.checkfilename(filename)
if self._map.hastrackeddir(filename): if self._map.hastrackeddir(filename):
msg = _(b'directory %r already in dirstate') msg = _(b'directory %r already in dirstate')
msg %= pycompat.bytestr(filename) msg %= pycompat.bytestr(filename)
raise error.Abort(msg) raise error.Abort(msg)
# shadows # shadows
""" """
if self._checkcase: if self._checkcase:
return self._normalize(path, isknown, ignoremissing) return self._normalize(path, isknown, ignoremissing)
return path return path
def clear(self): def clear(self):
self._map.clear() self._map.clear()
self._lastnormaltime = timestamp.zero()
self._dirty = True self._dirty = True
def rebuild(self, parent, allfiles, changedfiles=None): def rebuild(self, parent, allfiles, changedfiles=None):
if changedfiles is None: if changedfiles is None:
# Rebuild entire dirstate # Rebuild entire dirstate
to_lookup = allfiles to_lookup = allfiles
to_drop = [] to_drop = []
lastnormaltime = self._lastnormaltime
self.clear() self.clear()
self._lastnormaltime = lastnormaltime
elif len(changedfiles) < 10: elif len(changedfiles) < 10:
# Avoid turning allfiles into a set, which can be expensive if it's # Avoid turning allfiles into a set, which can be expensive if it's
# large. # large.
to_lookup = [] to_lookup = []
to_drop = [] to_drop = []
for f in changedfiles: for f in changedfiles:
if f in allfiles: if f in allfiles:
to_lookup.append(f) to_lookup.append(f)
start = int(clock) - (int(clock) % delaywrite) start = int(clock) - (int(clock) % delaywrite)
end = start + delaywrite end = start + delaywrite
time.sleep(end - clock) time.sleep(end - clock)
# trust our estimate that the end is near now # trust our estimate that the end is near now
now = timestamp.timestamp((end, 0)) now = timestamp.timestamp((end, 0))
break break
self._map.write(tr, st, now) self._map.write(tr, st, now)
self._lastnormaltime = timestamp.zero()
self._dirty = False self._dirty = False
def _dirignore(self, f): def _dirignore(self, f):
if self._ignore(f): if self._ignore(f):
return True return True
for p in pathutil.finddirs(f): for p in pathutil.finddirs(f):
if self._ignore(p): if self._ignore(p):
return True return True
traversed, traversed,
dirty, dirty,
) = rustmod.status( ) = rustmod.status(
self._map._map, self._map._map,
matcher, matcher,
self._rootdir, self._rootdir,
self._ignorefiles(), self._ignorefiles(),
self._checkexec, self._checkexec,
self._lastnormaltime,
bool(list_clean), bool(list_clean),
bool(list_ignored), bool(list_ignored),
bool(list_unknown), bool(list_unknown),
bool(matcher.traversedir), bool(matcher.traversedir),
) )
self._dirty |= dirty self._dirty |= dirty
radd = removed.append radd = removed.append
dadd = deleted.append dadd = deleted.append
cadd = clean.append if listclean else noop cadd = clean.append if listclean else noop
mexact = match.exact mexact = match.exact
dirignore = self._dirignore dirignore = self._dirignore
checkexec = self._checkexec checkexec = self._checkexec
checklink = self._checklink checklink = self._checklink
copymap = self._map.copymap copymap = self._map.copymap
lastnormaltime = self._lastnormaltime
# We need to do full walks when either # We need to do full walks when either
# - we're listing all clean files, or # - we're listing all clean files, or
# - match.traversedir does something, because match.traversedir should # - match.traversedir does something, because match.traversedir should
# be called for every dir in the working dir # be called for every dir in the working dir
full = listclean or match.traversedir is not None full = listclean or match.traversedir is not None
for fn, st in pycompat.iteritems( for fn, st in pycompat.iteritems(
self.walk(match, subrepos, listunknown, listignored, full=full) self.walk(match, subrepos, listunknown, listignored, full=full)
): ):
if stat.S_ISLNK(st.st_mode) and size != st.st_size: if stat.S_ISLNK(st.st_mode) and size != st.st_size:
# issue6456: Size returned may be longer due to # issue6456: Size returned may be longer due to
# encryption on EXT-4 fscrypt, undecided. # encryption on EXT-4 fscrypt, undecided.
ladd(fn) ladd(fn)
else: else:
madd(fn) madd(fn)
elif not t.mtime_likely_equal_to(timestamp.mtime_of(st)): elif not t.mtime_likely_equal_to(timestamp.mtime_of(st)):
ladd(fn) # There might be a change in the future if for example the
elif timestamp.mtime_of(st) == lastnormaltime: # internal clock is off, but this is a case where the issues
# fn may have just been marked as normal and it may have # the user would face would be a lot worse and there is
# changed in the same second without changing its size. # nothing we can really do.
# This can happen if we quickly do multiple commits.
# Force lookup, so we don't miss such a racy file change.
ladd(fn) ladd(fn)
elif listclean: elif listclean:
cadd(fn) cadd(fn)
status = scmutil.status( status = scmutil.status(
modified, added, removed, deleted, unknown, ignored, clean modified, added, removed, deleted, unknown, ignored, clean
) )
return (lookup, status, mtime_boundary) return (lookup, status, mtime_boundary)

rust/hg-core/src/dirstate/status.rs

// status.rs // status.rs
// //
// Copyright 2019 Raphaël Gomès <rgomes@octobus.net> // Copyright 2019 Raphaël Gomès <rgomes@octobus.net>
// //
// This software may be used and distributed according to the terms of the // This software may be used and distributed according to the terms of the
// GNU General Public License version 2 or any later version. // GNU General Public License version 2 or any later version.
//! Rust implementation of dirstate.status (dirstate.py). //! Rust implementation of dirstate.status (dirstate.py).
//! It is currently missing a lot of functionality compared to the Python one //! It is currently missing a lot of functionality compared to the Python one
//! and will only be triggered in narrow cases. //! and will only be triggered in narrow cases.
use crate::dirstate_tree::on_disk::DirstateV2ParseError; use crate::dirstate_tree::on_disk::DirstateV2ParseError;
use crate::{ use crate::{
dirstate::TruncatedTimestamp,
utils::hg_path::{HgPath, HgPathError}, utils::hg_path::{HgPath, HgPathError},
PatternError, PatternError,
}; };
use std::{borrow::Cow, fmt}; use std::{borrow::Cow, fmt};
/// Wrong type of file from a `BadMatch` /// Wrong type of file from a `BadMatch`
/// Note: a lot of those don't exist on all platforms. /// Note: a lot of those don't exist on all platforms.
Box<dyn for<'r> Fn(&'r HgPath) -> bool + Sync + 'a>; Box<dyn for<'r> Fn(&'r HgPath) -> bool + Sync + 'a>;
/// We have a good mix of owned (from directory traversal) and borrowed (from /// We have a good mix of owned (from directory traversal) and borrowed (from
/// the dirstate/explicit) paths, this comes up a lot. /// the dirstate/explicit) paths, this comes up a lot.
pub type HgPathCow<'a> = Cow<'a, HgPath>; pub type HgPathCow<'a> = Cow<'a, HgPath>;
#[derive(Debug, Copy, Clone)] #[derive(Debug, Copy, Clone)]
pub struct StatusOptions { pub struct StatusOptions {
/// Remember the most recent modification timeslot for status, to make
/// sure we won't miss future size-preserving file content modifications
/// that happen within the same timeslot.
pub last_normal_time: TruncatedTimestamp,
/// Whether we are on a filesystem with UNIX-like exec flags /// Whether we are on a filesystem with UNIX-like exec flags
pub check_exec: bool, pub check_exec: bool,
pub list_clean: bool, pub list_clean: bool,
pub list_unknown: bool, pub list_unknown: bool,
pub list_ignored: bool, pub list_ignored: bool,
/// Whether to collect traversed dirs for applying a callback later. /// Whether to collect traversed dirs for applying a callback later.
/// Used by `hg purge` for example. /// Used by `hg purge` for example.
pub collect_traversed_dirs: bool, pub collect_traversed_dirs: bool,

rust/hg-core/src/dirstate_tree/status.rs

.unwrap() .unwrap()
.modified .modified
.push(hg_path.detach_from_tree()) .push(hg_path.detach_from_tree())
} else { } else {
let mtime_looks_clean; let mtime_looks_clean;
if let Some(dirstate_mtime) = entry.truncated_mtime() { if let Some(dirstate_mtime) = entry.truncated_mtime() {
let fs_mtime = TruncatedTimestamp::for_mtime_of(fs_metadata) let fs_mtime = TruncatedTimestamp::for_mtime_of(fs_metadata)
.expect("OS/libc does not support mtime?"); .expect("OS/libc does not support mtime?");
// There might be a change in the future if for example the
// internal clock become off while process run, but this is a
// case where the issues the user would face
// would be a lot worse and there is nothing we
// can really do.
mtime_looks_clean = fs_mtime.likely_equal(dirstate_mtime) mtime_looks_clean = fs_mtime.likely_equal(dirstate_mtime)
&& !fs_mtime.likely_equal(self.options.last_normal_time)
} else { } else {
// No mtime in the dirstate entry // No mtime in the dirstate entry
mtime_looks_clean = false mtime_looks_clean = false
}; };
if !mtime_looks_clean { if !mtime_looks_clean {
self.outcome self.outcome
.lock() .lock()
.unwrap() .unwrap()

rust/hg-cpython/src/dirstate.rs

py_fn!( py_fn!(
py, py,
status_wrapper( status_wrapper(
dmap: DirstateMap, dmap: DirstateMap,
root_dir: PyObject, root_dir: PyObject,
matcher: PyObject, matcher: PyObject,
ignorefiles: PyList, ignorefiles: PyList,
check_exec: bool, check_exec: bool,
last_normal_time: (u32, u32),
list_clean: bool, list_clean: bool,
list_ignored: bool, list_ignored: bool,
list_unknown: bool, list_unknown: bool,
collect_traversed_dirs: bool collect_traversed_dirs: bool
) )
), ),
)?; )?;
let sys = PyModule::import(py, "sys")?; let sys = PyModule::import(py, "sys")?;
let sys_modules: PyDict = sys.get(py, "modules")?.extract(py)?; let sys_modules: PyDict = sys.get(py, "modules")?.extract(py)?;
sys_modules.set_item(py, dotted_name, &m)?; sys_modules.set_item(py, dotted_name, &m)?;
Ok(m) Ok(m)
} }

rust/hg-cpython/src/dirstate/status.rs

// status.rs // status.rs
// //
// Copyright 2019, Raphaël Gomès <rgomes@octobus.net> // Copyright 2019, Raphaël Gomès <rgomes@octobus.net>
// //
// This software may be used and distributed according to the terms of the // This software may be used and distributed according to the terms of the
// GNU General Public License version 2 or any later version. // GNU General Public License version 2 or any later version.
//! Bindings for the `hg::status` module provided by the //! Bindings for the `hg::status` module provided by the
//! `hg-core` crate. From Python, this will be seen as //! `hg-core` crate. From Python, this will be seen as
//! `rustext.dirstate.status`. //! `rustext.dirstate.status`.
use crate::dirstate::item::timestamp;
use crate::{dirstate::DirstateMap, exceptions::FallbackError}; use crate::{dirstate::DirstateMap, exceptions::FallbackError};
use cpython::exc::OSError; use cpython::exc::OSError;
use cpython::{ use cpython::{
exc::ValueError, ObjectProtocol, PyBytes, PyErr, PyList, PyObject, exc::ValueError, ObjectProtocol, PyBytes, PyErr, PyList, PyObject,
PyResult, PyTuple, Python, PythonObject, ToPyObject, PyResult, PyTuple, Python, PythonObject, ToPyObject,
}; };
use hg::{ use hg::{
matchers::{AlwaysMatcher, FileMatcher, IncludeMatcher}, matchers::{AlwaysMatcher, FileMatcher, IncludeMatcher},
pub fn status_wrapper( pub fn status_wrapper(
py: Python, py: Python,
dmap: DirstateMap, dmap: DirstateMap,
matcher: PyObject, matcher: PyObject,
root_dir: PyObject, root_dir: PyObject,
ignore_files: PyList, ignore_files: PyList,
check_exec: bool, check_exec: bool,
last_normal_time: (u32, u32),
list_clean: bool, list_clean: bool,
list_ignored: bool, list_ignored: bool,
list_unknown: bool, list_unknown: bool,
collect_traversed_dirs: bool, collect_traversed_dirs: bool,
) -> PyResult<PyTuple> { ) -> PyResult<PyTuple> {
let last_normal_time = timestamp(py, last_normal_time)?;
let bytes = root_dir.extract::<PyBytes>(py)?; let bytes = root_dir.extract::<PyBytes>(py)?;
let root_dir = get_path_from_bytes(bytes.data(py)); let root_dir = get_path_from_bytes(bytes.data(py));
let dmap: DirstateMap = dmap.to_py_object(py); let dmap: DirstateMap = dmap.to_py_object(py);
let mut dmap = dmap.get_inner_mut(py); let mut dmap = dmap.get_inner_mut(py);
let ignore_files: PyResult<Vec<_>> = ignore_files let ignore_files: PyResult<Vec<_>> = ignore_files
.iter(py) .iter(py)
let matcher = AlwaysMatcher; let matcher = AlwaysMatcher;
let (status_res, warnings) = dmap let (status_res, warnings) = dmap
.status( .status(
&matcher, &matcher,
root_dir.to_path_buf(), root_dir.to_path_buf(),
ignore_files, ignore_files,
StatusOptions { StatusOptions {
check_exec, check_exec,
last_normal_time,
list_clean, list_clean,
list_ignored, list_ignored,
list_unknown, list_unknown,
collect_traversed_dirs, collect_traversed_dirs,
}, },
) )
.map_err(|e| handle_fallback(py, e))?; .map_err(|e| handle_fallback(py, e))?;
build_response(py, status_res, warnings) build_response(py, status_res, warnings)
.map_err(|e| PyErr::new::<ValueError, _>(py, e.to_string()))?; .map_err(|e| PyErr::new::<ValueError, _>(py, e.to_string()))?;
let (status_res, warnings) = dmap let (status_res, warnings) = dmap
.status( .status(
&matcher, &matcher,
root_dir.to_path_buf(), root_dir.to_path_buf(),
ignore_files, ignore_files,
StatusOptions { StatusOptions {
check_exec, check_exec,
last_normal_time,
list_clean, list_clean,
list_ignored, list_ignored,
list_unknown, list_unknown,
collect_traversed_dirs, collect_traversed_dirs,
}, },
) )
.map_err(|e| handle_fallback(py, e))?; .map_err(|e| handle_fallback(py, e))?;
build_response(py, status_res, warnings) build_response(py, status_res, warnings)
let (status_res, warnings) = dmap let (status_res, warnings) = dmap
.status( .status(
&matcher, &matcher,
root_dir.to_path_buf(), root_dir.to_path_buf(),
ignore_files, ignore_files,
StatusOptions { StatusOptions {
check_exec, check_exec,
last_normal_time,
list_clean, list_clean,
list_ignored, list_ignored,
list_unknown, list_unknown,
collect_traversed_dirs, collect_traversed_dirs,
}, },
) )
.map_err(|e| handle_fallback(py, e))?; .map_err(|e| handle_fallback(py, e))?;

rust/rhg/src/commands/status.rs

// status.rs // status.rs
// //
// Copyright 2020, Georges Racinet <georges.racinets@octobus.net> // Copyright 2020, Georges Racinet <georges.racinets@octobus.net>
// //
// This software may be used and distributed according to the terms of the // This software may be used and distributed according to the terms of the
// GNU General Public License version 2 or any later version. // GNU General Public License version 2 or any later version.
use crate::error::CommandError; use crate::error::CommandError;
use crate::ui::Ui; use crate::ui::Ui;
use crate::utils::path_utils::relativize_paths; use crate::utils::path_utils::relativize_paths;
use clap::{Arg, SubCommand}; use clap::{Arg, SubCommand};
use format_bytes::format_bytes; use format_bytes::format_bytes;
use hg; use hg;
use hg::config::Config; use hg::config::Config;
use hg::dirstate::{has_exec_bit, TruncatedTimestamp}; use hg::dirstate::has_exec_bit;
use hg::errors::HgError; use hg::errors::HgError;
use hg::manifest::Manifest; use hg::manifest::Manifest;
use hg::matchers::AlwaysMatcher; use hg::matchers::AlwaysMatcher;
use hg::repo::Repo; use hg::repo::Repo;
use hg::utils::files::get_bytes_from_os_string; use hg::utils::files::get_bytes_from_os_string;
use hg::utils::hg_path::{hg_path_to_os_string, HgPath}; use hg::utils::hg_path::{hg_path_to_os_string, HgPath};
use hg::{HgPathCow, StatusOptions}; use hg::{HgPathCow, StatusOptions};
use log::{info, warn}; use log::{info, warn};
} }
}; };
let no_status = args.is_present("no-status"); let no_status = args.is_present("no-status");
let repo = invocation.repo?; let repo = invocation.repo?;
let mut dmap = repo.dirstate_map_mut()?; let mut dmap = repo.dirstate_map_mut()?;
let options = StatusOptions { let options = StatusOptions {
// TODO should be provided by the dirstate parsing and
// hence be stored on dmap. Using a value that assumes we aren't
// below the time resolution granularity of the FS and the
// dirstate.
last_normal_time: TruncatedTimestamp::new_truncate(0, 0),
// we're currently supporting file systems with exec flags only // we're currently supporting file systems with exec flags only
// anyway // anyway
check_exec: true, check_exec: true,
list_clean: display_states.clean, list_clean: display_states.clean,
list_unknown: display_states.unknown, list_unknown: display_states.unknown,
list_ignored: display_states.ignored, list_ignored: display_states.ignored,
collect_traversed_dirs: false, collect_traversed_dirs: false,
}; };
repo: &Repo, repo: &Repo,
manifest: &Manifest, manifest: &Manifest,
hg_path: &HgPath, hg_path: &HgPath,
) -> Result<bool, HgError> { ) -> Result<bool, HgError> {
let vfs = repo.working_directory_vfs(); let vfs = repo.working_directory_vfs();
let fs_path = hg_path_to_os_string(hg_path).expect("HgPath conversion"); let fs_path = hg_path_to_os_string(hg_path).expect("HgPath conversion");
let fs_metadata = vfs.symlink_metadata(&fs_path)?; let fs_metadata = vfs.symlink_metadata(&fs_path)?;
let is_symlink = fs_metadata.file_type().is_symlink(); let is_symlink = fs_metadata.file_type().is_symlink();
// TODO: Also account for `FALLBACK_SYMLINK` and `FALLBACK_EXEC` from the dirstate // TODO: Also account for `FALLBACK_SYMLINK` and `FALLBACK_EXEC` from the
// dirstate
let fs_flags = if is_symlink { let fs_flags = if is_symlink {
Some(b'l') Some(b'l')
} else if has_exec_bit(&fs_metadata) { } else if has_exec_bit(&fs_metadata) {
Some(b'x') Some(b'x')
} else { } else {
None None
}; };