( )⚙ D10909 status: Extend read_dir caching to directories with ignored files

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10909

status: Extend read_dir caching to directories with ignored files
ClosedPublic

Authored by SimonSapin on Jun 24 2021, 4:52 PM.

Details

Reviewers
Alphare
Group Reviewers
hg-reviewers
Commits
rHG94e38822d395: status: Extend read_dir caching to directories with ignored files
Summary

See code comments

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

Diff 28738

rust/hg-core/src/dirstate_tree/on_disk.rs

/// ///
/// - To be a directory /// - To be a directory
/// - With the modification time as given by `Timestamp` /// - With the modification time as given by `Timestamp`
/// - That timestamp was already strictly in the past when observed, /// - That timestamp was already strictly in the past when observed,
/// meaning that later changes cannot happen in the same clock tick /// meaning that later changes cannot happen in the same clock tick
/// and must cause a different modification time (unless the system /// and must cause a different modification time (unless the system
/// clock jumps back and we get unlucky, which is not impossible but /// clock jumps back and we get unlucky, which is not impossible but
/// but deemed unlikely enough). /// but deemed unlikely enough).
/// - The directory did not contain any child entry that did not have a /// - All direct children of this directory (as returned by
/// corresponding dirstate node. /// `std::fs::read_dir`) either have a corresponding dirstate node, or
/// are ignored by ignore patterns whose hash is in
/// `Header::ignore_patterns_hash`.
/// ///
/// This means that if `std::fs::symlink_metadata` later reports the /// This means that if `std::fs::symlink_metadata` later reports the
/// same modification time, we don’t need to call `std::fs::read_dir` /// same modification time and ignored patterns haven’t changed, a run
/// again for this directory and can iterate child dirstate nodes /// of status that is not listing ignored files can skip calling
/// instead. /// `std::fs::read_dir` again for this directory, iterate child
/// dirstate nodes instead.
state: u8, state: u8,
data: Entry, data: Entry,
} }
#[derive(BytesCast, Copy, Clone)] #[derive(BytesCast, Copy, Clone)]
#[repr(C)] #[repr(C)]
struct Entry { struct Entry {
mode: I32Be, mode: I32Be,

rust/hg-core/src/dirstate_tree/status.rs

cached_directory_mtime: Option<&Timestamp>, cached_directory_mtime: Option<&Timestamp>,
) -> bool { ) -> bool {
if !self.options.list_unknown && !self.options.list_ignored { if !self.options.list_unknown && !self.options.list_ignored {
// All states that we care about listing have corresponding // All states that we care about listing have corresponding
// dirstate entries. // dirstate entries.
// This happens for example with `hg status -mard`. // This happens for example with `hg status -mard`.
return true; return true;
} }
if !self.options.list_ignored
&& self.ignore_patterns_have_changed == Some(false)
{
if let Some(cached_mtime) = cached_directory_mtime { if let Some(cached_mtime) = cached_directory_mtime {
// The dirstate contains a cached mtime for this directory, set by // The dirstate contains a cached mtime for this directory, set
// a previous run of the `status` algorithm which found this // by a previous run of the `status` algorithm which found this
// directory eligible for `read_dir` caching. // directory eligible for `read_dir` caching.
if let Some(meta) = directory_metadata { if let Some(meta) = directory_metadata {
if let Ok(current_mtime) = meta.modified() { if let Ok(current_mtime) = meta.modified() {
if current_mtime == cached_mtime.into() { if current_mtime == cached_mtime.into() {
// The mtime of that directory has not changed since // The mtime of that directory has not changed
// then, which means that the // since then, which means that the results of
// results of `read_dir` should also // `read_dir` should also be unchanged.
// be unchanged.
return true; return true;
} }
} }
} }
} }
}
false false
} }
/// Returns whether the filesystem directory was found to have any entry /// Returns whether all child entries of the filesystem directory have a
/// that does not have a corresponding dirstate tree node. /// corresponding dirstate node or are ignored.
fn traverse_fs_directory_and_dirstate( fn traverse_fs_directory_and_dirstate(
&self, &self,
has_ignored_ancestor: bool, has_ignored_ancestor: bool,
dirstate_nodes: ChildNodesRef<'tree, 'on_disk>, dirstate_nodes: ChildNodesRef<'tree, 'on_disk>,
directory_hg_path: &BorrowedPath<'tree, 'on_disk>, directory_hg_path: &BorrowedPath<'tree, 'on_disk>,
directory_fs_path: &Path, directory_fs_path: &Path,
directory_metadata: Option<&std::fs::Metadata>, directory_metadata: Option<&std::fs::Metadata>,
cached_directory_mtime: Option<&Timestamp>, cached_directory_mtime: Option<&Timestamp>,
let hg_path = let hg_path =
dirstate_node.full_path(self.dmap.on_disk)?; dirstate_node.full_path(self.dmap.on_disk)?;
Ok(self.io_error(error, hg_path)) Ok(self.io_error(error, hg_path))
} }
} }
}) })
.collect::<Result<_, _>>()?; .collect::<Result<_, _>>()?;
// Conservatively don’t let the caller assume that there aren’t // We don’t know, so conservatively say this isn’t the case
// any, since we don’t know. let children_all_have_dirstate_node_or_are_ignored = false;
let directory_has_any_fs_only_entry = true;
return Ok(directory_has_any_fs_only_entry); return Ok(children_all_have_dirstate_node_or_are_ignored);
} }
let mut fs_entries = if let Ok(entries) = self.read_dir( let mut fs_entries = if let Ok(entries) = self.read_dir(
directory_hg_path, directory_hg_path,
directory_fs_path, directory_fs_path,
is_at_repo_root, is_at_repo_root,
) { ) {
entries entries
.base_name(self.dmap.on_disk) .base_name(self.dmap.on_disk)
.unwrap() .unwrap()
.cmp(&fs_entry.base_name) .cmp(&fs_entry.base_name)
}, },
) )
.par_bridge() .par_bridge()
.map(|pair| { .map(|pair| {
use itertools::EitherOrBoth::*; use itertools::EitherOrBoth::*;
let is_fs_only = pair.is_right(); let has_dirstate_node_or_is_ignored;
match pair { match pair {
Both(dirstate_node, fs_entry) => self Both(dirstate_node, fs_entry) => {
.traverse_fs_and_dirstate( self.traverse_fs_and_dirstate(
&fs_entry.full_path, &fs_entry.full_path,
&fs_entry.metadata, &fs_entry.metadata,
dirstate_node, dirstate_node,
has_ignored_ancestor, has_ignored_ancestor,
)?, )?;
has_dirstate_node_or_is_ignored = true
}
Left(dirstate_node) => { Left(dirstate_node) => {
self.traverse_dirstate_only(dirstate_node)? self.traverse_dirstate_only(dirstate_node)?;
has_dirstate_node_or_is_ignored = true;
} }
Right(fs_entry) => self.traverse_fs_only( Right(fs_entry) => {
has_dirstate_node_or_is_ignored = self.traverse_fs_only(
has_ignored_ancestor, has_ignored_ancestor,
directory_hg_path, directory_hg_path,
fs_entry, fs_entry,
), )
} }
Ok(is_fs_only) }
Ok(has_dirstate_node_or_is_ignored)
}) })
.try_reduce(|| false, |a, b| Ok(a || b)) .try_reduce(|| true, |a, b| Ok(a && b))
} }
fn traverse_fs_and_dirstate( fn traverse_fs_and_dirstate(
&self, &self,
fs_path: &Path, fs_path: &Path,
fs_metadata: &std::fs::Metadata, fs_metadata: &std::fs::Metadata,
dirstate_node: NodeRef<'tree, 'on_disk>, dirstate_node: NodeRef<'tree, 'on_disk>,
has_ignored_ancestor: bool, has_ignored_ancestor: bool,
self.outcome self.outcome
.lock() .lock()
.unwrap() .unwrap()
.traversed .traversed
.push(hg_path.detach_from_tree()) .push(hg_path.detach_from_tree())
} }
let is_ignored = has_ignored_ancestor || (self.ignore_fn)(hg_path); let is_ignored = has_ignored_ancestor || (self.ignore_fn)(hg_path);
let is_at_repo_root = false; let is_at_repo_root = false;
let directory_has_any_fs_only_entry = self let children_all_have_dirstate_node_or_are_ignored = self
.traverse_fs_directory_and_dirstate( .traverse_fs_directory_and_dirstate(
is_ignored, is_ignored,
dirstate_node.children(self.dmap.on_disk)?, dirstate_node.children(self.dmap.on_disk)?,
hg_path, hg_path,
fs_path, fs_path,
Some(fs_metadata), Some(fs_metadata),
dirstate_node.cached_directory_mtime(), dirstate_node.cached_directory_mtime(),
is_at_repo_root, is_at_repo_root,
)?; )?;
self.maybe_save_directory_mtime( self.maybe_save_directory_mtime(
directory_has_any_fs_only_entry, children_all_have_dirstate_node_or_are_ignored,
fs_metadata, fs_metadata,
dirstate_node, dirstate_node,
)? )?
} else { } else {
if file_or_symlink && self.matcher.matches(hg_path) { if file_or_symlink && self.matcher.matches(hg_path) {
if let Some(state) = dirstate_node.state()? { if let Some(state) = dirstate_node.state()? {
match state { match state {
EntryState::Added => self EntryState::Added => self
.handle_normal_file(&dirstate_node, fs_metadata)?, .handle_normal_file(&dirstate_node, fs_metadata)?,
// This variant is not used in DirstateMap // This variant is not used in DirstateMap
// nodes // nodes
EntryState::Unknown => unreachable!(), EntryState::Unknown => unreachable!(),
} }
} else { } else {
// `node.entry.is_none()` indicates a "directory" // `node.entry.is_none()` indicates a "directory"
// node, but the filesystem has a file // node, but the filesystem has a file
self.mark_unknown_or_ignored(has_ignored_ancestor, hg_path) self.mark_unknown_or_ignored(
has_ignored_ancestor,
hg_path,
);
} }
} }
for child_node in dirstate_node.children(self.dmap.on_disk)?.iter() for child_node in dirstate_node.children(self.dmap.on_disk)?.iter()
{ {
self.traverse_dirstate_only(child_node)? self.traverse_dirstate_only(child_node)?
} }
} }
Ok(()) Ok(())
} }
fn maybe_save_directory_mtime( fn maybe_save_directory_mtime(
&self, &self,
directory_has_any_fs_only_entry: bool, children_all_have_dirstate_node_or_are_ignored: bool,
directory_metadata: &std::fs::Metadata, directory_metadata: &std::fs::Metadata,
dirstate_node: NodeRef<'tree, 'on_disk>, dirstate_node: NodeRef<'tree, 'on_disk>,
) -> Result<(), DirstateV2ParseError> { ) -> Result<(), DirstateV2ParseError> {
if !directory_has_any_fs_only_entry { if children_all_have_dirstate_node_or_are_ignored {
// All filesystem directory entries from `read_dir` have a // All filesystem directory entries from `read_dir` have a
// corresponding node in the dirstate, so we can reconstitute the // corresponding node in the dirstate, so we can reconstitute the
// names of those entries without calling `read_dir` again. // names of those entries without calling `read_dir` again.
if let (Some(status_start), Ok(directory_mtime)) = ( if let (Some(status_start), Ok(directory_mtime)) = (
&self.filesystem_time_at_status_start, &self.filesystem_time_at_status_start,
directory_metadata.modified(), directory_metadata.modified(),
) { ) {
// Although the Rust standard library’s `SystemTime` type // Although the Rust standard library’s `SystemTime` type
.deleted .deleted
.push(hg_path.detach_from_tree()) .push(hg_path.detach_from_tree())
} }
} }
} }
} }
/// Something in the filesystem has no corresponding dirstate node /// Something in the filesystem has no corresponding dirstate node
///
/// Returns whether that path is ignored
fn traverse_fs_only( fn traverse_fs_only(
&self, &self,
has_ignored_ancestor: bool, has_ignored_ancestor: bool,
directory_hg_path: &HgPath, directory_hg_path: &HgPath,
fs_entry: &DirEntry, fs_entry: &DirEntry,
) { ) -> bool {
let hg_path = directory_hg_path.join(&fs_entry.base_name); let hg_path = directory_hg_path.join(&fs_entry.base_name);
let file_type = fs_entry.metadata.file_type(); let file_type = fs_entry.metadata.file_type();
let file_or_symlink = file_type.is_file() || file_type.is_symlink(); let file_or_symlink = file_type.is_file() || file_type.is_symlink();
if file_type.is_dir() { if file_type.is_dir() {
let is_ignored = let is_ignored =
has_ignored_ancestor || (self.ignore_fn)(&hg_path); has_ignored_ancestor || (self.ignore_fn)(&hg_path);
let traverse_children = if is_ignored { let traverse_children = if is_ignored {
// Descendants of an ignored directory are all ignored // Descendants of an ignored directory are all ignored
&fs_entry.full_path, &fs_entry.full_path,
is_at_repo_root, is_at_repo_root,
) { ) {
children_fs_entries.par_iter().for_each(|child_fs_entry| { children_fs_entries.par_iter().for_each(|child_fs_entry| {
self.traverse_fs_only( self.traverse_fs_only(
is_ignored, is_ignored,
&hg_path, &hg_path,
child_fs_entry, child_fs_entry,
) );
}) })
} }
} }
if self.options.collect_traversed_dirs { if self.options.collect_traversed_dirs {
self.outcome.lock().unwrap().traversed.push(hg_path.into()) self.outcome.lock().unwrap().traversed.push(hg_path.into())
} }
} else if file_or_symlink && self.matcher.matches(&hg_path) { is_ignored
} else {
if file_or_symlink {
if self.matcher.matches(&hg_path) {
self.mark_unknown_or_ignored( self.mark_unknown_or_ignored(
has_ignored_ancestor, has_ignored_ancestor,
&BorrowedPath::InMemory(&hg_path), &BorrowedPath::InMemory(&hg_path),
) )
} else {
// We haven’t computed whether this path is ignored. It
// might not be, and a future run of status might have a
// different matcher that matches it. So treat it as not
// ignored. That is, inhibit readdir caching of the parent
// directory.
false
}
} else {
// This is neither a directory, a plain file, or a symlink.
// Treat it like an ignored file.
true
}
} }
} }
/// Returns whether that path is ignored
fn mark_unknown_or_ignored( fn mark_unknown_or_ignored(
&self, &self,
has_ignored_ancestor: bool, has_ignored_ancestor: bool,
hg_path: &BorrowedPath<'_, 'on_disk>, hg_path: &BorrowedPath<'_, 'on_disk>,
) { ) -> bool {
let is_ignored = has_ignored_ancestor || (self.ignore_fn)(&hg_path); let is_ignored = has_ignored_ancestor || (self.ignore_fn)(&hg_path);
if is_ignored { if is_ignored {
if self.options.list_ignored { if self.options.list_ignored {
self.outcome self.outcome
.lock() .lock()
.unwrap() .unwrap()
.ignored .ignored
.push(hg_path.detach_from_tree()) .push(hg_path.detach_from_tree())
} }
} else { } else {
if self.options.list_unknown { if self.options.list_unknown {
self.outcome self.outcome
.lock() .lock()
.unwrap() .unwrap()
.unknown .unknown
.push(hg_path.detach_from_tree()) .push(hg_path.detach_from_tree())
} }
} }
is_ignored
} }
} }
#[cfg(unix)] // TODO #[cfg(unix)] // TODO
fn mtime_seconds(metadata: &std::fs::Metadata) -> i64 { fn mtime_seconds(metadata: &std::fs::Metadata) -> i64 {
// Going through `Metadata::modified()` would be portable, but would take // Going through `Metadata::modified()` would be portable, but would take
// care to construct a `SystemTime` value with sub-second precision just // care to construct a `SystemTime` value with sub-second precision just
// for us to throw that away here. // for us to throw that away here.