( )⚙ D10909 status: Extend read_dir caching to directories with ignored files

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10909

status: Extend read_dir caching to directories with ignored files
ClosedPublic

Authored by SimonSapin on Jun 24 2021, 4:52 PM.

Details

Reviewers
Alphare
Group Reviewers
hg-reviewers
Commits
rHG94e38822d395: status: Extend read_dir caching to directories with ignored files
Summary

See code comments

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Revision Contents
Changeset List

Diff 28699

rust/hg-core/src/dirstate_tree/on_disk.rs

/// ///
/// - To be a directory /// - To be a directory
/// - With the modification time as given by `Timestamp` /// - With the modification time as given by `Timestamp`
/// - That timestamp was already strictly in the past when observed, /// - That timestamp was already strictly in the past when observed,
/// meaning that later changes cannot happen in the same clock tick /// meaning that later changes cannot happen in the same clock tick
/// and must cause a different modification time (unless the system /// and must cause a different modification time (unless the system
/// clock jumps back and we get unlucky, which is not impossible but /// clock jumps back and we get unlucky, which is not impossible but
/// but deemed unlikely enough). /// but deemed unlikely enough).
/// - The directory did not contain any child entry that did not have a /// - All direct children of this directory (as returned by
/// corresponding dirstate node. /// `std::fs::read_dir`) either have a corresponding dirstate node, or
/// are ignored by ignore patterns whose hash is in
/// `Header::ignore_patterns_hash`.
/// ///
/// This means that if `std::fs::symlink_metadata` later reports the /// This means that if `std::fs::symlink_metadata` later reports the
/// same modification time, we don’t need to call `std::fs::read_dir` /// same modification time and ignored patterns haven’t changed, a run
/// again for this directory and can iterate child dirstate nodes /// of status that is not listing ignored files can skip calling
/// instead. /// `std::fs::read_dir` again for this directory, iterate child
/// dirstate nodes instead.
state: u8, state: u8,
data: Entry, data: Entry,
} }
#[derive(BytesCast, Copy, Clone)] #[derive(BytesCast, Copy, Clone)]
#[repr(C)] #[repr(C)]
struct Entry { struct Entry {
mode: I32Be, mode: I32Be,

rust/hg-core/src/dirstate_tree/status.rs

cached_directory_mtime: Option<&Timestamp>, cached_directory_mtime: Option<&Timestamp>,
) -> bool { ) -> bool {
if !self.options.list_unknown && !self.options.list_ignored { if !self.options.list_unknown && !self.options.list_ignored {
// All states that we care about listing have corresponding // All states that we care about listing have corresponding
// dirstate entries. // dirstate entries.
// This happens for example with `hg status -mard`. // This happens for example with `hg status -mard`.
return true; return true;
} }
if !self.options.list_ignored
&& self.ignore_patterns_have_changed == Some(false)
{
if let Some(cached_mtime) = cached_directory_mtime { if let Some(cached_mtime) = cached_directory_mtime {
// The dirstate contains a cached mtime for this directory, set by // The dirstate contains a cached mtime for this directory, set
// a previous run of the `status` algorithm which found this // by a previous run of the `status` algorithm which found this
// directory eligible for `read_dir` caching. // directory eligible for `read_dir` caching.
if let Some(meta) = directory_metadata { if let Some(meta) = directory_metadata {
if let Ok(current_mtime) = meta.modified() { if let Ok(current_mtime) = meta.modified() {
if current_mtime == cached_mtime.into() { if current_mtime == cached_mtime.into() {
// The mtime of that directory has not changed since // The mtime of that directory has not changed
// then, which means that the // since then, which means that the results of
// results of `read_dir` should also // `read_dir` should also be unchanged.
// be unchanged.
return true; return true;
} }
} }
} }
} }
}
false false
} }
/// Returns whether the filesystem directory was found to have any entry /// Returns whether all child entries of the filesystem directory have a
/// that does not have a corresponding dirstate tree node. /// corresponding dirstate node or are ignored.
fn traverse_fs_directory_and_dirstate( fn traverse_fs_directory_and_dirstate(
&self, &self,
has_ignored_ancestor: bool, has_ignored_ancestor: bool,
dirstate_nodes: ChildNodesRef<'tree, 'on_disk>, dirstate_nodes: ChildNodesRef<'tree, 'on_disk>,
directory_hg_path: &BorrowedPath<'tree, 'on_disk>, directory_hg_path: &BorrowedPath<'tree, 'on_disk>,
directory_fs_path: &Path, directory_fs_path: &Path,
directory_metadata: Option<&std::fs::Metadata>, directory_metadata: Option<&std::fs::Metadata>,
cached_directory_mtime: Option<&Timestamp>, cached_directory_mtime: Option<&Timestamp>,
let hg_path = let hg_path =
dirstate_node.full_path(self.dmap.on_disk)?; dirstate_node.full_path(self.dmap.on_disk)?;
Ok(self.io_error(error, hg_path)) Ok(self.io_error(error, hg_path))
} }
} }
}) })
.collect::<Result<_, _>>()?; .collect::<Result<_, _>>()?;
// Conservatively don’t let the caller assume that there aren’t // We don’t know, so conservatively say this isn’t the case
// any, since we don’t know. let children_all_have_dirstate_node_or_are_ignored = false;
let directory_has_any_fs_only_entry = true;
return Ok(directory_has_any_fs_only_entry); return Ok(children_all_have_dirstate_node_or_are_ignored);
} }
let mut fs_entries = if let Ok(entries) = self.read_dir( let mut fs_entries = if let Ok(entries) = self.read_dir(
directory_hg_path, directory_hg_path,
directory_fs_path, directory_fs_path,
is_at_repo_root, is_at_repo_root,
) { ) {
entries entries
.base_name(self.dmap.on_disk) .base_name(self.dmap.on_disk)
.unwrap() .unwrap()
.cmp(&fs_entry.base_name) .cmp(&fs_entry.base_name)
}, },
) )
.par_bridge() .par_bridge()
.map(|pair| { .map(|pair| {
use itertools::EitherOrBoth::*; use itertools::EitherOrBoth::*;
let is_fs_only = pair.is_right(); let has_dirstate_node_or_is_ignored;
match pair { match pair {
Both(dirstate_node, fs_entry) => self Both(dirstate_node, fs_entry) => {
.traverse_fs_and_dirstate( self.traverse_fs_and_dirstate(
&fs_entry.full_path, &fs_entry.full_path,
&fs_entry.metadata, &fs_entry.metadata,
dirstate_node, dirstate_node,
has_ignored_ancestor, has_ignored_ancestor,
)?, )?;
has_dirstate_node_or_is_ignored = true
}
Left(dirstate_node) => { Left(dirstate_node) => {
self.traverse_dirstate_only(dirstate_node)? self.traverse_dirstate_only(dirstate_node)?;
has_dirstate_node_or_is_ignored = true;
} }
Right(fs_entry) => self.traverse_fs_only( Right(fs_entry) => {
has_dirstate_node_or_is_ignored = self.traverse_fs_only(
has_ignored_ancestor, has_ignored_ancestor,
directory_hg_path, directory_hg_path,
fs_entry, fs_entry,
), )
} }
Ok(is_fs_only) }
Ok(has_dirstate_node_or_is_ignored)
}) })
.try_reduce(|| false, |a, b| Ok(a || b)) .try_reduce(|| true, |a, b| Ok(a && b))
} }
fn traverse_fs_and_dirstate( fn traverse_fs_and_dirstate(
&self, &self,
fs_path: &Path, fs_path: &Path,
fs_metadata: &std::fs::Metadata, fs_metadata: &std::fs::Metadata,
dirstate_node: NodeRef<'tree, 'on_disk>, dirstate_node: NodeRef<'tree, 'on_disk>,
has_ignored_ancestor: bool, has_ignored_ancestor: bool,
self.outcome self.outcome
.lock() .lock()
.unwrap() .unwrap()
.traversed .traversed
.push(hg_path.detach_from_tree()) .push(hg_path.detach_from_tree())
} }
let is_ignored = has_ignored_ancestor || (self.ignore_fn)(hg_path); let is_ignored = has_ignored_ancestor || (self.ignore_fn)(hg_path);
let is_at_repo_root = false; let is_at_repo_root = false;
let directory_has_any_fs_only_entry = self let children_all_have_dirstate_node_or_are_ignored = self
.traverse_fs_directory_and_dirstate( .traverse_fs_directory_and_dirstate(
is_ignored, is_ignored,
dirstate_node.children(self.dmap.on_disk)?, dirstate_node.children(self.dmap.on_disk)?,
hg_path, hg_path,
fs_path, fs_path,
Some(fs_metadata), Some(fs_metadata),
dirstate_node.cached_directory_mtime(), dirstate_node.cached_directory_mtime(),
is_at_repo_root, is_at_repo_root,
)?; )?;
self.maybe_save_directory_mtime( self.maybe_save_directory_mtime(
directory_has_any_fs_only_entry, children_all_have_dirstate_node_or_are_ignored,
fs_metadata, fs_metadata,
dirstate_node, dirstate_node,
)? )?
} else { } else {
if file_or_symlink && self.matcher.matches(hg_path) { if file_or_symlink && self.matcher.matches(hg_path) {
if let Some(state) = dirstate_node.state()? { if let Some(state) = dirstate_node.state()? {
match state { match state {
EntryState::Added => self EntryState::Added => self
.handle_normal_file(&dirstate_node, fs_metadata)?, .handle_normal_file(&dirstate_node, fs_metadata)?,
// This variant is not used in DirstateMap // This variant is not used in DirstateMap
// nodes // nodes
EntryState::Unknown => unreachable!(), EntryState::Unknown => unreachable!(),
} }
} else { } else {
// `node.entry.is_none()` indicates a "directory" // `node.entry.is_none()` indicates a "directory"
// node, but the filesystem has a file // node, but the filesystem has a file
self.mark_unknown_or_ignored(has_ignored_ancestor, hg_path) self.mark_unknown_or_ignored(
has_ignored_ancestor,
hg_path,
);
} }
} }
for child_node in dirstate_node.children(self.dmap.on_disk)?.iter() for child_node in dirstate_node.children(self.dmap.on_disk)?.iter()
{ {
self.traverse_dirstate_only(child_node)? self.traverse_dirstate_only(child_node)?
} }
} }
Ok(()) Ok(())
} }
fn maybe_save_directory_mtime( fn maybe_save_directory_mtime(
&self, &self,
directory_has_any_fs_only_entry: bool, children_all_have_dirstate_node_or_are_ignored: bool,
directory_metadata: &std::fs::Metadata, directory_metadata: &std::fs::Metadata,
dirstate_node: NodeRef<'tree, 'on_disk>, dirstate_node: NodeRef<'tree, 'on_disk>,
) -> Result<(), DirstateV2ParseError> { ) -> Result<(), DirstateV2ParseError> {
if !directory_has_any_fs_only_entry { if children_all_have_dirstate_node_or_are_ignored {
// All filesystem directory entries from `read_dir` have a // All filesystem directory entries from `read_dir` have a
// corresponding node in the dirstate, so we can reconstitute the // corresponding node in the dirstate, so we can reconstitute the
// names of those entries without calling `read_dir` again. // names of those entries without calling `read_dir` again.
if let (Some(status_start), Ok(directory_mtime)) = ( if let (Some(status_start), Ok(directory_mtime)) = (
&self.filesystem_time_at_status_start, &self.filesystem_time_at_status_start,
directory_metadata.modified(), directory_metadata.modified(),
) { ) {
// Although the Rust standard library’s `SystemTime` type // Although the Rust standard library’s `SystemTime` type
.deleted .deleted
.push(hg_path.detach_from_tree()) .push(hg_path.detach_from_tree())
} }
} }
} }
} }
/// Something in the filesystem has no corresponding dirstate node /// Something in the filesystem has no corresponding dirstate node
///
/// Returns whether that path is ignored
fn traverse_fs_only( fn traverse_fs_only(
&self, &self,
has_ignored_ancestor: bool, has_ignored_ancestor: bool,
directory_hg_path: &HgPath, directory_hg_path: &HgPath,
fs_entry: &DirEntry, fs_entry: &DirEntry,
) { ) -> bool {
let hg_path = directory_hg_path.join(&fs_entry.base_name); let hg_path = directory_hg_path.join(&fs_entry.base_name);
let file_type = fs_entry.metadata.file_type(); let file_type = fs_entry.metadata.file_type();
let file_or_symlink = file_type.is_file() || file_type.is_symlink(); let file_or_symlink = file_type.is_file() || file_type.is_symlink();
if file_type.is_dir() { if file_type.is_dir() {
let is_ignored = let is_ignored =
has_ignored_ancestor || (self.ignore_fn)(&hg_path); has_ignored_ancestor || (self.ignore_fn)(&hg_path);
let traverse_children = if is_ignored { let traverse_children = if is_ignored {
// Descendants of an ignored directory are all ignored // Descendants of an ignored directory are all ignored
&fs_entry.full_path, &fs_entry.full_path,
is_at_repo_root, is_at_repo_root,
) { ) {
children_fs_entries.par_iter().for_each(|child_fs_entry| { children_fs_entries.par_iter().for_each(|child_fs_entry| {
self.traverse_fs_only( self.traverse_fs_only(
is_ignored, is_ignored,
&hg_path, &hg_path,
child_fs_entry, child_fs_entry,
) );
}) })
} }
} }
if self.options.collect_traversed_dirs { if self.options.collect_traversed_dirs {
self.outcome.lock().unwrap().traversed.push(hg_path.into()) self.outcome.lock().unwrap().traversed.push(hg_path.into())
} }
} else if file_or_symlink && self.matcher.matches(&hg_path) { is_ignored
} else {
if file_or_symlink {
if self.matcher.matches(&hg_path) {
self.mark_unknown_or_ignored( self.mark_unknown_or_ignored(
has_ignored_ancestor, has_ignored_ancestor,
&BorrowedPath::InMemory(&hg_path), &BorrowedPath::InMemory(&hg_path),
) )
} else {
// We haven’t computed whether this path is ignored. It
// might not be, and a future run of status might have a
// different matcher that matches it. So treat it as not
// ignored. That is, inhibit readdir caching of the parent
// directory.
false
}
} else {
// This is neither a directory, a plain file, or a symlink.
// Treat it like an ignored file.
true
}
} }
} }
/// Returns whether that path is ignored
fn mark_unknown_or_ignored( fn mark_unknown_or_ignored(
&self, &self,
has_ignored_ancestor: bool, has_ignored_ancestor: bool,
hg_path: &BorrowedPath<'_, 'on_disk>, hg_path: &BorrowedPath<'_, 'on_disk>,
) { ) -> bool {
let is_ignored = has_ignored_ancestor || (self.ignore_fn)(&hg_path); let is_ignored = has_ignored_ancestor || (self.ignore_fn)(&hg_path);
if is_ignored { if is_ignored {
if self.options.list_ignored { if self.options.list_ignored {
self.outcome self.outcome
.lock() .lock()
.unwrap() .unwrap()
.ignored .ignored
.push(hg_path.detach_from_tree()) .push(hg_path.detach_from_tree())
} }
} else { } else {
if self.options.list_unknown { if self.options.list_unknown {
self.outcome self.outcome
.lock() .lock()
.unwrap() .unwrap()
.unknown .unknown
.push(hg_path.detach_from_tree()) .push(hg_path.detach_from_tree())
} }
} }
is_ignored
} }
} }
#[cfg(unix)] // TODO #[cfg(unix)] // TODO
fn mtime_seconds(metadata: &std::fs::Metadata) -> i64 { fn mtime_seconds(metadata: &std::fs::Metadata) -> i64 {
// Going through `Metadata::modified()` would be portable, but would take // Going through `Metadata::modified()` would be portable, but would take
// care to construct a `SystemTime` value with sub-second precision just // care to construct a `SystemTime` value with sub-second precision just
// for us to throw that away here. // for us to throw that away here.