This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10752

dirstate-tree: Skip readdir() in `hg status -mard`
ClosedPublic

Authored by SimonSapin on May 19 2021, 12:37 PM.

Details

Reviewers
Alphare
Group Reviewers
hg-reviewers
Commits
rHGf27f2afb15da: dirstate-tree: Skip readdir() in `hg status -mard`
Summary

When running the status algorithm in a mode where we don’t list unknown
or ignored files, all we care about are files that are listed in the dirstate.
We can there for skip making expensive calls to readdir() to list the contents
of filesystem directories, and instead only run stat() to get the filesystem
state of files listed in the dirstate. (This state may be an error for files
that don’t exist anymore on the filesystem.)

On 16 CPU threads, this reduces the time spent in the status() function for
hg status -mard on an old snapshot of mozilla-central from ~70ms to ~50ms.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

SimonSapin created this revision.May 19 2021, 12:37 PM
baymax updated this revision to Diff 28176.May 21 2021, 4:37 AM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

Alphare accepted this revision.May 21 2021, 5:54 AM
This revision is now accepted and ready to land.May 21 2021, 5:54 AM
baymax updated this revision to Diff 28275.May 30 2021, 3:27 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

SimonSapin added a commit: rHGf27f2afb15da: dirstate-tree: Skip readdir() in `hg status -mard`.May 31 2021, 3:57 AM
Closed by commit rHGf27f2afb15da: dirstate-tree: Skip readdir() in `hg status -mard` (authored by SimonSapin). · Explain Why
This revision was automatically updated to reflect the committed changes.
alomax added a subscriber: alomax.Oct 29 2021, 10:28 PM

The motive of providing accounts of all ranges on our platform is to give customers what they demand and what they deserve.<br>
cheap csgo accounts
buy csgo prime accounts
buy high tier csgo accounts
cheap valorant accounts
buy gta 5 accounts

Revision Contents
Changeset List

Diff 28287

rust/hg-core/src/dirstate_tree/status.rs

use crate::dirstate::status::IgnoreFnType; use crate::dirstate::status::IgnoreFnType;
use crate::dirstate_tree::dirstate_map::ChildNodesRef; use crate::dirstate_tree::dirstate_map::ChildNodesRef;
use crate::dirstate_tree::dirstate_map::DirstateMap; use crate::dirstate_tree::dirstate_map::DirstateMap;
use crate::dirstate_tree::dirstate_map::NodeRef; use crate::dirstate_tree::dirstate_map::NodeRef;
use crate::dirstate_tree::on_disk::DirstateV2ParseError; use crate::dirstate_tree::on_disk::DirstateV2ParseError;
use crate::matchers::get_ignore_function; use crate::matchers::get_ignore_function;
use crate::matchers::Matcher; use crate::matchers::Matcher;
use crate::utils::files::get_bytes_from_os_string; use crate::utils::files::get_bytes_from_os_string;
use crate::utils::files::get_path_from_bytes;
use crate::utils::hg_path::HgPath; use crate::utils::hg_path::HgPath;
use crate::BadMatch; use crate::BadMatch;
use crate::DirstateStatus; use crate::DirstateStatus;
use crate::EntryState; use crate::EntryState;
use crate::HgPathBuf; use crate::HgPathBuf;
use crate::PatternFileWarning; use crate::PatternFileWarning;
use crate::StatusError; use crate::StatusError;
use crate::StatusOptions; use crate::StatusOptions;
impl<'tree, 'a> StatusCommon<'tree, 'a, '_> { impl<'tree, 'a> StatusCommon<'tree, 'a, '_> {
fn read_dir( fn read_dir(
&self, &self,
hg_path: &HgPath, hg_path: &HgPath,
fs_path: &Path, fs_path: &Path,
is_at_repo_root: bool, is_at_repo_root: bool,
) -> Result<Vec<DirEntry>, ()> { ) -> Result<Vec<DirEntry>, ()> {
DirEntry::read_dir(fs_path, is_at_repo_root).map_err(|error| { DirEntry::read_dir(fs_path, is_at_repo_root)
.map_err(|error| self.io_error(error, hg_path))
}
fn io_error(&self, error: std::io::Error, hg_path: &HgPath) {
let errno = error.raw_os_error().expect("expected real OS error"); let errno = error.raw_os_error().expect("expected real OS error");
self.outcome self.outcome
.lock() .lock()
.unwrap() .unwrap()
.bad .bad
.push((hg_path.to_owned().into(), BadMatch::OsError(errno))) .push((hg_path.to_owned().into(), BadMatch::OsError(errno)))
})
} }
fn traverse_fs_directory_and_dirstate( fn traverse_fs_directory_and_dirstate(
&self, &self,
has_ignored_ancestor: bool, has_ignored_ancestor: bool,
dirstate_nodes: ChildNodesRef<'tree, '_>, dirstate_nodes: ChildNodesRef<'tree, '_>,
directory_hg_path: &'tree HgPath, directory_hg_path: &'tree HgPath,
directory_fs_path: &Path, directory_fs_path: &Path,
is_at_repo_root: bool, is_at_repo_root: bool,
) -> Result<(), DirstateV2ParseError> { ) -> Result<(), DirstateV2ParseError> {
if !self.options.list_unknown && !self.options.list_ignored {
// We only care about files in the dirstate, so we can skip listing
// filesystem directories entirely.
return dirstate_nodes
.par_iter()
.map(|dirstate_node| {
let fs_path = directory_fs_path.join(get_path_from_bytes(
dirstate_node.base_name(self.dmap.on_disk)?.as_bytes(),
));
match std::fs::symlink_metadata(&fs_path) {
Ok(fs_metadata) => self.traverse_fs_and_dirstate(
&fs_path,
&fs_metadata,
dirstate_node,
has_ignored_ancestor,
),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
self.traverse_dirstate_only(dirstate_node)
}
Err(error) => {
let hg_path =
dirstate_node.full_path(self.dmap.on_disk)?;
Ok(self.io_error(error, hg_path))
}
}
})
.collect();
}
let mut fs_entries = if let Ok(entries) = self.read_dir( let mut fs_entries = if let Ok(entries) = self.read_dir(
directory_hg_path, directory_hg_path,
directory_fs_path, directory_fs_path,
is_at_repo_root, is_at_repo_root,
) { ) {
entries entries
} else { } else {
return Ok(()); return Ok(());
}, },
) )
.par_bridge() .par_bridge()
.map(|pair| { .map(|pair| {
use itertools::EitherOrBoth::*; use itertools::EitherOrBoth::*;
match pair { match pair {
Both(dirstate_node, fs_entry) => self Both(dirstate_node, fs_entry) => self
.traverse_fs_and_dirstate( .traverse_fs_and_dirstate(
fs_entry, &fs_entry.full_path,
&fs_entry.metadata,
dirstate_node, dirstate_node,
has_ignored_ancestor, has_ignored_ancestor,
), ),
Left(dirstate_node) => { Left(dirstate_node) => {
self.traverse_dirstate_only(dirstate_node) self.traverse_dirstate_only(dirstate_node)
} }
Right(fs_entry) => Ok(self.traverse_fs_only( Right(fs_entry) => Ok(self.traverse_fs_only(
has_ignored_ancestor, has_ignored_ancestor,
directory_hg_path, directory_hg_path,
fs_entry, fs_entry,
)), )),
} }
}) })
.collect() .collect()
} }
fn traverse_fs_and_dirstate( fn traverse_fs_and_dirstate(
&self, &self,
fs_entry: &DirEntry, fs_path: &Path,
fs_metadata: &std::fs::Metadata,
dirstate_node: NodeRef<'tree, '_>, dirstate_node: NodeRef<'tree, '_>,
has_ignored_ancestor: bool, has_ignored_ancestor: bool,
) -> Result<(), DirstateV2ParseError> { ) -> Result<(), DirstateV2ParseError> {
let hg_path = dirstate_node.full_path(self.dmap.on_disk)?; let hg_path = dirstate_node.full_path(self.dmap.on_disk)?;
let file_type = fs_entry.metadata.file_type(); let file_type = fs_metadata.file_type();
let file_or_symlink = file_type.is_file() || file_type.is_symlink(); let file_or_symlink = file_type.is_file() || file_type.is_symlink();
if !file_or_symlink { if !file_or_symlink {
// If we previously had a file here, it was removed (with // If we previously had a file here, it was removed (with
// `hg rm` or similar) or deleted before it could be // `hg rm` or similar) or deleted before it could be
// replaced by a directory or something else. // replaced by a directory or something else.
self.mark_removed_or_deleted_if_file( self.mark_removed_or_deleted_if_file(
hg_path, hg_path,
dirstate_node.state()?, dirstate_node.state()?,
); );
} }
if file_type.is_dir() { if file_type.is_dir() {
if self.options.collect_traversed_dirs { if self.options.collect_traversed_dirs {
self.outcome.lock().unwrap().traversed.push(hg_path.into()) self.outcome.lock().unwrap().traversed.push(hg_path.into())
} }
let is_ignored = has_ignored_ancestor || (self.ignore_fn)(hg_path); let is_ignored = has_ignored_ancestor || (self.ignore_fn)(hg_path);
let is_at_repo_root = false; let is_at_repo_root = false;
self.traverse_fs_directory_and_dirstate( self.traverse_fs_directory_and_dirstate(
is_ignored, is_ignored,
dirstate_node.children(self.dmap.on_disk)?, dirstate_node.children(self.dmap.on_disk)?,
hg_path, hg_path,
&fs_entry.full_path, fs_path,
is_at_repo_root, is_at_repo_root,
)? )?
} else { } else {
if file_or_symlink && self.matcher.matches(hg_path) { if file_or_symlink && self.matcher.matches(hg_path) {
let full_path = Cow::from(hg_path); let full_path = Cow::from(hg_path);
if let Some(state) = dirstate_node.state()? { if let Some(state) = dirstate_node.state()? {
match state { match state {
EntryState::Added => { EntryState::Added => {
self.outcome.lock().unwrap().added.push(full_path) self.outcome.lock().unwrap().added.push(full_path)
} }
EntryState::Removed => self EntryState::Removed => self
.outcome .outcome
.lock() .lock()
.unwrap() .unwrap()
.removed .removed
.push(full_path), .push(full_path),
EntryState::Merged => self EntryState::Merged => self
.outcome .outcome
.lock() .lock()
.unwrap() .unwrap()
.modified .modified
.push(full_path), .push(full_path),
EntryState::Normal => { EntryState::Normal => self
self.handle_normal_file(&dirstate_node, fs_entry)? .handle_normal_file(&dirstate_node, fs_metadata)?,
}
// This variant is not used in DirstateMap // This variant is not used in DirstateMap
// nodes // nodes
EntryState::Unknown => unreachable!(), EntryState::Unknown => unreachable!(),
} }
} else { } else {
// `node.entry.is_none()` indicates a "directory" // `node.entry.is_none()` indicates a "directory"
// node, but the filesystem has a file // node, but the filesystem has a file
self.mark_unknown_or_ignored( self.mark_unknown_or_ignored(
Ok(()) Ok(())
} }
/// A file with `EntryState::Normal` in the dirstate was found in the /// A file with `EntryState::Normal` in the dirstate was found in the
/// filesystem /// filesystem
fn handle_normal_file( fn handle_normal_file(
&self, &self,
dirstate_node: &NodeRef<'tree, '_>, dirstate_node: &NodeRef<'tree, '_>,
fs_entry: &DirEntry, fs_metadata: &std::fs::Metadata,
) -> Result<(), DirstateV2ParseError> { ) -> Result<(), DirstateV2ParseError> {
// Keep the low 31 bits // Keep the low 31 bits
fn truncate_u64(value: u64) -> i32 { fn truncate_u64(value: u64) -> i32 {
(value & 0x7FFF_FFFF) as i32 (value & 0x7FFF_FFFF) as i32
} }
fn truncate_i64(value: i64) -> i32 { fn truncate_i64(value: i64) -> i32 {
(value & 0x7FFF_FFFF) as i32 (value & 0x7FFF_FFFF) as i32
} }
let entry = dirstate_node let entry = dirstate_node
.entry()? .entry()?
.expect("handle_normal_file called with entry-less node"); .expect("handle_normal_file called with entry-less node");
let full_path = Cow::from(dirstate_node.full_path(self.dmap.on_disk)?); let full_path = Cow::from(dirstate_node.full_path(self.dmap.on_disk)?);
let mode_changed = || { let mode_changed =
self.options.check_exec && entry.mode_changed(&fs_entry.metadata) || self.options.check_exec && entry.mode_changed(fs_metadata);
}; let size_changed = entry.size != truncate_u64(fs_metadata.len());
let size_changed = entry.size != truncate_u64(fs_entry.metadata.len());
if entry.size >= 0 if entry.size >= 0
&& size_changed && size_changed
&& fs_entry.metadata.file_type().is_symlink() && fs_metadata.file_type().is_symlink()
{ {
// issue6456: Size returned may be longer due to encryption // issue6456: Size returned may be longer due to encryption
// on EXT-4 fscrypt. TODO maybe only do it on EXT4? // on EXT-4 fscrypt. TODO maybe only do it on EXT4?
self.outcome.lock().unwrap().unsure.push(full_path) self.outcome.lock().unwrap().unsure.push(full_path)
} else if dirstate_node.has_copy_source() } else if dirstate_node.has_copy_source()
|| entry.is_from_other_parent() || entry.is_from_other_parent()
|| (entry.size >= 0 && (size_changed || mode_changed())) || (entry.size >= 0 && (size_changed || mode_changed()))
{ {
self.outcome.lock().unwrap().modified.push(full_path) self.outcome.lock().unwrap().modified.push(full_path)
} else { } else {
let mtime = mtime_seconds(&fs_entry.metadata); let mtime = mtime_seconds(fs_metadata);
if truncate_i64(mtime) != entry.mtime if truncate_i64(mtime) != entry.mtime
|| mtime == self.options.last_normal_time || mtime == self.options.last_normal_time
{ {
self.outcome.lock().unwrap().unsure.push(full_path) self.outcome.lock().unwrap().unsure.push(full_path)
} else if self.options.list_clean { } else if self.options.list_clean {
self.outcome.lock().unwrap().clean.push(full_path) self.outcome.lock().unwrap().clean.push(full_path)
} }
} }