This is an archive of the discontinued Mercurial Phabricator instance.

Differential D8661

rust-status: refactor status into a struct
ClosedPublic

Authored by Alphare on Jun 25 2020, 5:02 AM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG7528699c6ccb: rust-status: refactor status into a struct
Summary

The code for dirstate/status has grown too large for comfort, this is the
first of three patches that try to improve maintainability.
In this patch, refactoring dirstate's status into a struct allows for slimming
down function signatures drastically, keeping the mental (and maintenance)
burden lower, since pretty much all of them shared a few common arguments.

This had the pleasant side-effect of simplifying lifetimes a little. This has
no observable impact on performance.

The next patch will add/improve documentation and refactor some types. I tried
to keep new code down to a minimum in this patch because it's already pretty
big.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

Alphare created this revision.Jun 25 2020, 5:02 AM
Alphare added a commit: rHG7528699c6ccb: rust-status: refactor status into a struct.Jul 15 2020, 12:13 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG7528699c6ccb: rust-status: refactor status into a struct (authored by Alphare). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 21912

rust/hg-core/src/dirstate/status.rs

OsError(i32), OsError(i32),
BadType(BadType), BadType(BadType),
} }
/// Marker enum used to dispatch new status entries into the right collections. /// Marker enum used to dispatch new status entries into the right collections.
/// Is similar to `crate::EntryState`, but represents the transient state of /// Is similar to `crate::EntryState`, but represents the transient state of
/// entries during the lifetime of a command. /// entries during the lifetime of a command.
#[derive(Debug, Copy, Clone)] #[derive(Debug, Copy, Clone)]
enum Dispatch { pub enum Dispatch {
Unsure, Unsure,
Modified, Modified,
Added, Added,
Removed, Removed,
Deleted, Deleted,
Clean, Clean,
Unknown, Unknown,
Ignored, Ignored,
lazy_static! { lazy_static! {
static ref DEFAULT_WORK: HashSet<&'static HgPath> = { static ref DEFAULT_WORK: HashSet<&'static HgPath> = {
let mut h = HashSet::new(); let mut h = HashSet::new();
h.insert(HgPath::new(b"")); h.insert(HgPath::new(b""));
h h
}; };
} }
#[derive(Debug, Copy, Clone)]
pub struct StatusOptions {
/// Remember the most recent modification timeslot for status, to make
/// sure we won't miss future size-preserving file content modifications
/// that happen within the same timeslot.
pub last_normal_time: i64,
/// Whether we are on a filesystem with UNIX-like exec flags
pub check_exec: bool,
pub list_clean: bool,
pub list_unknown: bool,
pub list_ignored: bool,
/// Whether to collect traversed dirs for applying a callback later.
/// Used by `hg purge` for example.
pub collect_traversed_dirs: bool,
}
#[derive(Debug)]
pub struct DirstateStatus<'a> {
pub modified: Vec<Cow<'a, HgPath>>,
pub added: Vec<Cow<'a, HgPath>>,
pub removed: Vec<Cow<'a, HgPath>>,
pub deleted: Vec<Cow<'a, HgPath>>,
pub clean: Vec<Cow<'a, HgPath>>,
pub ignored: Vec<Cow<'a, HgPath>>,
pub unknown: Vec<Cow<'a, HgPath>>,
pub bad: Vec<(Cow<'a, HgPath>, BadMatch)>,
/// Only filled if `collect_traversed_dirs` is `true`
pub traversed: Vec<HgPathBuf>,
}
#[derive(Debug)]
pub enum StatusError {
IO(std::io::Error),
Path(HgPathError),
Pattern(PatternError),
}
pub type StatusResult<T> = Result<T, StatusError>;
impl From<PatternError> for StatusError {
fn from(e: PatternError) -> Self {
StatusError::Pattern(e)
}
}
impl From<HgPathError> for StatusError {
fn from(e: HgPathError) -> Self {
StatusError::Path(e)
}
}
impl From<std::io::Error> for StatusError {
fn from(e: std::io::Error) -> Self {
StatusError::IO(e)
}
}
impl ToString for StatusError {
fn to_string(&self) -> String {
match self {
StatusError::IO(e) => e.to_string(),
StatusError::Path(e) => e.to_string(),
StatusError::Pattern(e) => e.to_string(),
}
}
}
pub struct Status<'a, M: Matcher + Sync> {
dmap: &'a DirstateMap,
matcher: &'a M,
root_dir: PathBuf,
options: StatusOptions,
ignore_fn: IgnoreFnType<'a>,
}
impl<'a, M> Status<'a, M>
where
M: Matcher + Sync,
{
pub fn new(
dmap: &'a DirstateMap,
matcher: &'a M,
root_dir: PathBuf,
ignore_files: Vec<PathBuf>,
options: StatusOptions,
) -> StatusResult<(Self, Vec<PatternFileWarning>)> {
// Needs to outlive `dir_ignore_fn` since it's captured.
let (ignore_fn, warnings): (IgnoreFnType, _) =
if options.list_ignored || options.list_unknown {
get_ignore_function(ignore_files, &root_dir)?
} else {
(Box::new(|&_| true), vec![])
};
Ok((
Self {
dmap,
matcher,
root_dir,
options,
ignore_fn,
},
warnings,
))
}
pub fn is_ignored(&self, path: impl AsRef<HgPath>) -> bool {
(self.ignore_fn)(path.as_ref())
}
/// Is the path or one of its ancestors ignored?
pub fn dir_ignore(&self, dir: impl AsRef<HgPath>) -> bool {
// Only involve ignore mechanism if we're listing unknowns or ignored.
if self.options.list_ignored || self.options.list_unknown {
if self.is_ignored(&dir) {
true
} else {
for p in find_dirs(dir.as_ref()) {
if self.is_ignored(p) {
return true;
}
}
false
}
} else {
true
}
}
/// Get stat data about the files explicitly specified by match. /// Get stat data about the files explicitly specified by match.
/// TODO subrepos /// TODO subrepos
#[timed] #[timed]
fn walk_explicit<'a>( pub fn walk_explicit(
files: Option<&'a HashSet<&HgPath>>, &self,
dmap: &'a DirstateMap,
root_dir: impl AsRef<Path> + Sync + Send + 'a,
options: StatusOptions,
traversed_sender: crossbeam::Sender<HgPathBuf>, traversed_sender: crossbeam::Sender<HgPathBuf>,
) -> impl ParallelIterator<Item = IoResult<(&'a HgPath, Dispatch)>> { ) -> (
files Vec<(Cow<'a, HgPath>, Dispatch)>,
Vec<(Cow<'a, HgPath>, Dispatch)>,
) {
self.matcher
.file_set()
.unwrap_or(&DEFAULT_WORK) .unwrap_or(&DEFAULT_WORK)
.par_iter() .par_iter()
.map(move |&filename| { .map(|&filename| -> Option<IoResult<_>> {
// TODO normalization // TODO normalization
let normalized = filename; let normalized = filename;
let buf = match hg_path_to_path_buf(normalized) { let buf = match hg_path_to_path_buf(normalized) {
Ok(x) => x, Ok(x) => x,
Err(e) => return Some(Err(e.into())), Err(e) => return Some(Err(e.into())),
}; };
let target = root_dir.as_ref().join(buf); let target = self.root_dir.join(buf);
let st = target.symlink_metadata(); let st = target.symlink_metadata();
let in_dmap = dmap.get(normalized); let in_dmap = self.dmap.get(normalized);
match st { match st {
Ok(meta) => { Ok(meta) => {
let file_type = meta.file_type(); let file_type = meta.file_type();
return if file_type.is_file() || file_type.is_symlink() { return if file_type.is_file() || file_type.is_symlink()
{
if let Some(entry) = in_dmap { if let Some(entry) = in_dmap {
return Some(Ok(( return Some(Ok((
normalized, Cow::Borrowed(normalized),
dispatch_found( dispatch_found(
&normalized, &normalized,
*entry, *entry,
HgMetadata::from_metadata(meta), HgMetadata::from_metadata(meta),
&dmap.copy_map, &self.dmap.copy_map,
options, self.options,
), ),
))); )));
} }
Some(Ok((normalized, Dispatch::Unknown))) Some(Ok((
Cow::Borrowed(normalized),
Dispatch::Unknown,
)))
} else if file_type.is_dir() { } else if file_type.is_dir() {
if options.collect_traversed_dirs { if self.options.collect_traversed_dirs {
traversed_sender traversed_sender
.send(normalized.to_owned()) .send(normalized.to_owned())
.expect("receiver should outlive sender"); .expect("receiver should outlive sender");
} }
Some(Ok(( Some(Ok((
normalized, Cow::Borrowed(normalized),
Dispatch::Directory { Dispatch::Directory {
was_file: in_dmap.is_some(), was_file: in_dmap.is_some(),
}, },
))) )))
} else { } else {
Some(Ok(( Some(Ok((
normalized, Cow::Borrowed(normalized),
Dispatch::Bad(BadMatch::BadType( Dispatch::Bad(BadMatch::BadType(
// TODO do more than unknown // TODO do more than unknown
// Support for all `BadType` variant // Support for all `BadType` variant
// varies greatly between platforms. // varies greatly between platforms.
// So far, no tests check the type and // So far, no tests check the type and
// this should be good enough for most // this should be good enough for most
// users. // users.
BadType::Unknown, BadType::Unknown,
)), )),
))) )))
}; };
} }
Err(_) => { Err(_) => {
if let Some(entry) = in_dmap { if let Some(entry) = in_dmap {
return Some(Ok(( return Some(Ok((
normalized, Cow::Borrowed(normalized),
dispatch_missing(entry.state), dispatch_missing(entry.state),
))); )));
} }
} }
}; };
None None
}) })
.flatten() .flatten()
.filter_map(Result::ok)
.partition(|(_, dispatch)| match dispatch {
Dispatch::Directory { .. } => true,
_ => false,
})
} }
#[derive(Debug, Copy, Clone)] /// Walk the working directory recursively to look for changes compared to
pub struct StatusOptions { /// the current `DirstateMap`.
/// Remember the most recent modification timeslot for status, to make ///
/// sure we won't miss future size-preserving file content modifications /// This takes a mutable reference to the results to account for the
/// that happen within the same timeslot. /// `extend` in timings
pub last_normal_time: i64, #[timed]
/// Whether we are on a filesystem with UNIX-like exec flags pub fn traverse(
pub check_exec: bool, &self,
pub list_clean: bool, path: impl AsRef<HgPath>,
pub list_unknown: bool, old_results: &FastHashMap<Cow<'a, HgPath>, Dispatch>,
pub list_ignored: bool, results: &mut Vec<(Cow<'a, HgPath>, Dispatch)>,
/// Whether to collect traversed dirs for applying a callback later. traversed_sender: crossbeam::Sender<HgPathBuf>,
/// Used by `hg purge` for example. ) -> IoResult<()> {
pub collect_traversed_dirs: bool, // The traversal is done in parallel, so use a channel to gather
// entries. `crossbeam::Sender` is `Sync`, while `mpsc::Sender`
// is not.
let (files_transmitter, files_receiver) =
crossbeam::channel::unbounded();
self.traverse_dir(
&files_transmitter,
path,
&old_results,
traversed_sender,
)?;
// Disconnect the channel so the receiver stops waiting
drop(files_transmitter);
// TODO don't collect. Find a way of replicating the behavior of
// `itertools::process_results`, but for `rayon::ParallelIterator`
let new_results: IoResult<Vec<(Cow<HgPath>, Dispatch)>> =
files_receiver
.into_iter()
.map(|item| {
let (f, d) = item?;
Ok((Cow::Owned(f), d))
})
.collect();
results.par_extend(new_results?);
Ok(())
} }
/// Dispatch a single entry (file, folder, symlink...) found during `traverse`. /// Dispatch a single entry (file, folder, symlink...) found during
/// If the entry is a folder that needs to be traversed, it will be handled /// `traverse`. If the entry is a folder that needs to be traversed, it
/// in a separate thread. /// will be handled in a separate thread.
fn handle_traversed_entry<'a>( fn handle_traversed_entry<'b>(
scope: &rayon::Scope<'a>, &'a self,
files_sender: &'a crossbeam::Sender<IoResult<(HgPathBuf, Dispatch)>>, scope: &rayon::Scope<'b>,
matcher: &'a (impl Matcher + Sync), files_sender: &'b crossbeam::Sender<IoResult<(HgPathBuf, Dispatch)>>,
root_dir: impl AsRef<Path> + Sync + Send + Copy + 'a,
dmap: &'a DirstateMap,
old_results: &'a FastHashMap<Cow<HgPath>, Dispatch>, old_results: &'a FastHashMap<Cow<HgPath>, Dispatch>,
ignore_fn: &'a IgnoreFnType,
dir_ignore_fn: &'a IgnoreFnType,
options: StatusOptions,
filename: HgPathBuf, filename: HgPathBuf,
dir_entry: DirEntry, dir_entry: DirEntry,
traversed_sender: crossbeam::Sender<HgPathBuf>, traversed_sender: crossbeam::Sender<HgPathBuf>,
) -> IoResult<()> { ) -> IoResult<()>
where
'a: 'b,
{
let file_type = dir_entry.file_type()?; let file_type = dir_entry.file_type()?;
let entry_option = dmap.get(&filename); let entry_option = self.dmap.get(&filename);
if filename.as_bytes() == b".hg" { if filename.as_bytes() == b".hg" {
// Could be a directory or a symlink // Could be a directory or a symlink
return Ok(()); return Ok(());
} }
if file_type.is_dir() { if file_type.is_dir() {
handle_traversed_dir( self.handle_traversed_dir(
scope, scope,
files_sender, files_sender,
matcher,
root_dir,
dmap,
old_results, old_results,
ignore_fn,
dir_ignore_fn,
options,
entry_option, entry_option,
filename, filename,
traversed_sender, traversed_sender,
); );
} else if file_type.is_file() || file_type.is_symlink() { } else if file_type.is_file() || file_type.is_symlink() {
if let Some(entry) = entry_option { if let Some(entry) = entry_option {
if matcher.matches_everything() || matcher.matches(&filename) { if self.matcher.matches_everything()
|| self.matcher.matches(&filename)
{
let metadata = dir_entry.metadata()?; let metadata = dir_entry.metadata()?;
files_sender files_sender
.send(Ok(( .send(Ok((
filename.to_owned(), filename.to_owned(),
dispatch_found( dispatch_found(
&filename, &filename,
*entry, *entry,
HgMetadata::from_metadata(metadata), HgMetadata::from_metadata(metadata),
&dmap.copy_map, &self.dmap.copy_map,
options, self.options,
), ),
))) )))
.unwrap(); .unwrap();
} }
} else if (matcher.matches_everything() || matcher.matches(&filename)) } else if (self.matcher.matches_everything()
&& !ignore_fn(&filename) || self.matcher.matches(&filename))
&& !self.is_ignored(&filename)
{ {
if (options.list_ignored || matcher.exact_match(&filename)) if (self.options.list_ignored
&& dir_ignore_fn(&filename) || self.matcher.exact_match(&filename))
&& self.dir_ignore(&filename)
{ {
if options.list_ignored { if self.options.list_ignored {
files_sender files_sender
.send(Ok((filename.to_owned(), Dispatch::Ignored))) .send(Ok((filename.to_owned(), Dispatch::Ignored)))
.unwrap(); .unwrap();
} }
} else if options.list_unknown { } else if self.options.list_unknown {
files_sender files_sender
.send(Ok((filename.to_owned(), Dispatch::Unknown))) .send(Ok((filename.to_owned(), Dispatch::Unknown)))
.unwrap(); .unwrap();
} }
} else if ignore_fn(&filename) && options.list_ignored { } else if self.is_ignored(&filename) && self.options.list_ignored {
files_sender files_sender
.send(Ok((filename.to_owned(), Dispatch::Ignored))) .send(Ok((filename.to_owned(), Dispatch::Ignored)))
.unwrap(); .unwrap();
} }
} else if let Some(entry) = entry_option { } else if let Some(entry) = entry_option {
// Used to be a file or a folder, now something else. // Used to be a file or a folder, now something else.
if matcher.matches_everything() || matcher.matches(&filename) { if self.matcher.matches_everything()
|| self.matcher.matches(&filename)
{
files_sender files_sender
.send(Ok((filename.to_owned(), dispatch_missing(entry.state)))) .send(Ok((
filename.to_owned(),
dispatch_missing(entry.state),
)))
.unwrap(); .unwrap();
} }
} }
Ok(()) Ok(())
} }
/// A directory was found in the filesystem and needs to be traversed /// A directory was found in the filesystem and needs to be traversed
fn handle_traversed_dir<'a>( fn handle_traversed_dir<'b>(
scope: &rayon::Scope<'a>, &'a self,
files_sender: &'a crossbeam::Sender<IoResult<(HgPathBuf, Dispatch)>>, scope: &rayon::Scope<'b>,
matcher: &'a (impl Matcher + Sync), files_sender: &'b crossbeam::Sender<IoResult<(HgPathBuf, Dispatch)>>,
root_dir: impl AsRef<Path> + Sync + Send + Copy + 'a,
dmap: &'a DirstateMap,
old_results: &'a FastHashMap<Cow<HgPath>, Dispatch>, old_results: &'a FastHashMap<Cow<HgPath>, Dispatch>,
ignore_fn: &'a IgnoreFnType,
dir_ignore_fn: &'a IgnoreFnType,
options: StatusOptions,
entry_option: Option<&'a DirstateEntry>, entry_option: Option<&'a DirstateEntry>,
directory: HgPathBuf, directory: HgPathBuf,
traversed_sender: crossbeam::Sender<HgPathBuf>, traversed_sender: crossbeam::Sender<HgPathBuf>,
) { ) where
'a: 'b,
{
scope.spawn(move |_| { scope.spawn(move |_| {
// Nested `if` until `rust-lang/rust#53668` is stable // Nested `if` until `rust-lang/rust#53668` is stable
if let Some(entry) = entry_option { if let Some(entry) = entry_option {
// Used to be a file, is now a folder // Used to be a file, is now a folder
if matcher.matches_everything() || matcher.matches(&directory) { if self.matcher.matches_everything()
|| self.matcher.matches(&directory)
{
files_sender files_sender
.send(Ok(( .send(Ok((
directory.to_owned(), directory.to_owned(),
dispatch_missing(entry.state), dispatch_missing(entry.state),
))) )))
.unwrap(); .unwrap();
} }
} }
// Do we need to traverse it? // Do we need to traverse it?
if !ignore_fn(&directory) || options.list_ignored { if !self.is_ignored(&directory) || self.options.list_ignored {
traverse_dir( self.traverse_dir(
files_sender, files_sender,
matcher,
root_dir,
dmap,
directory, directory,
&old_results, &old_results,
ignore_fn,
dir_ignore_fn,
options,
traversed_sender, traversed_sender,
) )
.unwrap_or_else(|e| files_sender.send(Err(e)).unwrap()) .unwrap_or_else(|e| files_sender.send(Err(e)).unwrap())
} }
}); });
} }
/// Decides whether the directory needs to be listed, and if so handles the /// Decides whether the directory needs to be listed, and if so handles the
/// entries in a separate thread. /// entries in a separate thread.
fn traverse_dir<'a>( fn traverse_dir(
&self,
files_sender: &crossbeam::Sender<IoResult<(HgPathBuf, Dispatch)>>, files_sender: &crossbeam::Sender<IoResult<(HgPathBuf, Dispatch)>>,
matcher: &'a (impl Matcher + Sync),
root_dir: impl AsRef<Path> + Sync + Send + Copy,
dmap: &'a DirstateMap,
directory: impl AsRef<HgPath>, directory: impl AsRef<HgPath>,
old_results: &FastHashMap<Cow<'a, HgPath>, Dispatch>, old_results: &FastHashMap<Cow<HgPath>, Dispatch>,
ignore_fn: &IgnoreFnType,
dir_ignore_fn: &IgnoreFnType,
options: StatusOptions,
traversed_sender: crossbeam::Sender<HgPathBuf>, traversed_sender: crossbeam::Sender<HgPathBuf>,
) -> IoResult<()> { ) -> IoResult<()> {
let directory = directory.as_ref(); let directory = directory.as_ref();
if options.collect_traversed_dirs { if self.options.collect_traversed_dirs {
traversed_sender traversed_sender
.send(directory.to_owned()) .send(directory.to_owned())
.expect("receiver should outlive sender"); .expect("receiver should outlive sender");
} }
let visit_entries = match matcher.visit_children_set(directory) { let visit_entries = match self.matcher.visit_children_set(directory) {
VisitChildrenSet::Empty => return Ok(()), VisitChildrenSet::Empty => return Ok(()),
VisitChildrenSet::This | VisitChildrenSet::Recursive => None, VisitChildrenSet::This | VisitChildrenSet::Recursive => None,
VisitChildrenSet::Set(set) => Some(set), VisitChildrenSet::Set(set) => Some(set),
}; };
let buf = hg_path_to_path_buf(directory)?; let buf = hg_path_to_path_buf(directory)?;
let dir_path = root_dir.as_ref().join(buf); let dir_path = self.root_dir.join(buf);
let skip_dot_hg = !directory.as_bytes().is_empty(); let skip_dot_hg = !directory.as_bytes().is_empty();
let entries = match list_directory(dir_path, skip_dot_hg) { let entries = match list_directory(dir_path, skip_dot_hg) {
Err(e) => match e.kind() { Err(e) => match e.kind() {
ErrorKind::NotFound | ErrorKind::PermissionDenied => { ErrorKind::NotFound | ErrorKind::PermissionDenied => {
files_sender files_sender
.send(Ok(( .send(Ok((
directory.to_owned(), directory.to_owned(),
Dispatch::Bad(BadMatch::OsError( Dispatch::Bad(BadMatch::OsError(
// Unwrapping here is OK because the error always // Unwrapping here is OK because the error
// is a real os error // always is a
// real os error
e.raw_os_error().unwrap(), e.raw_os_error().unwrap(),
)), )),
))) )))
.unwrap(); .unwrap();
return Ok(()); return Ok(());
} }
_ => return Err(e), _ => return Err(e),
}, },
Ok(entries) => entries, Ok(entries) => entries,
}; };
rayon::scope(|scope| -> IoResult<()> { rayon::scope(|scope| -> IoResult<()> {
for (filename, dir_entry) in entries { for (filename, dir_entry) in entries {
if let Some(ref set) = visit_entries { if let Some(ref set) = visit_entries {
if !set.contains(filename.deref()) { if !set.contains(filename.deref()) {
continue; continue;
} }
} }
// TODO normalize // TODO normalize
let filename = if directory.is_empty() { let filename = if directory.is_empty() {
filename.to_owned() filename.to_owned()
} else { } else {
directory.join(&filename) directory.join(&filename)
}; };
if !old_results.contains_key(filename.deref()) { if !old_results.contains_key(filename.deref()) {
handle_traversed_entry( self.handle_traversed_entry(
scope, scope,
files_sender, files_sender,
matcher,
root_dir,
dmap,
old_results, old_results,
ignore_fn,
dir_ignore_fn,
options,
filename, filename,
dir_entry, dir_entry,
traversed_sender.clone(), traversed_sender.clone(),
)?; )?;
} }
} }
Ok(()) Ok(())
}) })
} }
/// Walk the working directory recursively to look for changes compared to the /// This takes a mutable reference to the results to account for the
/// current `DirstateMap`. /// `extend` in timings
///
/// This takes a mutable reference to the results to account for the `extend`
/// in timings
#[timed] #[timed]
fn traverse<'a>( fn handle_unknowns(
matcher: &'a (impl Matcher + Sync), &self,
root_dir: impl AsRef<Path> + Sync + Send + Copy,
dmap: &'a DirstateMap,
path: impl AsRef<HgPath>,
old_results: &FastHashMap<Cow<'a, HgPath>, Dispatch>,
ignore_fn: &IgnoreFnType,
dir_ignore_fn: &IgnoreFnType,
options: StatusOptions,
results: &mut Vec<(Cow<'a, HgPath>, Dispatch)>, results: &mut Vec<(Cow<'a, HgPath>, Dispatch)>,
traversed_sender: crossbeam::Sender<HgPathBuf>,
) -> IoResult<()> { ) -> IoResult<()> {
let root_dir = root_dir.as_ref(); let to_visit: Vec<(&HgPath, &DirstateEntry)> =
if results.is_empty() && self.matcher.matches_everything() {
// The traversal is done in parallel, so use a channel to gather entries. self.dmap.iter().map(|(f, e)| (f.deref(), e)).collect()
// `crossbeam::Sender` is `Sync`, while `mpsc::Sender` is not. } else {
let (files_transmitter, files_receiver) = crossbeam::channel::unbounded(); // Only convert to a hashmap if needed.
let old_results: FastHashMap<_, _> =
traverse_dir( results.iter().cloned().collect();
&files_transmitter, self.dmap
matcher, .iter()
root_dir, .filter_map(move |(f, e)| {
&dmap, if !old_results.contains_key(f.deref())
path, && self.matcher.matches(f)
&old_results, {
&ignore_fn, Some((f.deref(), e))
&dir_ignore_fn, } else {
options, None
traversed_sender, }
)?; })
.collect()
};
// Disconnect the channel so the receiver stops waiting // We walked all dirs under the roots that weren't ignored, and
drop(files_transmitter); // everything that matched was stat'ed and is already in results.
// The rest must thus be ignored or under a symlink.
let path_auditor = PathAuditor::new(&self.root_dir);
// TODO don't collect. Find a way of replicating the behavior of // TODO don't collect. Find a way of replicating the behavior of
// `itertools::process_results`, but for `rayon::ParallelIterator` // `itertools::process_results`, but for `rayon::ParallelIterator`
let new_results: IoResult<Vec<(Cow<'a, HgPath>, Dispatch)>> = let new_results: IoResult<Vec<_>> = to_visit
files_receiver .into_par_iter()
.into_iter() .filter_map(|(filename, entry)| -> Option<IoResult<_>> {
.map(|item| { // Report ignored items in the dmap as long as they are not
let (f, d) = item?; // under a symlink directory.
Ok((Cow::Owned(f), d)) if path_auditor.check(filename) {
// TODO normalize for case-insensitive filesystems
let buf = match hg_path_to_path_buf(filename) {
Ok(x) => x,
Err(e) => return Some(Err(e.into())),
};
Some(Ok((
Cow::Borrowed(filename),
match self.root_dir.join(&buf).symlink_metadata() {
// File was just ignored, no links, and exists
Ok(meta) => {
let metadata = HgMetadata::from_metadata(meta);
dispatch_found(
filename,
*entry,
metadata,
&self.dmap.copy_map,
self.options,
)
}
// File doesn't exist
Err(_) => dispatch_missing(entry.state),
},
)))
} else {
// It's either missing or under a symlink directory which
// we, in this case, report as missing.
Some(Ok((
Cow::Borrowed(filename),
dispatch_missing(entry.state),
)))
}
}) })
.collect(); .collect();
results.par_extend(new_results?); results.par_extend(new_results?);
Ok(()) Ok(())
} }
/// Stat all entries in the `DirstateMap` and mark them for dispatch. /// This takes a mutable reference to the results to account for the
fn stat_dmap_entries( /// `extend` in timings
dmap: &DirstateMap, #[timed]
root_dir: impl AsRef<Path> + Sync + Send, fn extend_from_dmap(
options: StatusOptions, &self,
) -> impl ParallelIterator<Item = IoResult<(&HgPath, Dispatch)>> { results: &mut Vec<(Cow<'a, HgPath>, Dispatch)>,
dmap.par_iter().map(move |(filename, entry)| { ) {
results.par_extend(self.dmap.par_iter().flat_map(
move |(filename, entry)| {
let filename: &HgPath = filename; let filename: &HgPath = filename;
let filename_as_path = hg_path_to_path_buf(filename)?; let filename_as_path = hg_path_to_path_buf(filename)?;
let meta = root_dir.as_ref().join(filename_as_path).symlink_metadata(); let meta =
self.root_dir.join(filename_as_path).symlink_metadata();
match meta { match meta {
Ok(ref m) Ok(ref m)
if !(m.file_type().is_file() if !(m.file_type().is_file()
|| m.file_type().is_symlink()) => || m.file_type().is_symlink()) =>
{ {
Ok((filename, dispatch_missing(entry.state))) Ok((
Cow::Borrowed(filename),
dispatch_missing(entry.state),
))
} }
Ok(m) => Ok(( Ok(m) => Ok((
filename, Cow::Borrowed(filename),
dispatch_found( dispatch_found(
filename, filename,
*entry, *entry,
HgMetadata::from_metadata(m), HgMetadata::from_metadata(m),
&dmap.copy_map, &self.dmap.copy_map,
options, self.options,
), ),
)), )),
Err(ref e) Err(ref e)
if e.kind() == ErrorKind::NotFound if e.kind() == ErrorKind::NotFound
|| e.raw_os_error() == Some(20) => || e.raw_os_error() == Some(20) =>
{ {
// Rust does not yet have an `ErrorKind` for // Rust does not yet have an `ErrorKind` for
// `NotADirectory` (errno 20) // `NotADirectory` (errno 20)
// It happens if the dirstate contains `foo/bar` and // It happens if the dirstate contains `foo/bar`
// foo is not a directory // and foo is not a
Ok((filename, dispatch_missing(entry.state))) // directory
Ok((
Cow::Borrowed(filename),
dispatch_missing(entry.state),
))
} }
Err(e) => Err(e), Err(e) => Err(e),
} }
}) },
} ));
/// This takes a mutable reference to the results to account for the `extend`
/// in timings
#[timed]
fn extend_from_dmap<'a>(
dmap: &'a DirstateMap,
root_dir: impl AsRef<Path> + Sync + Send,
options: StatusOptions,
results: &mut Vec<(Cow<'a, HgPath>, Dispatch)>,
) {
results.par_extend(
stat_dmap_entries(dmap, root_dir, options)
.flatten()
.map(|(filename, dispatch)| (Cow::Borrowed(filename), dispatch)),
);
} }
#[derive(Debug)]
pub struct DirstateStatus<'a> {
pub modified: Vec<Cow<'a, HgPath>>,
pub added: Vec<Cow<'a, HgPath>>,
pub removed: Vec<Cow<'a, HgPath>>,
pub deleted: Vec<Cow<'a, HgPath>>,
pub clean: Vec<Cow<'a, HgPath>>,
pub ignored: Vec<Cow<'a, HgPath>>,
pub unknown: Vec<Cow<'a, HgPath>>,
pub bad: Vec<(Cow<'a, HgPath>, BadMatch)>,
/// Only filled if `collect_traversed_dirs` is `true`
pub traversed: Vec<HgPathBuf>,
} }
#[timed] #[timed]
fn build_response<'a>( fn build_response<'a>(
results: impl IntoIterator<Item = (Cow<'a, HgPath>, Dispatch)>, results: impl IntoIterator<Item = (Cow<'a, HgPath>, Dispatch)>,
traversed: Vec<HgPathBuf>, traversed: Vec<HgPathBuf>,
) -> (Vec<Cow<'a, HgPath>>, DirstateStatus<'a>) { ) -> (Vec<Cow<'a, HgPath>>, DirstateStatus<'a>) {
let mut lookup = vec![]; let mut lookup = vec![];
ignored, ignored,
unknown, unknown,
bad, bad,
traversed, traversed,
}, },
) )
} }
#[derive(Debug)]
pub enum StatusError {
IO(std::io::Error),
Path(HgPathError),
Pattern(PatternError),
}
pub type StatusResult<T> = Result<T, StatusError>;
impl From<PatternError> for StatusError {
fn from(e: PatternError) -> Self {
StatusError::Pattern(e)
}
}
impl From<HgPathError> for StatusError {
fn from(e: HgPathError) -> Self {
StatusError::Path(e)
}
}
impl From<std::io::Error> for StatusError {
fn from(e: std::io::Error) -> Self {
StatusError::IO(e)
}
}
impl ToString for StatusError {
fn to_string(&self) -> String {
match self {
StatusError::IO(e) => e.to_string(),
StatusError::Path(e) => e.to_string(),
StatusError::Pattern(e) => e.to_string(),
}
}
}
/// This takes a mutable reference to the results to account for the `extend`
/// in timings
#[timed]
fn handle_unknowns<'a>(
dmap: &'a DirstateMap,
matcher: &(impl Matcher + Sync),
root_dir: impl AsRef<Path> + Sync + Send + Copy,
options: StatusOptions,
results: &mut Vec<(Cow<'a, HgPath>, Dispatch)>,
) -> IoResult<()> {
let to_visit: Vec<(&HgPath, &DirstateEntry)> = if results.is_empty()
&& matcher.matches_everything()
{
dmap.iter().map(|(f, e)| (f.deref(), e)).collect()
} else {
// Only convert to a hashmap if needed.
let old_results: FastHashMap<_, _> = results.iter().cloned().collect();
dmap.iter()
.filter_map(move |(f, e)| {
if !old_results.contains_key(f.deref()) && matcher.matches(f) {
Some((f.deref(), e))
} else {
None
}
})
.collect()
};
// We walked all dirs under the roots that weren't ignored, and
// everything that matched was stat'ed and is already in results.
// The rest must thus be ignored or under a symlink.
let path_auditor = PathAuditor::new(root_dir);
// TODO don't collect. Find a way of replicating the behavior of
// `itertools::process_results`, but for `rayon::ParallelIterator`
let new_results: IoResult<Vec<_>> = to_visit
.into_par_iter()
.filter_map(|(filename, entry)| -> Option<IoResult<_>> {
// Report ignored items in the dmap as long as they are not
// under a symlink directory.
if path_auditor.check(filename) {
// TODO normalize for case-insensitive filesystems
let buf = match hg_path_to_path_buf(filename) {
Ok(x) => x,
Err(e) => return Some(Err(e.into())),
};
Some(Ok((
Cow::Borrowed(filename),
match root_dir.as_ref().join(&buf).symlink_metadata() {
// File was just ignored, no links, and exists
Ok(meta) => {
let metadata = HgMetadata::from_metadata(meta);
dispatch_found(
filename,
*entry,
metadata,
&dmap.copy_map,
options,
)
}
// File doesn't exist
Err(_) => dispatch_missing(entry.state),
},
)))
} else {
// It's either missing or under a symlink directory which
// we, in this case, report as missing.
Some(Ok((
Cow::Borrowed(filename),
dispatch_missing(entry.state),
)))
}
})
.collect();
results.par_extend(new_results?);
Ok(())
}
/// Get the status of files in the working directory. /// Get the status of files in the working directory.
/// ///
/// This is the current entry-point for `hg-core` and is realistically unusable /// This is the current entry-point for `hg-core` and is realistically unusable
/// outside of a Python context because its arguments need to provide a lot of /// outside of a Python context because its arguments need to provide a lot of
/// information that will not be necessary in the future. /// information that will not be necessary in the future.
#[timed] #[timed]
pub fn status<'a: 'c, 'b: 'c, 'c>( pub fn status<'a>(
dmap: &'a DirstateMap, dmap: &'a DirstateMap,
matcher: &'b (impl Matcher + Sync), matcher: &'a (impl Matcher + Sync),
root_dir: impl AsRef<Path> + Sync + Send + Copy + 'c, root_dir: PathBuf,
ignore_files: Vec<PathBuf>, ignore_files: Vec<PathBuf>,
options: StatusOptions, options: StatusOptions,
) -> StatusResult<( ) -> StatusResult<(
(Vec<Cow<'c, HgPath>>, DirstateStatus<'c>), (Vec<Cow<'a, HgPath>>, DirstateStatus<'a>),
Vec<PatternFileWarning>, Vec<PatternFileWarning>,
)> { )> {
// Needs to outlive `dir_ignore_fn` since it's captured. let (traversed_sender, traversed_receiver) =
let ignore_fn: IgnoreFnType; crossbeam::channel::unbounded();
let (st, warnings) =
// Only involve real ignore mechanism if we're listing unknowns or ignored. Status::new(dmap, matcher, root_dir, ignore_files, options)?;
let (dir_ignore_fn, warnings): (IgnoreFnType, _) = if options.list_ignored
|| options.list_unknown
{
let (ignore, warnings) = get_ignore_function(ignore_files, root_dir)?;
ignore_fn = ignore;
let dir_ignore_fn = Box::new(|dir: &_| {
// Is the path or one of its ancestors ignored?
if ignore_fn(dir) {
true
} else {
for p in find_dirs(dir) {
if ignore_fn(p) {
return true;
}
}
false
}
});
(dir_ignore_fn, warnings)
} else {
ignore_fn = Box::new(|&_| true);
(Box::new(|&_| true), vec![])
};
let files = matcher.file_set();
// `crossbeam::Sender` is `Sync`, while `mpsc::Sender` is not.
let (traversed_sender, traversed_recv) = crossbeam::channel::unbounded();
// Step 1: check the files explicitly mentioned by the user // Step 1: check the files explicitly mentioned by the user
let explicit = walk_explicit( let (work, mut results) = st.walk_explicit(traversed_sender.clone());
files,
&dmap,
root_dir,
options,
traversed_sender.clone(),
);
// Collect results into a `Vec` because we do very few lookups in most
// cases.
let (work, mut results): (Vec<_>, Vec<_>) = explicit
.filter_map(Result::ok)
.map(|(filename, dispatch)| (Cow::Borrowed(filename), dispatch))
.partition(|(_, dispatch)| match dispatch {
Dispatch::Directory { .. } => true,
_ => false,
});
if !work.is_empty() { if !work.is_empty() {
// Hashmaps are quite a bit slower to build than vecs, so only build it // Hashmaps are quite a bit slower to build than vecs, so only build it
// if needed. // if needed.
let old_results = results.iter().cloned().collect(); let old_results = results.iter().cloned().collect();
// Step 2: recursively check the working directory for changes if // Step 2: recursively check the working directory for changes if
// needed // needed
for (dir, dispatch) in work { for (dir, dispatch) in work {
match dispatch { match dispatch {
Dispatch::Directory { was_file } => { Dispatch::Directory { was_file } => {
if was_file { if was_file {
results.push((dir.to_owned(), Dispatch::Removed)); results.push((dir.to_owned(), Dispatch::Removed));
} }
if options.list_ignored if options.list_ignored
|| options.list_unknown && !dir_ignore_fn(&dir) || options.list_unknown && !st.dir_ignore(&dir)
{ {
traverse( st.traverse(
matcher,
root_dir,
&dmap,
&dir, &dir,
&old_results, &old_results,
&ignore_fn,
&dir_ignore_fn,
options,
&mut results, &mut results,
traversed_sender.clone(), traversed_sender.clone(),
)?; )?;
} }
} }
_ => unreachable!("There can only be directories in `work`"), _ => unreachable!("There can only be directories in `work`"),
} }
} }
} }
if !matcher.is_exact() { if !matcher.is_exact() {
// Step 3: Check the remaining files from the dmap. // Step 3: Check the remaining files from the dmap.
// If a dmap file is not in results yet, it was either // If a dmap file is not in results yet, it was either
// a) not matched b) ignored, c) missing, or d) under a // a) not matched b) ignored, c) missing, or d) under a
// symlink directory. // symlink directory.
if options.list_unknown { if options.list_unknown {
handle_unknowns(dmap, matcher, root_dir, options, &mut results)?; st.handle_unknowns(&mut results)?;
} else { } else {
// We may not have walked the full directory tree above, so stat // We may not have walked the full directory tree above, so stat
// and check everything we missed. // and check everything we missed.
extend_from_dmap(&dmap, root_dir, options, &mut results); st.extend_from_dmap(&mut results);
} }
} }
// Close the channel
drop(traversed_sender); drop(traversed_sender);
let traversed_dirs = traversed_recv.into_iter().collect(); let traversed = traversed_receiver.into_iter().collect();
Ok((build_response(results, traversed_dirs), warnings)) Ok((build_response(results, traversed), warnings))
} }

rust/hg-cpython/src/dirstate/status.rs

let ignore_files = ignore_files?; let ignore_files = ignore_files?;
match matcher.get_type(py).name(py).borrow() { match matcher.get_type(py).name(py).borrow() {
"alwaysmatcher" => { "alwaysmatcher" => {
let matcher = AlwaysMatcher; let matcher = AlwaysMatcher;
let ((lookup, status_res), warnings) = status( let ((lookup, status_res), warnings) = status(
&dmap, &dmap,
&matcher, &matcher,
&root_dir, root_dir.to_path_buf(),
ignore_files, ignore_files,
StatusOptions { StatusOptions {
check_exec, check_exec,
last_normal_time, last_normal_time,
list_clean, list_clean,
list_ignored, list_ignored,
list_unknown, list_unknown,
collect_traversed_dirs, collect_traversed_dirs,
.collect(); .collect();
let files = files?; let files = files?;
let matcher = FileMatcher::new(&files) let matcher = FileMatcher::new(&files)
.map_err(|e| PyErr::new::<ValueError, _>(py, e.to_string()))?; .map_err(|e| PyErr::new::<ValueError, _>(py, e.to_string()))?;
let ((lookup, status_res), warnings) = status( let ((lookup, status_res), warnings) = status(
&dmap, &dmap,
&matcher, &matcher,
&root_dir, root_dir.to_path_buf(),
ignore_files, ignore_files,
StatusOptions { StatusOptions {
check_exec, check_exec,
last_normal_time, last_normal_time,
list_clean, list_clean,
list_ignored, list_ignored,
list_unknown, list_unknown,
collect_traversed_dirs, collect_traversed_dirs,
let (matcher, warnings) = let (matcher, warnings) =
IncludeMatcher::new(ignore_patterns, &root_dir) IncludeMatcher::new(ignore_patterns, &root_dir)
.map_err(|e| handle_fallback(py, e.into()))?; .map_err(|e| handle_fallback(py, e.into()))?;
all_warnings.extend(warnings); all_warnings.extend(warnings);
let ((lookup, status_res), warnings) = status( let ((lookup, status_res), warnings) = status(
&dmap, &dmap,
&matcher, &matcher,
&root_dir, root_dir.to_path_buf(),
ignore_files, ignore_files,
StatusOptions { StatusOptions {
check_exec, check_exec,
last_normal_time, last_normal_time,
list_clean, list_clean,
list_ignored, list_ignored,
list_unknown, list_unknown,
collect_traversed_dirs, collect_traversed_dirs,