This is an archive of the discontinued Mercurial Phabricator instance.

Differential D8661

rust-status: refactor status into a struct
ClosedPublic

Authored by Alphare on Jun 25 2020, 5:02 AM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG7528699c6ccb: rust-status: refactor status into a struct
Summary

The code for dirstate/status has grown too large for comfort, this is the
first of three patches that try to improve maintainability.
In this patch, refactoring dirstate's status into a struct allows for slimming
down function signatures drastically, keeping the mental (and maintenance)
burden lower, since pretty much all of them shared a few common arguments.

This had the pleasant side-effect of simplifying lifetimes a little. This has
no observable impact on performance.

The next patch will add/improve documentation and refactor some types. I tried
to keep new code down to a minimum in this patch because it's already pretty
big.

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

Alphare created this revision.Jun 25 2020, 5:02 AM
Alphare added a commit: rHG7528699c6ccb: rust-status: refactor status into a struct.Jul 15 2020, 12:13 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG7528699c6ccb: rust-status: refactor status into a struct (authored by Alphare). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 21708

rust/hg-core/src/dirstate/status.rs

OsError(i32), OsError(i32),
BadType(BadType), BadType(BadType),
} }
/// Marker enum used to dispatch new status entries into the right collections. /// Marker enum used to dispatch new status entries into the right collections.
/// Is similar to `crate::EntryState`, but represents the transient state of /// Is similar to `crate::EntryState`, but represents the transient state of
/// entries during the lifetime of a command. /// entries during the lifetime of a command.
#[derive(Debug, Copy, Clone)] #[derive(Debug, Copy, Clone)]
enum Dispatch { pub enum Dispatch {
Unsure, Unsure,
Modified, Modified,
Added, Added,
Removed, Removed,
Deleted, Deleted,
Clean, Clean,
Unknown, Unknown,
Ignored, Ignored,
lazy_static! { lazy_static! {
static ref DEFAULT_WORK: HashSet<&'static HgPath> = { static ref DEFAULT_WORK: HashSet<&'static HgPath> = {
let mut h = HashSet::new(); let mut h = HashSet::new();
h.insert(HgPath::new(b"")); h.insert(HgPath::new(b""));
h h
}; };
} }
#[derive(Debug, Copy, Clone)]
pub struct StatusOptions {
/// Remember the most recent modification timeslot for status, to make
/// sure we won't miss future size-preserving file content modifications
/// that happen within the same timeslot.
pub last_normal_time: i64,
/// Whether we are on a filesystem with UNIX-like exec flags
pub check_exec: bool,
pub list_clean: bool,
pub list_unknown: bool,
pub list_ignored: bool,
/// Whether to collect traversed dirs for applying a callback later.
/// Used by `hg purge` for example.
pub collect_traversed_dirs: bool,
}
#[derive(Debug)]
pub struct DirstateStatus<'a> {
pub modified: Vec<Cow<'a, HgPath>>,
pub added: Vec<Cow<'a, HgPath>>,
pub removed: Vec<Cow<'a, HgPath>>,
pub deleted: Vec<Cow<'a, HgPath>>,
pub clean: Vec<Cow<'a, HgPath>>,
pub ignored: Vec<Cow<'a, HgPath>>,
pub unknown: Vec<Cow<'a, HgPath>>,
pub bad: Vec<(Cow<'a, HgPath>, BadMatch)>,
/// Only filled if `collect_traversed_dirs` is `true`
pub traversed: Vec<HgPathBuf>,
}
#[derive(Debug)]
pub enum StatusError {
IO(std::io::Error),
Path(HgPathError),
Pattern(PatternError),
}
pub type StatusResult<T> = Result<T, StatusError>;
impl From<PatternError> for StatusError {
fn from(e: PatternError) -> Self {
StatusError::Pattern(e)
}
}
impl From<HgPathError> for StatusError {
fn from(e: HgPathError) -> Self {
StatusError::Path(e)
}
}
impl From<std::io::Error> for StatusError {
fn from(e: std::io::Error) -> Self {
StatusError::IO(e)
}
}
impl ToString for StatusError {
fn to_string(&self) -> String {
match self {
StatusError::IO(e) => e.to_string(),
StatusError::Path(e) => e.to_string(),
StatusError::Pattern(e) => e.to_string(),
}
}
}
pub struct Status<'a, M: Matcher + Sync> {
dmap: &'a DirstateMap,
matcher: &'a M,
root_dir: PathBuf,
options: StatusOptions,
ignore_fn: IgnoreFnType<'a>,
}
impl<'a, M> Status<'a, M>
where
M: Matcher + Sync,
{
pub fn new(
dmap: &'a DirstateMap,
matcher: &'a M,
root_dir: PathBuf,
ignore_files: Vec<PathBuf>,
options: StatusOptions,
) -> StatusResult<(Self, Vec<PatternFileWarning>)> {
// Needs to outlive `dir_ignore_fn` since it's captured.
let (ignore_fn, warnings): (IgnoreFnType, _) =
if options.list_ignored || options.list_unknown {
get_ignore_function(ignore_files, &root_dir)?
} else {
(Box::new(|&_| true), vec![])
};
Ok((
Self {
dmap,
matcher,
root_dir,
options,
ignore_fn,
},
warnings,
))
}
pub fn is_ignored(&self, path: impl AsRef<HgPath>) -> bool {
(self.ignore_fn)(path.as_ref())
}
/// Is the path or one of its ancestors ignored?
pub fn dir_ignore(&self, dir: impl AsRef<HgPath>) -> bool {
// Only involve ignore mechanism if we're listing unknowns or ignored.
if self.options.list_ignored || self.options.list_unknown {
if self.is_ignored(&dir) {
true
} else {
for p in find_dirs(dir.as_ref()) {
if self.is_ignored(p) {
return true;
}
}
false
}
} else {
true
}
}
/// Get stat data about the files explicitly specified by match. /// Get stat data about the files explicitly specified by match.
/// TODO subrepos /// TODO subrepos
#[timed] #[timed]
fn walk_explicit<'a>( pub fn walk_explicit(
files: Option<&'a HashSet<&HgPath>>, &self,
dmap: &'a DirstateMap,
root_dir: impl AsRef<Path> + Sync + Send + 'a,
options: StatusOptions,
traversed_sender: crossbeam::Sender<HgPathBuf>, traversed_sender: crossbeam::Sender<HgPathBuf>,
) -> impl ParallelIterator<Item = IoResult<(&'a HgPath, Dispatch)>> { ) -> (
files Vec<(Cow<'a, HgPath>, Dispatch)>,
Vec<(Cow<'a, HgPath>, Dispatch)>,
) {
self.matcher
.file_set()
.unwrap_or(&DEFAULT_WORK) .unwrap_or(&DEFAULT_WORK)
.par_iter() .par_iter()
.map(move |&filename| { .map(|&filename| -> Option<IoResult<_>> {
// TODO normalization // TODO normalization
let normalized = filename; let normalized = filename;
let buf = match hg_path_to_path_buf(normalized) { let buf = match hg_path_to_path_buf(normalized) {
Ok(x) => x, Ok(x) => x,
Err(e) => return Some(Err(e.into())), Err(e) => return Some(Err(e.into())),
}; };
let target = root_dir.as_ref().join(buf); let target = self.root_dir.join(buf);
let st = target.symlink_metadata(); let st = target.symlink_metadata();
let in_dmap = dmap.get(normalized); let in_dmap = self.dmap.get(normalized);
match st { match st {
Ok(meta) => { Ok(meta) => {
let file_type = meta.file_type(); let file_type = meta.file_type();
return if file_type.is_file() || file_type.is_symlink() { return if file_type.is_file() || file_type.is_symlink()
{
if let Some(entry) = in_dmap { if let Some(entry) = in_dmap {
return Some(Ok(( return Some(Ok((
normalized, Cow::Borrowed(normalized),
dispatch_found( dispatch_found(
&normalized, &normalized,
*entry, *entry,
HgMetadata::from_metadata(meta), HgMetadata::from_metadata(meta),
&dmap.copy_map, &self.dmap.copy_map,
options, self.options,
), ),
))); )));
} }
Some(Ok((normalized, Dispatch::Unknown))) Some(Ok((
Cow::Borrowed(normalized),
Dispatch::Unknown,
)))
} else if file_type.is_dir() { } else if file_type.is_dir() {
if options.collect_traversed_dirs { if self.options.collect_traversed_dirs {
traversed_sender traversed_sender
.send(normalized.to_owned()) .send(normalized.to_owned())
.expect("receiver should outlive sender"); .expect("receiver should outlive sender");
} }
Some(Ok(( Some(Ok((
normalized, Cow::Borrowed(normalized),
Dispatch::Directory { Dispatch::Directory {
was_file: in_dmap.is_some(), was_file: in_dmap.is_some(),
}, },
))) )))
} else { } else {
Some(Ok(( Some(Ok((
normalized, Cow::Borrowed(normalized),
Dispatch::Bad(BadMatch::BadType( Dispatch::Bad(BadMatch::BadType(
// TODO do more than unknown // TODO do more than unknown
// Support for all `BadType` variant // Support for all `BadType` variant
// varies greatly between platforms. // varies greatly between platforms.
// So far, no tests check the type and // So far, no tests check the type and
// this should be good enough for most // this should be good enough for most
// users. // users.
BadType::Unknown, BadType::Unknown,
)), )),
))) )))
}; };
} }
Err(_) => { Err(_) => {
if let Some(entry) = in_dmap { if let Some(entry) = in_dmap {
return Some(Ok(( return Some(Ok((
normalized, Cow::Borrowed(normalized),
dispatch_missing(entry.state), dispatch_missing(entry.state),
))); )));
} }
} }
}; };
None None
}) })
.flatten() .flatten()
.filter_map(Result::ok)
.partition(|(_, dispatch)| match dispatch {
Dispatch::Directory { .. } => true,
_ => false,
})
} }
#[derive(Debug, Copy, Clone)] /// Walk the working directory recursively to look for changes compared to
pub struct StatusOptions { /// the current `DirstateMap`.
/// Remember the most recent modification timeslot for status, to make ///
/// sure we won't miss future size-preserving file content modifications /// This takes a mutable reference to the results to account for the
/// that happen within the same timeslot. /// `extend` in timings
pub last_normal_time: i64, #[timed]
/// Whether we are on a filesystem with UNIX-like exec flags pub fn traverse(
pub check_exec: bool, &self,
pub list_clean: bool, path: impl AsRef<HgPath>,
pub list_unknown: bool, old_results: &FastHashMap<Cow<'a, HgPath>, Dispatch>,
pub list_ignored: bool, results: &mut Vec<(Cow<'a, HgPath>, Dispatch)>,
/// Whether to collect traversed dirs for applying a callback later. traversed_sender: crossbeam::Sender<HgPathBuf>,
/// Used by `hg purge` for example. ) -> IoResult<()> {
pub collect_traversed_dirs: bool, // The traversal is done in parallel, so use a channel to gather
// entries. `crossbeam::Sender` is `Sync`, while `mpsc::Sender`
// is not.
let (files_transmitter, files_receiver) =
crossbeam::channel::unbounded();
self.traverse_dir(
&files_transmitter,
path,
&old_results,
traversed_sender,
)?;
// Disconnect the channel so the receiver stops waiting
drop(files_transmitter);
// TODO don't collect. Find a way of replicating the behavior of
// `itertools::process_results`, but for `rayon::ParallelIterator`
let new_results: IoResult<Vec<(Cow<HgPath>, Dispatch)>> =
files_receiver
.into_iter()
.map(|item| {
let (f, d) = item?;
Ok((Cow::Owned(f), d))
})
.collect();
results.par_extend(new_results?);
Ok(())
} }
/// Dispatch a single entry (file, folder, symlink...) found during `traverse`. /// Dispatch a single entry (file, folder, symlink...) found during
/// If the entry is a folder that needs to be traversed, it will be handled /// `traverse`. If the entry is a folder that needs to be traversed, it
/// in a separate thread. /// will be handled in a separate thread.
fn handle_traversed_entry<'a>( fn handle_traversed_entry<'b>(
scope: &rayon::Scope<'a>, &'a self,
files_sender: &'a crossbeam::Sender<IoResult<(HgPathBuf, Dispatch)>>, scope: &rayon::Scope<'b>,
matcher: &'a (impl Matcher + Sync), files_sender: &'b crossbeam::Sender<IoResult<(HgPathBuf, Dispatch)>>,
root_dir: impl AsRef<Path> + Sync + Send + Copy + 'a,
dmap: &'a DirstateMap,
old_results: &'a FastHashMap<Cow<HgPath>, Dispatch>, old_results: &'a FastHashMap<Cow<HgPath>, Dispatch>,
ignore_fn: &'a IgnoreFnType,
dir_ignore_fn: &'a IgnoreFnType,
options: StatusOptions,
filename: HgPathBuf, filename: HgPathBuf,
dir_entry: DirEntry, dir_entry: DirEntry,
traversed_sender: crossbeam::Sender<HgPathBuf>, traversed_sender: crossbeam::Sender<HgPathBuf>,
) -> IoResult<()> { ) -> IoResult<()>
where
'a: 'b,
{
let file_type = dir_entry.file_type()?; let file_type = dir_entry.file_type()?;
let entry_option = dmap.get(&filename); let entry_option = self.dmap.get(&filename);
if filename.as_bytes() == b".hg" { if filename.as_bytes() == b".hg" {
// Could be a directory or a symlink // Could be a directory or a symlink
return Ok(()); return Ok(());
} }
if file_type.is_dir() { if file_type.is_dir() {
handle_traversed_dir( self.handle_traversed_dir(
scope, scope,
files_sender, files_sender,
matcher,
root_dir,
dmap,
old_results, old_results,
ignore_fn,
dir_ignore_fn,
options,
entry_option, entry_option,
filename, filename,
traversed_sender, traversed_sender,
); );
} else if file_type.is_file() || file_type.is_symlink() { } else if file_type.is_file() || file_type.is_symlink() {
if let Some(entry) = entry_option { if let Some(entry) = entry_option {
if matcher.matches_everything() || matcher.matches(&filename) { if self.matcher.matches_everything()
|| self.matcher.matches(&filename)
{
let metadata = dir_entry.metadata()?; let metadata = dir_entry.metadata()?;
files_sender files_sender
.send(Ok(( .send(Ok((
filename.to_owned(), filename.to_owned(),
dispatch_found( dispatch_found(
&filename, &filename,
*entry, *entry,
HgMetadata::from_metadata(metadata), HgMetadata::from_metadata(metadata),
&dmap.copy_map, &self.dmap.copy_map,
options, self.options,
), ),
))) )))
.unwrap(); .unwrap();
} }
} else if (matcher.matches_everything() || matcher.matches(&filename)) } else if (self.matcher.matches_everything()
&& !ignore_fn(&filename) || self.matcher.matches(&filename))
&& !self.is_ignored(&filename)
{ {
if (options.list_ignored || matcher.exact_match(&filename)) if (self.options.list_ignored
&& dir_ignore_fn(&filename) || self.matcher.exact_match(&filename))
&& self.dir_ignore(&filename)
{ {
if options.list_ignored { if self.options.list_ignored {
files_sender files_sender
.send(Ok((filename.to_owned(), Dispatch::Ignored))) .send(Ok((filename.to_owned(), Dispatch::Ignored)))
.unwrap(); .unwrap();
} }
} else if options.list_unknown { } else if self.options.list_unknown {
files_sender files_sender
.send(Ok((filename.to_owned(), Dispatch::Unknown))) .send(Ok((filename.to_owned(), Dispatch::Unknown)))
.unwrap(); .unwrap();
} }
} else if ignore_fn(&filename) && options.list_ignored { } else if self.is_ignored(&filename) && self.options.list_ignored {
files_sender files_sender
.send(Ok((filename.to_owned(), Dispatch::Ignored))) .send(Ok((filename.to_owned(), Dispatch::Ignored)))
.unwrap(); .unwrap();
} }
} else if let Some(entry) = entry_option { } else if let Some(entry) = entry_option {
// Used to be a file or a folder, now something else. // Used to be a file or a folder, now something else.
if matcher.matches_everything() || matcher.matches(&filename) { if self.matcher.matches_everything()
|| self.matcher.matches(&filename)
{
files_sender files_sender
.send(Ok((filename.to_owned(), dispatch_missing(entry.state)))) .send(Ok((
filename.to_owned(),
dispatch_missing(entry.state),
)))
.unwrap(); .unwrap();
} }
} }
Ok(()) Ok(())
} }
/// A directory was found in the filesystem and needs to be traversed /// A directory was found in the filesystem and needs to be traversed
fn handle_traversed_dir<'a>( fn handle_traversed_dir<'b>(
scope: &rayon::Scope<'a>, &'a self,
files_sender: &'a crossbeam::Sender<IoResult<(HgPathBuf, Dispatch)>>, scope: &rayon::Scope<'b>,
matcher: &'a (impl Matcher + Sync), files_sender: &'b crossbeam::Sender<IoResult<(HgPathBuf, Dispatch)>>,
root_dir: impl AsRef<Path> + Sync + Send + Copy + 'a,
dmap: &'a DirstateMap,
old_results: &'a FastHashMap<Cow<HgPath>, Dispatch>, old_results: &'a FastHashMap<Cow<HgPath>, Dispatch>,
ignore_fn: &'a IgnoreFnType,
dir_ignore_fn: &'a IgnoreFnType,
options: StatusOptions,
entry_option: Option<&'a DirstateEntry>, entry_option: Option<&'a DirstateEntry>,
directory: HgPathBuf, directory: HgPathBuf,
traversed_sender: crossbeam::Sender<HgPathBuf>, traversed_sender: crossbeam::Sender<HgPathBuf>,
) { ) where
'a: 'b,
{
scope.spawn(move |_| { scope.spawn(move |_| {
// Nested `if` until `rust-lang/rust#53668` is stable // Nested `if` until `rust-lang/rust#53668` is stable
if let Some(entry) = entry_option { if let Some(entry) = entry_option {
// Used to be a file, is now a folder // Used to be a file, is now a folder
if matcher.matches_everything() || matcher.matches(&directory) { if self.matcher.matches_everything()
|| self.matcher.matches(&directory)
{
files_sender files_sender
.send(Ok(( .send(Ok((
directory.to_owned(), directory.to_owned(),
dispatch_missing(entry.state), dispatch_missing(entry.state),
))) )))
.unwrap(); .unwrap();
} }
} }
// Do we need to traverse it? // Do we need to traverse it?
if !ignore_fn(&directory) || options.list_ignored { if !self.is_ignored(&directory) || self.options.list_ignored {
traverse_dir( self.traverse_dir(
files_sender, files_sender,
matcher,
root_dir,
dmap,
directory, directory,
&old_results, &old_results,
ignore_fn,
dir_ignore_fn,
options,
traversed_sender, traversed_sender,
) )
.unwrap_or_else(|e| files_sender.send(Err(e)).unwrap()) .unwrap_or_else(|e| files_sender.send(Err(e)).unwrap())
} }
}); });
} }
/// Decides whether the directory needs to be listed, and if so handles the /// Decides whether the directory needs to be listed, and if so handles the
/// entries in a separate thread. /// entries in a separate thread.
fn traverse_dir<'a>( fn traverse_dir(
&self,
files_sender: &crossbeam::Sender<IoResult<(HgPathBuf, Dispatch)>>, files_sender: &crossbeam::Sender<IoResult<(HgPathBuf, Dispatch)>>,
matcher: &'a (impl Matcher + Sync),
root_dir: impl AsRef<Path> + Sync + Send + Copy,
dmap: &'a DirstateMap,
directory: impl AsRef<HgPath>, directory: impl AsRef<HgPath>,
old_results: &FastHashMap<Cow<'a, HgPath>, Dispatch>, old_results: &FastHashMap<Cow<HgPath>, Dispatch>,
ignore_fn: &IgnoreFnType,
dir_ignore_fn: &IgnoreFnType,
options: StatusOptions,
traversed_sender: crossbeam::Sender<HgPathBuf>, traversed_sender: crossbeam::Sender<HgPathBuf>,
) -> IoResult<()> { ) -> IoResult<()> {
let directory = directory.as_ref(); let directory = directory.as_ref();
if options.collect_traversed_dirs { if self.options.collect_traversed_dirs {
traversed_sender traversed_sender
.send(directory.to_owned()) .send(directory.to_owned())
.expect("receiver should outlive sender"); .expect("receiver should outlive sender");
} }
let visit_entries = match matcher.visit_children_set(directory) { let visit_entries = match self.matcher.visit_children_set(directory) {
VisitChildrenSet::Empty => return Ok(()), VisitChildrenSet::Empty => return Ok(()),
VisitChildrenSet::This | VisitChildrenSet::Recursive => None, VisitChildrenSet::This | VisitChildrenSet::Recursive => None,
VisitChildrenSet::Set(set) => Some(set), VisitChildrenSet::Set(set) => Some(set),
}; };
let buf = hg_path_to_path_buf(directory)?; let buf = hg_path_to_path_buf(directory)?;
let dir_path = root_dir.as_ref().join(buf); let dir_path = self.root_dir.join(buf);
let skip_dot_hg = !directory.as_bytes().is_empty(); let skip_dot_hg = !directory.as_bytes().is_empty();
let entries = match list_directory(dir_path, skip_dot_hg) { let entries = match list_directory(dir_path, skip_dot_hg) {
Err(e) => match e.kind() { Err(e) => match e.kind() {
ErrorKind::NotFound | ErrorKind::PermissionDenied => { ErrorKind::NotFound | ErrorKind::PermissionDenied => {
files_sender files_sender
.send(Ok(( .send(Ok((
directory.to_owned(), directory.to_owned(),
Dispatch::Bad(BadMatch::OsError( Dispatch::Bad(BadMatch::OsError(
// Unwrapping here is OK because the error always // Unwrapping here is OK because the error
// is a real os error // always is a
// real os error
e.raw_os_error().unwrap(), e.raw_os_error().unwrap(),
)), )),
))) )))
.unwrap(); .unwrap();
return Ok(()); return Ok(());
} }
_ => return Err(e), _ => return Err(e),
}, },
Ok(entries) => entries, Ok(entries) => entries,
}; };
rayon::scope(|scope| -> IoResult<()> { rayon::scope(|scope| -> IoResult<()> {
for (filename, dir_entry) in entries { for (filename, dir_entry) in entries {
if let Some(ref set) = visit_entries { if let Some(ref set) = visit_entries {
if !set.contains(filename.deref()) { if !set.contains(filename.deref()) {
continue; continue;
} }
} }
// TODO normalize // TODO normalize
let filename = if directory.is_empty() { let filename = if directory.is_empty() {
filename.to_owned() filename.to_owned()
} else { } else {
directory.join(&filename) directory.join(&filename)
}; };
if !old_results.contains_key(filename.deref()) { if !old_results.contains_key(filename.deref()) {
handle_traversed_entry( self.handle_traversed_entry(
scope, scope,
files_sender, files_sender,
matcher,
root_dir,
dmap,
old_results, old_results,
ignore_fn,
dir_ignore_fn,
options,
filename, filename,
dir_entry, dir_entry,
traversed_sender.clone(), traversed_sender.clone(),
)?; )?;
} }
} }
Ok(()) Ok(())
}) })
} }
/// Walk the working directory recursively to look for changes compared to the /// This takes a mutable reference to the results to account for the
/// current `DirstateMap`. /// `extend` in timings
///
/// This takes a mutable reference to the results to account for the `extend`
/// in timings
#[timed] #[timed]
fn traverse<'a>( fn handle_unknowns(
matcher: &'a (impl Matcher + Sync), &self,
root_dir: impl AsRef<Path> + Sync + Send + Copy,
dmap: &'a DirstateMap,
path: impl AsRef<HgPath>,
old_results: &FastHashMap<Cow<'a, HgPath>, Dispatch>,
ignore_fn: &IgnoreFnType,
dir_ignore_fn: &IgnoreFnType,
options: StatusOptions,
results: &mut Vec<(Cow<'a, HgPath>, Dispatch)>, results: &mut Vec<(Cow<'a, HgPath>, Dispatch)>,
traversed_sender: crossbeam::Sender<HgPathBuf>,
) -> IoResult<()> { ) -> IoResult<()> {
let root_dir = root_dir.as_ref(); let to_visit: Vec<(&HgPath, &DirstateEntry)> =
if results.is_empty() && self.matcher.matches_everything() {
// The traversal is done in parallel, so use a channel to gather entries. self.dmap.iter().map(|(f, e)| (f.deref(), e)).collect()
// `crossbeam::Sender` is `Sync`, while `mpsc::Sender` is not. } else {
let (files_transmitter, files_receiver) = crossbeam::channel::unbounded(); // Only convert to a hashmap if needed.
let old_results: FastHashMap<_, _> =
traverse_dir( results.iter().cloned().collect();
&files_transmitter, self.dmap
matcher, .iter()
root_dir, .filter_map(move |(f, e)| {
&dmap, if !old_results.contains_key(f.deref())
path, && self.matcher.matches(f)
&old_results, {
&ignore_fn, Some((f.deref(), e))
&dir_ignore_fn, } else {
options, None
traversed_sender, }
)?; })
.collect()
};
// Disconnect the channel so the receiver stops waiting // We walked all dirs under the roots that weren't ignored, and
drop(files_transmitter); // everything that matched was stat'ed and is already in results.
// The rest must thus be ignored or under a symlink.
let path_auditor = PathAuditor::new(&self.root_dir);
// TODO don't collect. Find a way of replicating the behavior of // TODO don't collect. Find a way of replicating the behavior of
// `itertools::process_results`, but for `rayon::ParallelIterator` // `itertools::process_results`, but for `rayon::ParallelIterator`
let new_results: IoResult<Vec<(Cow<'a, HgPath>, Dispatch)>> = let new_results: IoResult<Vec<_>> = to_visit
files_receiver .into_par_iter()
.into_iter() .filter_map(|(filename, entry)| -> Option<IoResult<_>> {
.map(|item| { // Report ignored items in the dmap as long as they are not
let (f, d) = item?; // under a symlink directory.
Ok((Cow::Owned(f), d)) if path_auditor.check(filename) {
// TODO normalize for case-insensitive filesystems
let buf = match hg_path_to_path_buf(filename) {
Ok(x) => x,
Err(e) => return Some(Err(e.into())),
};
Some(Ok((
Cow::Borrowed(filename),
match self.root_dir.join(&buf).symlink_metadata() {
// File was just ignored, no links, and exists
Ok(meta) => {
let metadata = HgMetadata::from_metadata(meta);
dispatch_found(
filename,
*entry,
metadata,
&self.dmap.copy_map,
self.options,
)
}
// File doesn't exist
Err(_) => dispatch_missing(entry.state),
},
)))
} else {
// It's either missing or under a symlink directory which
// we, in this case, report as missing.
Some(Ok((
Cow::Borrowed(filename),
dispatch_missing(entry.state),
)))
}
}) })
.collect(); .collect();
results.par_extend(new_results?); results.par_extend(new_results?);
Ok(()) Ok(())
} }
/// Stat all entries in the `DirstateMap` and mark them for dispatch. /// This takes a mutable reference to the results to account for the
fn stat_dmap_entries( /// `extend` in timings
dmap: &DirstateMap, #[timed]
root_dir: impl AsRef<Path> + Sync + Send, fn extend_from_dmap(
options: StatusOptions, &self,
) -> impl ParallelIterator<Item = IoResult<(&HgPath, Dispatch)>> { results: &mut Vec<(Cow<'a, HgPath>, Dispatch)>,
dmap.par_iter().map(move |(filename, entry)| { ) {
results.par_extend(self.dmap.par_iter().flat_map(
move |(filename, entry)| {
let filename: &HgPath = filename; let filename: &HgPath = filename;
let filename_as_path = hg_path_to_path_buf(filename)?; let filename_as_path = hg_path_to_path_buf(filename)?;
let meta = root_dir.as_ref().join(filename_as_path).symlink_metadata(); let meta =
self.root_dir.join(filename_as_path).symlink_metadata();
match meta { match meta {
Ok(ref m) Ok(ref m)
if !(m.file_type().is_file() if !(m.file_type().is_file()
|| m.file_type().is_symlink()) => || m.file_type().is_symlink()) =>
{ {
Ok((filename, dispatch_missing(entry.state))) Ok((
Cow::Borrowed(filename),
dispatch_missing(entry.state),
))
} }
Ok(m) => Ok(( Ok(m) => Ok((
filename, Cow::Borrowed(filename),
dispatch_found( dispatch_found(
filename, filename,
*entry, *entry,
HgMetadata::from_metadata(m), HgMetadata::from_metadata(m),
&dmap.copy_map, &self.dmap.copy_map,
options, self.options,
), ),
)), )),
Err(ref e) Err(ref e)
if e.kind() == ErrorKind::NotFound if e.kind() == ErrorKind::NotFound
|| e.raw_os_error() == Some(20) => || e.raw_os_error() == Some(20) =>
{ {
// Rust does not yet have an `ErrorKind` for // Rust does not yet have an `ErrorKind` for
// `NotADirectory` (errno 20) // `NotADirectory` (errno 20)
// It happens if the dirstate contains `foo/bar` and // It happens if the dirstate contains `foo/bar`
// foo is not a directory // and foo is not a
Ok((filename, dispatch_missing(entry.state))) // directory
Ok((
Cow::Borrowed(filename),
dispatch_missing(entry.state),
))
} }
Err(e) => Err(e), Err(e) => Err(e),
} }
}) },
} ));
/// This takes a mutable reference to the results to account for the `extend`
/// in timings
#[timed]
fn extend_from_dmap<'a>(
dmap: &'a DirstateMap,
root_dir: impl AsRef<Path> + Sync + Send,
options: StatusOptions,
results: &mut Vec<(Cow<'a, HgPath>, Dispatch)>,
) {
results.par_extend(
stat_dmap_entries(dmap, root_dir, options)
.flatten()
.map(|(filename, dispatch)| (Cow::Borrowed(filename), dispatch)),
);
} }
#[derive(Debug)]
pub struct DirstateStatus<'a> {
pub modified: Vec<Cow<'a, HgPath>>,
pub added: Vec<Cow<'a, HgPath>>,
pub removed: Vec<Cow<'a, HgPath>>,
pub deleted: Vec<Cow<'a, HgPath>>,
pub clean: Vec<Cow<'a, HgPath>>,
pub ignored: Vec<Cow<'a, HgPath>>,
pub unknown: Vec<Cow<'a, HgPath>>,
pub bad: Vec<(Cow<'a, HgPath>, BadMatch)>,
/// Only filled if `collect_traversed_dirs` is `true`
pub traversed: Vec<HgPathBuf>,
} }
#[timed] #[timed]
fn build_response<'a>( fn build_response<'a>(
results: impl IntoIterator<Item = (Cow<'a, HgPath>, Dispatch)>, results: impl IntoIterator<Item = (Cow<'a, HgPath>, Dispatch)>,
traversed: Vec<HgPathBuf>, traversed: Vec<HgPathBuf>,
) -> (Vec<Cow<'a, HgPath>>, DirstateStatus<'a>) { ) -> (Vec<Cow<'a, HgPath>>, DirstateStatus<'a>) {
let mut lookup = vec![]; let mut lookup = vec![];
ignored, ignored,
unknown, unknown,
bad, bad,
traversed, traversed,
}, },
) )
} }
#[derive(Debug)]
pub enum StatusError {
IO(std::io::Error),
Path(HgPathError),
Pattern(PatternError),
}
pub type StatusResult<T> = Result<T, StatusError>;
impl From<PatternError> for StatusError {
fn from(e: PatternError) -> Self {
StatusError::Pattern(e)
}
}
impl From<HgPathError> for StatusError {
fn from(e: HgPathError) -> Self {
StatusError::Path(e)
}
}
impl From<std::io::Error> for StatusError {
fn from(e: std::io::Error) -> Self {
StatusError::IO(e)
}
}
impl ToString for StatusError {
fn to_string(&self) -> String {
match self {
StatusError::IO(e) => e.to_string(),
StatusError::Path(e) => e.to_string(),
StatusError::Pattern(e) => e.to_string(),
}
}
}
/// This takes a mutable reference to the results to account for the `extend`
/// in timings
#[timed]
fn handle_unknowns<'a>(
dmap: &'a DirstateMap,
matcher: &(impl Matcher + Sync),
root_dir: impl AsRef<Path> + Sync + Send + Copy,
options: StatusOptions,
results: &mut Vec<(Cow<'a, HgPath>, Dispatch)>,
) -> IoResult<()> {
let to_visit: Vec<(&HgPath, &DirstateEntry)> = if results.is_empty()
&& matcher.matches_everything()
{
dmap.iter().map(|(f, e)| (f.deref(), e)).collect()
} else {
// Only convert to a hashmap if needed.
let old_results: FastHashMap<_, _> = results.iter().cloned().collect();
dmap.iter()
.filter_map(move |(f, e)| {
if !old_results.contains_key(f.deref()) && matcher.matches(f) {
Some((f.deref(), e))
} else {
None
}
})
.collect()
};
// We walked all dirs under the roots that weren't ignored, and
// everything that matched was stat'ed and is already in results.
// The rest must thus be ignored or under a symlink.
let path_auditor = PathAuditor::new(root_dir);
// TODO don't collect. Find a way of replicating the behavior of
// `itertools::process_results`, but for `rayon::ParallelIterator`
let new_results: IoResult<Vec<_>> = to_visit
.into_par_iter()
.filter_map(|(filename, entry)| -> Option<IoResult<_>> {
// Report ignored items in the dmap as long as they are not
// under a symlink directory.
if path_auditor.check(filename) {
// TODO normalize for case-insensitive filesystems
let buf = match hg_path_to_path_buf(filename) {
Ok(x) => x,
Err(e) => return Some(Err(e.into())),
};
Some(Ok((
Cow::Borrowed(filename),
match root_dir.as_ref().join(&buf).symlink_metadata() {
// File was just ignored, no links, and exists
Ok(meta) => {
let metadata = HgMetadata::from_metadata(meta);
dispatch_found(
filename,
*entry,
metadata,
&dmap.copy_map,
options,
)
}
// File doesn't exist
Err(_) => dispatch_missing(entry.state),
},
)))
} else {
// It's either missing or under a symlink directory which
// we, in this case, report as missing.
Some(Ok((
Cow::Borrowed(filename),
dispatch_missing(entry.state),
)))
}
})
.collect();
results.par_extend(new_results?);
Ok(())
}
/// Get the status of files in the working directory. /// Get the status of files in the working directory.
/// ///
/// This is the current entry-point for `hg-core` and is realistically unusable /// This is the current entry-point for `hg-core` and is realistically unusable
/// outside of a Python context because its arguments need to provide a lot of /// outside of a Python context because its arguments need to provide a lot of
/// information that will not be necessary in the future. /// information that will not be necessary in the future.
#[timed] #[timed]
pub fn status<'a: 'c, 'b: 'c, 'c>( pub fn status<'a>(
dmap: &'a DirstateMap, dmap: &'a DirstateMap,
matcher: &'b (impl Matcher + Sync), matcher: &'a (impl Matcher + Sync),
root_dir: impl AsRef<Path> + Sync + Send + Copy + 'c, root_dir: PathBuf,
ignore_files: Vec<PathBuf>, ignore_files: Vec<PathBuf>,
options: StatusOptions, options: StatusOptions,
) -> StatusResult<( ) -> StatusResult<(
(Vec<Cow<'c, HgPath>>, DirstateStatus<'c>), (Vec<Cow<'a, HgPath>>, DirstateStatus<'a>),
Vec<PatternFileWarning>, Vec<PatternFileWarning>,
)> { )> {
// Needs to outlive `dir_ignore_fn` since it's captured. let (traversed_sender, traversed_receiver) =
let ignore_fn: IgnoreFnType; crossbeam::channel::unbounded();
let (st, warnings) =
// Only involve real ignore mechanism if we're listing unknowns or ignored. Status::new(dmap, matcher, root_dir, ignore_files, options)?;
let (dir_ignore_fn, warnings): (IgnoreFnType, _) = if options.list_ignored
|| options.list_unknown
{
let (ignore, warnings) = get_ignore_function(ignore_files, root_dir)?;
ignore_fn = ignore;
let dir_ignore_fn = Box::new(|dir: &_| {
// Is the path or one of its ancestors ignored?
if ignore_fn(dir) {
true
} else {
for p in find_dirs(dir) {
if ignore_fn(p) {
return true;
}
}
false
}
});
(dir_ignore_fn, warnings)
} else {
ignore_fn = Box::new(|&_| true);
(Box::new(|&_| true), vec![])
};
let files = matcher.file_set();
// `crossbeam::Sender` is `Sync`, while `mpsc::Sender` is not.
let (traversed_sender, traversed_recv) = crossbeam::channel::unbounded();
// Step 1: check the files explicitly mentioned by the user // Step 1: check the files explicitly mentioned by the user
let explicit = walk_explicit( let (work, mut results) = st.walk_explicit(traversed_sender.clone());
files,
&dmap,
root_dir,
options,
traversed_sender.clone(),
);
// Collect results into a `Vec` because we do very few lookups in most
// cases.
let (work, mut results): (Vec<_>, Vec<_>) = explicit
.filter_map(Result::ok)
.map(|(filename, dispatch)| (Cow::Borrowed(filename), dispatch))
.partition(|(_, dispatch)| match dispatch {
Dispatch::Directory { .. } => true,
_ => false,
});
if !work.is_empty() { if !work.is_empty() {
// Hashmaps are quite a bit slower to build than vecs, so only build it // Hashmaps are quite a bit slower to build than vecs, so only build it
// if needed. // if needed.
let old_results = results.iter().cloned().collect(); let old_results = results.iter().cloned().collect();
// Step 2: recursively check the working directory for changes if // Step 2: recursively check the working directory for changes if
// needed // needed
for (dir, dispatch) in work { for (dir, dispatch) in work {
match dispatch { match dispatch {
Dispatch::Directory { was_file } => { Dispatch::Directory { was_file } => {
if was_file { if was_file {
results.push((dir.to_owned(), Dispatch::Removed)); results.push((dir.to_owned(), Dispatch::Removed));
} }
if options.list_ignored if options.list_ignored
|| options.list_unknown && !dir_ignore_fn(&dir) || options.list_unknown && !st.dir_ignore(&dir)
{ {
traverse( st.traverse(
matcher,
root_dir,
&dmap,
&dir, &dir,
&old_results, &old_results,
&ignore_fn,
&dir_ignore_fn,
options,
&mut results, &mut results,
traversed_sender.clone(), traversed_sender.clone(),
)?; )?;
} }
} }
_ => unreachable!("There can only be directories in `work`"), _ => unreachable!("There can only be directories in `work`"),
} }
} }
} }
if !matcher.is_exact() { if !matcher.is_exact() {
// Step 3: Check the remaining files from the dmap. // Step 3: Check the remaining files from the dmap.
// If a dmap file is not in results yet, it was either // If a dmap file is not in results yet, it was either
// a) not matched b) ignored, c) missing, or d) under a // a) not matched b) ignored, c) missing, or d) under a
// symlink directory. // symlink directory.
if options.list_unknown { if options.list_unknown {
handle_unknowns(dmap, matcher, root_dir, options, &mut results)?; st.handle_unknowns(&mut results)?;
} else { } else {
// We may not have walked the full directory tree above, so stat // We may not have walked the full directory tree above, so stat
// and check everything we missed. // and check everything we missed.
extend_from_dmap(&dmap, root_dir, options, &mut results); st.extend_from_dmap(&mut results);
} }
} }
// Close the channel
drop(traversed_sender); drop(traversed_sender);
let traversed_dirs = traversed_recv.into_iter().collect(); let traversed = traversed_receiver.into_iter().collect();
Ok((build_response(results, traversed_dirs), warnings)) Ok((build_response(results, traversed), warnings))
} }

rust/hg-cpython/src/dirstate/status.rs

let ignore_files = ignore_files?; let ignore_files = ignore_files?;
match matcher.get_type(py).name(py).borrow() { match matcher.get_type(py).name(py).borrow() {
"alwaysmatcher" => { "alwaysmatcher" => {
let matcher = AlwaysMatcher; let matcher = AlwaysMatcher;
let ((lookup, status_res), warnings) = status( let ((lookup, status_res), warnings) = status(
&dmap, &dmap,
&matcher, &matcher,
&root_dir, root_dir.to_path_buf(),
ignore_files, ignore_files,
StatusOptions { StatusOptions {
check_exec, check_exec,
last_normal_time, last_normal_time,
list_clean, list_clean,
list_ignored, list_ignored,
list_unknown, list_unknown,
collect_traversed_dirs, collect_traversed_dirs,
.collect(); .collect();
let files = files?; let files = files?;
let matcher = FileMatcher::new(&files) let matcher = FileMatcher::new(&files)
.map_err(|e| PyErr::new::<ValueError, _>(py, e.to_string()))?; .map_err(|e| PyErr::new::<ValueError, _>(py, e.to_string()))?;
let ((lookup, status_res), warnings) = status( let ((lookup, status_res), warnings) = status(
&dmap, &dmap,
&matcher, &matcher,
&root_dir, root_dir.to_path_buf(),
ignore_files, ignore_files,
StatusOptions { StatusOptions {
check_exec, check_exec,
last_normal_time, last_normal_time,
list_clean, list_clean,
list_ignored, list_ignored,
list_unknown, list_unknown,
collect_traversed_dirs, collect_traversed_dirs,
let (matcher, warnings) = let (matcher, warnings) =
IncludeMatcher::new(ignore_patterns, &root_dir) IncludeMatcher::new(ignore_patterns, &root_dir)
.map_err(|e| handle_fallback(py, e.into()))?; .map_err(|e| handle_fallback(py, e.into()))?;
all_warnings.extend(warnings); all_warnings.extend(warnings);
let ((lookup, status_res), warnings) = status( let ((lookup, status_res), warnings) = status(
&dmap, &dmap,
&matcher, &matcher,
&root_dir, root_dir.to_path_buf(),
ignore_files, ignore_files,
StatusOptions { StatusOptions {
check_exec, check_exec,
last_normal_time, last_normal_time,
list_clean, list_clean,
list_ignored, list_ignored,
list_unknown, list_unknown,
collect_traversed_dirs, collect_traversed_dirs,