This is an archive of the discontinued Mercurial Phabricator instance.

Differential D8251

rust-status: traverse working directory in parallel
ClosedPublic

Authored by Alphare on Mar 6 2020, 12:51 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHGfe7d2cf0b429: rust-status: traverse working directory in parallel
Summary

Using rayon for this task ensures that we are using the same work-stealing
threadpool for everything.

This change introduces crossbeam as an explicit dependency, although it is
already a dependency of rayon. It provides better structures for
multi-threaded tasks than the stdlib.

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

Alphare created this revision.Mar 6 2020, 12:51 PM
Alphare updated this revision to Diff 20708.Mar 11 2020, 11:52 AM
Alphare added a commit: rHGfe7d2cf0b429: rust-status: traverse working directory in parallel.Mar 11 2020, 12:25 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHGfe7d2cf0b429: rust-status: traverse working directory in parallel (authored by Alphare). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 20565

rust/Cargo.lock

dependencies = [ dependencies = [
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
"python27-sys 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "python27-sys 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"python3-sys 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "python3-sys 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]] [[package]]
name = "crossbeam"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "crossbeam-channel"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "crossbeam-deque" name = "crossbeam-deque"
version = "0.7.2" version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]] [[package]]
name = "hg-core" name = "hg-core"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)", "cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)",
"hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
"memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.66" version = "0.2.66"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "maybe-uninit"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "memchr" name = "memchr"
version = "2.3.0" version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "memmap" name = "memmap"
version = "0.7.0" version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" "checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
"checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" "checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
"checksum cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)" = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" "checksum cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)" = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd"
"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
"checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" "checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9"
"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
"checksum cpython 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bfaf3847ab963e40c4f6dd8d6be279bdf74007ae2413786a0dcbb28c52139a95" "checksum cpython 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bfaf3847ab963e40c4f6dd8d6be279bdf74007ae2413786a0dcbb28c52139a95"
"checksum crossbeam 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e"
"checksum crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061"
"checksum crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3aa945d63861bfe624b55d153a39684da1e8c0bc8fba932f7ee3a3c16cea3ca" "checksum crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3aa945d63861bfe624b55d153a39684da1e8c0bc8fba932f7ee3a3c16cea3ca"
"checksum crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac" "checksum crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac"
"checksum crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db" "checksum crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db"
"checksum crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4" "checksum crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4"
"checksum ctor 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "cd8ce37ad4184ab2ce004c33bf6379185d3b1c95801cab51026bd271bf68eedc" "checksum ctor 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "cd8ce37ad4184ab2ce004c33bf6379185d3b1c95801cab51026bd271bf68eedc"
"checksum difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" "checksum difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198"
"checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" "checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
"checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" "checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
"checksum getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" "checksum getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb"
"checksum hermit-abi 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eff2656d88f158ce120947499e971d743c05dbcbed62e5bd2f38f1698bbc3772" "checksum hermit-abi 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eff2656d88f158ce120947499e971d743c05dbcbed62e5bd2f38f1698bbc3772"
"checksum hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "023b39be39e3a2da62a94feb433e91e8bcd37676fbc8bea371daf52b7a769a3e" "checksum hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "023b39be39e3a2da62a94feb433e91e8bcd37676fbc8bea371daf52b7a769a3e"
"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" "checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558"
"checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
"checksum memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223" "checksum memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223"
"checksum memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" "checksum memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
"checksum memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9" "checksum memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9"
"checksum num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" "checksum num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096"
"checksum num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" "checksum num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6"
"checksum output_vt100 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9" "checksum output_vt100 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9"
"checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" "checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
"checksum pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3f81e1644e1b54f5a68959a29aa86cde704219254669da328ecfdf6a1f09d427" "checksum pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3f81e1644e1b54f5a68959a29aa86cde704219254669da328ecfdf6a1f09d427"

rust/hg-core/Cargo.toml

libc = { version = "0.2.66", optional = true } libc = { version = "0.2.66", optional = true }
memchr = "2.2.0" memchr = "2.2.0"
rand = "0.6.5" rand = "0.6.5"
rand_pcg = "0.1.1" rand_pcg = "0.1.1"
rayon = "1.3.0" rayon = "1.3.0"
regex = "1.1.0" regex = "1.1.0"
twox-hash = "1.5.0" twox-hash = "1.5.0"
same-file = "1.0.6" same-file = "1.0.6"
crossbeam = "0.7.3"
[dev-dependencies] [dev-dependencies]
clap = "*" clap = "*"
memmap = "0.7.0" memmap = "0.7.0"
pretty_assertions = "0.6.1" pretty_assertions = "0.6.1"
tempfile = "3.1.0" tempfile = "3.1.0"
[build-dependencies] [build-dependencies]
cc = { version = "1.0.48", optional = true } cc = { version = "1.0.48", optional = true }
[features] [features]
default = [] default = []
with-re2 = ["cc", "libc"] with-re2 = ["cc", "libc"]

rust/hg-core/src/dirstate/status.rs

/// Whether we are on a filesystem with UNIX-like exec flags /// Whether we are on a filesystem with UNIX-like exec flags
pub check_exec: bool, pub check_exec: bool,
pub list_clean: bool, pub list_clean: bool,
pub list_unknown: bool, pub list_unknown: bool,
pub list_ignored: bool, pub list_ignored: bool,
} }
/// Dispatch a single entry (file, folder, symlink...) found during `traverse`. /// Dispatch a single entry (file, folder, symlink...) found during `traverse`.
/// If the entry is a folder that needs to be traversed, it will be pushed into /// If the entry is a folder that needs to be traversed, it will be handled
/// `work`. /// in a separate thread.
fn handle_traversed_entry<'a>( fn handle_traversed_entry<'a>(
dir_entry: &DirEntry, scope: &rayon::Scope<'a>,
matcher: &(impl Matcher + Sync), files_sender: &'a crossbeam::Sender<IoResult<(HgPathBuf, Dispatch)>>,
root_dir: impl AsRef<Path>, matcher: &'a (impl Matcher + Sync),
dmap: &DirstateMap, root_dir: impl AsRef<Path> + Sync + Send + Copy + 'a,
filename: impl AsRef<HgPath>, dmap: &'a DirstateMap,
old_results: &FastHashMap<Cow<'a, HgPath>, Dispatch>, old_results: &'a FastHashMap<Cow<HgPath>, Dispatch>,
ignore_fn: &(impl for<'r> Fn(&'r HgPath) -> bool + Sync), ignore_fn: &'a (impl for<'r> Fn(&'r HgPath) -> bool + Sync),
dir_ignore_fn: &(impl for<'r> Fn(&'r HgPath) -> bool + Sync), dir_ignore_fn: &'a (impl for<'r> Fn(&'r HgPath) -> bool + Sync),
options: StatusOptions, options: StatusOptions,
) -> IoResult<Vec<(Cow<'a, HgPath>, Dispatch)>> { filename: HgPathBuf,
dir_entry: DirEntry,
) -> IoResult<()> {
let file_type = dir_entry.file_type()?; let file_type = dir_entry.file_type()?;
let filename = filename.as_ref(); let entry_option = dmap.get(&filename);
let entry_option = dmap.get(filename);
if file_type.is_dir() { if file_type.is_dir() {
// Do we need to traverse it? handle_traversed_dir(
if !ignore_fn(&filename) || options.list_ignored { scope,
return traverse_dir( files_sender,
matcher, matcher,
root_dir, root_dir,
dmap, dmap,
filename.to_owned(), old_results,
&old_results,
ignore_fn, ignore_fn,
dir_ignore_fn, dir_ignore_fn,
options, options,
entry_option,
filename,
); );
}
// Nested `if` until `rust-lang/rust#53668` is stable
if let Some(entry) = entry_option {
// Used to be a file, is now a folder
if matcher.matches_everything() || matcher.matches(&filename) {
return Ok(vec![(
Cow::Owned(filename.to_owned()),
dispatch_missing(entry.state),
)]);
}
}
} else if file_type.is_file() || file_type.is_symlink() { } else if file_type.is_file() || file_type.is_symlink() {
if let Some(entry) = entry_option { if let Some(entry) = entry_option {
if matcher.matches_everything() || matcher.matches(&filename) { if matcher.matches_everything() || matcher.matches(&filename) {
let metadata = dir_entry.metadata()?; let metadata = dir_entry.metadata()?;
return Ok(vec![( files_sender
Cow::Owned(filename.to_owned()), .send(Ok((
filename.to_owned(),
dispatch_found( dispatch_found(
&filename, &filename,
*entry, *entry,
HgMetadata::from_metadata(metadata), HgMetadata::from_metadata(metadata),
&dmap.copy_map, &dmap.copy_map,
options, options,
), ),
)]); )))
.unwrap();
} }
} else if (matcher.matches_everything() || matcher.matches(&filename)) } else if (matcher.matches_everything() || matcher.matches(&filename))
&& !ignore_fn(&filename) && !ignore_fn(&filename)
{ {
if (options.list_ignored || matcher.exact_match(&filename)) if (options.list_ignored || matcher.exact_match(&filename))
&& dir_ignore_fn(&filename) && dir_ignore_fn(&filename)
{ {
if options.list_ignored { if options.list_ignored {
return Ok(vec![( files_sender
Cow::Owned(filename.to_owned()), .send(Ok((filename.to_owned(), Dispatch::Ignored)))
Dispatch::Ignored, .unwrap();
)]);
} }
} else { } else {
return Ok(vec![( files_sender
Cow::Owned(filename.to_owned()), .send(Ok((filename.to_owned(), Dispatch::Unknown)))
Dispatch::Unknown, .unwrap();
)]);
} }
} else if ignore_fn(&filename) && options.list_ignored { } else if ignore_fn(&filename) && options.list_ignored {
return Ok(vec![( files_sender
Cow::Owned(filename.to_owned()), .send(Ok((filename.to_owned(), Dispatch::Ignored)))
Dispatch::Ignored, .unwrap();
)]);
} }
} else if let Some(entry) = entry_option { } else if let Some(entry) = entry_option {
// Used to be a file or a folder, now something else. // Used to be a file or a folder, now something else.
if matcher.matches_everything() || matcher.matches(&filename) { if matcher.matches_everything() || matcher.matches(&filename) {
return Ok(vec![( files_sender
Cow::Owned(filename.to_owned()), .send(Ok((filename.to_owned(), dispatch_missing(entry.state))))
.unwrap();
}
}
Ok(())
}
/// A directory was found in the filesystem and needs to be traversed
fn handle_traversed_dir<'a>(
scope: &rayon::Scope<'a>,
files_sender: &'a crossbeam::Sender<IoResult<(HgPathBuf, Dispatch)>>,
matcher: &'a (impl Matcher + Sync),
root_dir: impl AsRef<Path> + Sync + Send + Copy + 'a,
dmap: &'a DirstateMap,
old_results: &'a FastHashMap<Cow<HgPath>, Dispatch>,
ignore_fn: &'a (impl for<'r> Fn(&'r HgPath) -> bool + Sync),
dir_ignore_fn: &'a (impl for<'r> Fn(&'r HgPath) -> bool + Sync),
options: StatusOptions,
entry_option: Option<&'a DirstateEntry>,
directory: HgPathBuf,
) {
scope.spawn(move |_| {
// Nested `if` until `rust-lang/rust#53668` is stable
if let Some(entry) = entry_option {
// Used to be a file, is now a folder
if matcher.matches_everything() || matcher.matches(&directory) {
files_sender
.send(Ok((
directory.to_owned(),
dispatch_missing(entry.state), dispatch_missing(entry.state),
)]); )))
.unwrap();
} }
} }
return Ok(vec![]); // Do we need to traverse it?
if !ignore_fn(&directory) || options.list_ignored {
traverse_dir(
files_sender,
matcher,
root_dir,
dmap,
directory,
&old_results,
ignore_fn,
dir_ignore_fn,
options,
)
.unwrap_or_else(|e| files_sender.send(Err(e)).unwrap())
}
});
} }
/// Decides whether the directory needs to be listed, and if so dispatches its /// Decides whether the directory needs to be listed, and if so handles the
/// entries /// entries in a separate thread.
fn traverse_dir<'a>( fn traverse_dir<'a>(
matcher: &(impl Matcher + Sync), files_sender: &crossbeam::Sender<IoResult<(HgPathBuf, Dispatch)>>,
root_dir: impl AsRef<Path>, matcher: &'a (impl Matcher + Sync),
dmap: &DirstateMap, root_dir: impl AsRef<Path> + Sync + Send + Copy,
path: impl AsRef<HgPath>, dmap: &'a DirstateMap,
directory: impl AsRef<HgPath>,
old_results: &FastHashMap<Cow<'a, HgPath>, Dispatch>, old_results: &FastHashMap<Cow<'a, HgPath>, Dispatch>,
ignore_fn: &(impl for<'r> Fn(&'r HgPath) -> bool + Sync), ignore_fn: &(impl for<'r> Fn(&'r HgPath) -> bool + Sync),
dir_ignore_fn: &(impl for<'r> Fn(&'r HgPath) -> bool + Sync), dir_ignore_fn: &(impl for<'r> Fn(&'r HgPath) -> bool + Sync),
options: StatusOptions, options: StatusOptions,
) -> IoResult<Vec<(Cow<'a, HgPath>, Dispatch)>> { ) -> IoResult<()> {
let directory = path.as_ref(); let directory = directory.as_ref();
if directory.as_bytes() == b".hg" { if directory.as_bytes() == b".hg" {
return Ok(vec![]); return Ok(());
} }
let visit_entries = match matcher.visit_children_set(directory) { let visit_entries = match matcher.visit_children_set(directory) {
VisitChildrenSet::Empty => return Ok(vec![]), VisitChildrenSet::Empty => return Ok(()),
VisitChildrenSet::This | VisitChildrenSet::Recursive => None, VisitChildrenSet::This | VisitChildrenSet::Recursive => None,
VisitChildrenSet::Set(set) => Some(set), VisitChildrenSet::Set(set) => Some(set),
}; };
let buf = hg_path_to_path_buf(directory)?; let buf = hg_path_to_path_buf(directory)?;
let dir_path = root_dir.as_ref().join(buf); let dir_path = root_dir.as_ref().join(buf);
let skip_dot_hg = !directory.as_bytes().is_empty(); let skip_dot_hg = !directory.as_bytes().is_empty();
let entries = match list_directory(dir_path, skip_dot_hg) { let entries = match list_directory(dir_path, skip_dot_hg) {
Err(e) => match e.kind() { Err(e) => match e.kind() {
ErrorKind::NotFound | ErrorKind::PermissionDenied => { ErrorKind::NotFound | ErrorKind::PermissionDenied => {
return Ok(vec![( files_sender
Cow::Owned(directory.to_owned()), .send(Ok((
directory.to_owned(),
Dispatch::Bad(BadMatch::OsError( Dispatch::Bad(BadMatch::OsError(
// Unwrapping here is OK because the error always // Unwrapping here is OK because the error always
// is a real os error // is a real os error
e.raw_os_error().unwrap(), e.raw_os_error().unwrap(),
)), )),
)]); )))
.unwrap();
return Ok(());
} }
_ => return Err(e), _ => return Err(e),
}, },
Ok(entries) => entries, Ok(entries) => entries,
}; };
let mut new_results = vec![]; rayon::scope(|scope| -> IoResult<()> {
for (filename, dir_entry) in entries { for (filename, dir_entry) in entries {
if let Some(ref set) = visit_entries { if let Some(ref set) = visit_entries {
if !set.contains(filename.deref()) { if !set.contains(filename.deref()) {
continue; continue;
} }
} }
// TODO normalize // TODO normalize
let filename = if directory.is_empty() { let filename = if directory.is_empty() {
filename.to_owned() filename.to_owned()
} else { } else {
directory.join(&filename) directory.join(&filename)
}; };
if !old_results.contains_key(filename.deref()) { if !old_results.contains_key(filename.deref()) {
new_results.extend(handle_traversed_entry( handle_traversed_entry(
&dir_entry, scope,
files_sender,
matcher, matcher,
root_dir.as_ref(), root_dir,
&dmap, dmap,
&filename,
old_results, old_results,
ignore_fn, ignore_fn,
dir_ignore_fn, dir_ignore_fn,
options, options,
)?); filename,
dir_entry,
)?;
}
} }
Ok(())
})
} }
Ok(new_results)
/// Walk the working directory recursively to look for changes compared to the
/// current `DirstateMap`.
///
/// This takes a mutable reference to the results to account for the `extend`
/// in timings
fn traverse<'a>(
matcher: &'a (impl Matcher + Sync),
root_dir: impl AsRef<Path> + Sync + Send + Copy,
dmap: &'a DirstateMap,
path: impl AsRef<HgPath>,
old_results: &FastHashMap<Cow<'a, HgPath>, Dispatch>,
ignore_fn: &(impl for<'r> Fn(&'r HgPath) -> bool + Sync),
dir_ignore_fn: &(impl for<'r> Fn(&'r HgPath) -> bool + Sync),
options: StatusOptions,
results: &mut Vec<(Cow<'a, HgPath>, Dispatch)>,
) -> IoResult<()> {
let root_dir = root_dir.as_ref();
// The traversal is done in parallel, so use a channel to gather entries.
// `crossbeam::Sender` is `Send`, while `mpsc::Sender` is not.
let (files_transmitter, files_receiver) = crossbeam::channel::unbounded();
traverse_dir(
&files_transmitter,
matcher,
root_dir,
&dmap,
path,
&old_results,
&ignore_fn,
&dir_ignore_fn,
options,
)?;
// Disconnect the channel so the receiver stops waiting
drop(files_transmitter);
// TODO don't collect. Find a way of replicating the behavior of
// `itertools::process_results`, but for `rayon::ParallelIterator`
let new_results: IoResult<Vec<(Cow<'a, HgPath>, Dispatch)>> =
files_receiver
.into_iter()
.map(|item| {
let (f, d) = item?;
Ok((Cow::Owned(f), d))
})
.collect();
results.par_extend(new_results?);
Ok(())
} }
/// Stat all entries in the `DirstateMap` and mark them for dispatch. /// Stat all entries in the `DirstateMap` and mark them for dispatch.
fn stat_dmap_entries( fn stat_dmap_entries(
dmap: &DirstateMap, dmap: &DirstateMap,
root_dir: impl AsRef<Path> + Sync + Send, root_dir: impl AsRef<Path> + Sync + Send,
options: StatusOptions, options: StatusOptions,
) -> impl ParallelIterator<Item = IoResult<(&HgPath, Dispatch)>> { ) -> impl ParallelIterator<Item = IoResult<(&HgPath, Dispatch)>> {
match dispatch { match dispatch {
Dispatch::Directory { was_file } => { Dispatch::Directory { was_file } => {
if was_file { if was_file {
results.push((dir.to_owned(), Dispatch::Removed)); results.push((dir.to_owned(), Dispatch::Removed));
} }
if options.list_ignored if options.list_ignored
|| options.list_unknown && !dir_ignore_fn(&dir) || options.list_unknown && !dir_ignore_fn(&dir)
{ {
results.par_extend(traverse_dir( traverse(
matcher, matcher,
root_dir, root_dir,
&dmap, &dmap,
&dir, &dir,
&old_results, &old_results,
&ignore_fn, &ignore_fn,
&dir_ignore_fn, &dir_ignore_fn,
options, options,
)?); &mut results,
)?;
} }
} }
_ => unreachable!("There can only be directories in `work`"), _ => unreachable!("There can only be directories in `work`"),
} }
} }
} }
if !matcher.is_exact() { if !matcher.is_exact() {