diff --git a/rust/Cargo.lock b/rust/Cargo.lock --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -9,6 +9,14 @@ ] [[package]] +name = "arrayvec" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] name = "autocfg" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -24,6 +32,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] +name = "cfg-if" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] name = "cloudabi" version = "0.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -43,6 +56,50 @@ ] [[package]] +name = "crossbeam-deque" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayvec 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memoffset 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-queue" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-utils" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "either" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] name = "fuchsia-cprng" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -56,6 +113,7 @@ "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand_pcg 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -94,11 +152,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] +name = "memoffset" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "nodrop" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] name = "num-traits" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] +name = "num_cpus" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] name = "python27-sys" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -213,6 +292,28 @@ ] [[package]] +name = "rayon" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-deque 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon-core 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rayon-core" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-deque 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] name = "rdrand" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -249,6 +350,11 @@ ] [[package]] +name = "scopeguard" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] name = "semver" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -300,16 +406,26 @@ [metadata] "checksum aho-corasick 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)" = "1e9a933f4e58658d7b12defcf96dc5c720f20832deebe3e0a19efd3b6aaeeb9e" +"checksum arrayvec 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)" = "b8d73f9beda665eaa98ab9e4f7442bd4e7de6652587de55b2525e52e29c1b0ba" "checksum autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799" "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" "checksum byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a019b10a2a7cdeb292db131fc8113e57ea2a908f6e7894b0c3c671893b65dbeb" +"checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum cpython 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b489034e723e7f5109fecd19b719e664f89ef925be785885252469e9822fa940" +"checksum crossbeam-deque 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b18cd2e169ad86297e6bc0ad9aa679aee9daa4f19e8163860faf7c164e4f5a71" +"checksum crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fedcd6772e37f3da2a9af9bf12ebe046c0dfe657992377b4df982a2b54cd37a9" +"checksum crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b" +"checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" +"checksum either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5527cfe0d098f36e3f8839852688e63c8fff1c90b2b405aef730615f9a7bcf7b" "checksum fuchsia-cprng 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "81f7f8eb465745ea9b02e2704612a9946a59fa40572086c6fd49d6ddcf30bf31" "checksum lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bc5729f27f159ddd61f4df6228e827e86643d4d3e7c32183cb30a1c08f604a14" "checksum libc 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)" = "2d2857ec59fadc0773853c664d2d18e7198e83883e7060b63c924cb077bd5c74" "checksum memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2efc7bc57c883d4a4d6e3246905283d8dae951bb3bd32f49d6ef297f546e1c39" +"checksum memoffset 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ce6075db033bbbb7ee5a0bbd3a3186bbae616f57fb001c485c7ff77955f8177f" +"checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" "checksum num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1" +"checksum num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bcef43580c035376c0705c42792c294b66974abbfd2789b511784023f71f3273" "checksum python27-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "56114c37d4dca82526d74009df7782a28c871ac9d36b19d4cb9e67672258527e" "checksum python3-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "61e4aac43f833fd637e429506cb2ac9d7df672c4b68f2eaaa163649b7fdc0444" "checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" @@ -322,10 +438,13 @@ "checksum rand_os 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b7c690732391ae0abafced5015ffb53656abfaec61b342290e5eb56b286a679d" "checksum rand_pcg 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "086bd09a33c7044e56bb44d5bdde5a60e7f119a9e95b0775f545de759a32fe05" "checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +"checksum rayon 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "83a27732a533a1be0a0035a111fe76db89ad312f6f0347004c220c57f209a123" +"checksum rayon-core 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "98dcf634205083b17d0861252431eb2acbfb698ab7478a2d20de07954f47ec7b" "checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" "checksum regex 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "37e7cbbd370869ce2e8dff25c7018702d10b21a20ef7135316f8daecd6c25b7f" "checksum regex-syntax 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4e47a2ed29da7a9e1960e1639e7a982e6edc6d49be308a3b02daf511504a16d1" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +"checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d" "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" diff --git a/rust/hg-core/Cargo.toml b/rust/hg-core/Cargo.toml --- a/rust/hg-core/Cargo.toml +++ b/rust/hg-core/Cargo.toml @@ -15,3 +15,4 @@ rand = "> 0.6.4" rand_pcg = "> 0.1.0" regex = "^1.1" +rayon = "1.2.0" diff --git a/rust/hg-core/src/dirstate.rs b/rust/hg-core/src/dirstate.rs --- a/rust/hg-core/src/dirstate.rs +++ b/rust/hg-core/src/dirstate.rs @@ -13,6 +13,7 @@ pub mod dirs_multiset; pub mod dirstate_map; pub mod parsers; +pub mod status; #[derive(Debug, PartialEq, Clone)] pub struct DirstateParents { diff --git a/rust/hg-core/src/dirstate/status.rs b/rust/hg-core/src/dirstate/status.rs new file mode 100644 --- /dev/null +++ b/rust/hg-core/src/dirstate/status.rs @@ -0,0 +1,254 @@ +// status.rs +// +// Copyright 2019 Raphaël Gomès +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2 or any later version. + +//! Rust implementation of dirstate.status (dirstate.py). +//! It is currently missing a lot of functionality compared to the Python one +//! and will only be triggered in narrow cases. + +use crate::utils::files::HgMetadata; +use crate::utils::hg_path::{hg_path_to_path_buf, HgPathBuf}; +use crate::{DirstateEntry, DirstateMap, EntryState}; +use rayon::prelude::*; +use std::collections::HashMap; +use std::fs::Metadata; +use std::path::Path; + +/// Get stat data about the files explicitly specified by match. +/// TODO subrepos +fn walk_explicit + Sync>( + files: Vec, + dmap: &mut DirstateMap, + root_dir: P, +) -> std::io::Result>> { + let mut results = HashMap::new(); + + // A tuple of the normalized filename and the `Result` of the call to + // `symlink_metadata` for separate handling. + type WalkTuple<'a> = (&'a HgPathBuf, std::io::Result); + + let stats_res: std::io::Result> = files + .par_iter() + .map(|filename| { + // TODO normalization + let normalized = filename; + + let target_filename = + root_dir.as_ref().join(hg_path_to_path_buf(normalized)?); + + Ok((normalized, target_filename.symlink_metadata())) + }) + .collect(); + + for res in stats_res? { + match res { + (normalized, Ok(stat)) => { + if stat.is_file() { + results.insert( + normalized.to_owned(), + Some(HgMetadata::from_metadata(stat)), + ); + } else { + if dmap.contains_key(normalized) { + results.insert(normalized.to_owned(), None); + } + } + } + (normalized, Err(_)) => { + if dmap.contains_key(normalized) { + results.insert(normalized.to_owned(), None); + } + } + }; + } + + Ok(results) +} + +// Stat all entries in the `DirstateMap` and return their new metadata. +pub fn stat_dmap_entries<'a, P: AsRef + Sync>( + dmap: &'a DirstateMap, + results: &HashMap>, + root_dir: P, +) -> std::io::Result)>> { + dmap.par_iter() + .filter_map( + // Getting file metadata is costly, so we don't do it if the + // file is already present in the results, hence `filter_map` + |(filename, _)| -> Option< + std::io::Result<(HgPathBuf, Option)> + > { + if results.contains_key(filename) { + return None; + } + let meta = match hg_path_to_path_buf(filename) { + Ok(p) => root_dir.as_ref().join(p).symlink_metadata(), + Err(e) => return Some(Err(e.into())), + }; + + Some(match meta { + Ok(ref m) + if !(m.file_type().is_file() + || m.file_type().is_symlink()) => + { + Ok((filename.to_owned(), None)) + } + Ok(m) => Ok(( + filename.to_owned(), + Some(HgMetadata::from_metadata(m)), + )), + Err(ref e) + if e.kind() == std::io::ErrorKind::NotFound + || e.raw_os_error() == Some(20) => + { + // Rust does not yet have an `ErrorKind` for + // `NotADirectory` (errno 20) + Ok((filename.to_owned(), None)) + } + Err(e) => Err(e), + }) + }, + ) + .collect() +} + +pub struct StatusResult { + pub modified: Vec, + pub added: Vec, + pub removed: Vec, + pub deleted: Vec, + pub clean: Vec, + // TODO ignored + // TODO unknown +} + +fn build_response( + dmap: &DirstateMap, + list_clean: bool, + last_normal_time: i64, + check_exec: bool, + results: HashMap>, +) -> ((Vec, StatusResult)) { + let mut lookup = Vec::new(); + let mut modified = Vec::new(); + let mut added = Vec::new(); + let mut removed = Vec::new(); + let mut deleted = Vec::new(); + let mut clean = Vec::new(); + + for (filename, metadata_option) in results.into_iter() { + let DirstateEntry { + state, + mode, + mtime, + size, + } = match dmap.get(&filename) { + None => { + continue; + } + Some(e) => *e, + }; + + match metadata_option { + None if match state { + EntryState::Normal + | EntryState::Merged + | EntryState::Added => true, + _ => false, + } => + { + deleted.push(filename); + } + None => match state { + EntryState::Removed => removed.push(filename), + _ => {} + }, + Some(HgMetadata { + st_mode, + st_size, + st_mtime, + .. + }) => { + match state { + EntryState::Normal => { + // Dates and times that are outside the 31-bit signed + // range are compared modulo 2^31. This should prevent + // it from behaving badly with very large files or + // corrupt dates while still having a high probability + // of detecting changes. (issue2608) + let range_mask = 0x7fffffff; + + let size_changed = (size != st_size as i32) + && size != (st_size as i32 & range_mask); + let mode_changed = (mode ^ st_mode as i32) & 0o100 + != 0o000 + && check_exec; + if size >= 0 + && (size_changed || mode_changed) + || size == -2 // other parent + || dmap.copy_map.contains_key(&filename) + { + modified.push(filename); + } else if mtime != st_mtime as i32 + && mtime != (st_mtime as i32 & range_mask) + { + lookup.push(filename); + } else if st_mtime == last_normal_time { + // the file may have just been marked as normal and + // it may have changed in the same second without + // changing its size. This can happen if we quickly + // do multiple commits. Force lookup, so we don't + // miss such a racy file change. + lookup.push(filename); + } else if list_clean { + clean.push(filename); + } + } + EntryState::Merged => modified.push(filename), + EntryState::Added => added.push(filename), + EntryState::Removed => removed.push(filename), + EntryState::Unknown => {} + } + } + } + } + + ( + lookup, + StatusResult { + modified, + added, + removed, + deleted, + clean, + }, + ) +} + +pub fn status>( + mut dmap: &mut DirstateMap, + root_dir: P, + files: Vec, + list_clean: bool, + last_normal_time: i64, + check_exec: bool, +) -> std::io::Result<(Vec, StatusResult)> +where + P: Sync, +{ + let mut results = + walk_explicit(files, &mut dmap, root_dir.as_ref().to_owned())?; + + results.extend(stat_dmap_entries(&dmap, &results, root_dir)?); + + Ok(build_response( + &dmap, + list_clean, + last_normal_time, + check_exec, + results, + )) +} diff --git a/rust/hg-core/src/lib.rs b/rust/hg-core/src/lib.rs --- a/rust/hg-core/src/lib.rs +++ b/rust/hg-core/src/lib.rs @@ -12,6 +12,7 @@ dirs_multiset::{DirsMultiset, DirsMultisetIter}, dirstate_map::DirstateMap, parsers::{pack_dirstate, parse_dirstate, PARENT_SIZE}, + status::status, CopyMap, CopyMapIter, DirstateEntry, DirstateParents, EntryState, StateMap, StateMapIter, }; diff --git a/rust/hg-core/src/utils/files.rs b/rust/hg-core/src/utils/files.rs --- a/rust/hg-core/src/utils/files.rs +++ b/rust/hg-core/src/utils/files.rs @@ -12,6 +12,7 @@ use crate::utils::hg_path::{HgPath, HgPathBuf}; use std::iter::FusedIterator; +use std::fs::{read_link, Metadata}; use std::path::Path; pub fn get_path_from_bytes(bytes: &[u8]) -> &Path { @@ -79,6 +80,57 @@ path.to_ascii_lowercase() } +/// Get name in the case stored in the filesystem +/// The name should be relative to root, and be normcase-ed for efficiency. +/// +/// Note that this function is unnecessary, and should not be +// called, for case-sensitive filesystems (simply because it's expensive). +/// The root should be normcase-ed, too. +pub fn filesystem_path>(root: &HgPath, path: P) -> HgPathBuf { + // TODO path for case-insensitive filesystems, for now this is transparent + root.to_owned().join(path.as_ref()) +} + +/// Returns `true` if path refers to an existing path. +/// Returns `true` for broken symbolic links. +/// Equivalent to `exists()` on platforms lacking `lstat`. +pub fn lexists>(path: P) -> bool { + if !path.as_ref().exists() { + return read_link(path).is_ok(); + } + true +} + +#[derive(Eq, PartialEq, Ord, PartialOrd, Copy, Clone)] +pub struct HgMetadata { + pub st_dev: u64, + pub st_mode: u32, + pub st_nlink: u64, + pub st_size: u64, + pub st_mtime: i64, + pub st_ctime: i64, +} + +impl HgMetadata { + #[cfg(unix)] + pub fn from_metadata(metadata: Metadata) -> Self { + use std::os::linux::fs::MetadataExt; + Self { + st_dev: metadata.st_dev(), + st_mode: metadata.st_mode(), + st_nlink: metadata.st_nlink(), + st_size: metadata.st_size(), + st_mtime: metadata.st_mtime(), + st_ctime: metadata.st_ctime(), + } + } + #[cfg(not(unix))] + pub fn from_metdata(metadata: Metadata) -> Self { + // TODO support other platforms + unimplemented!() + } +} + #[cfg(test)] mod tests { use super::*;