This is an archive of the discontinued Mercurial Phabricator instance.

Differential D11731

rhg: lazily get filesystem metadata
Needs RevisionPublic

Authored by aalekseyev on Nov 2 2021, 8:06 AM.

Details

Reviewers
SimonSapin
baymax
Summary

rhg: use openat library for more efficient filesystem access

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

aalekseyev created this revision.Nov 2 2021, 8:06 AM
aalekseyev added a subscriber: SimonSapin.Nov 2 2021, 8:21 AM

@SimonSapin, this is a proof-of-concept patch I mentioned to you. It provides a large performance improvement in my benchmarks (in particular when running stat -mard), although I'm not entirely sure it's covered with tests.
It's not quite finished, in particular it doesn't handle the case when the filesystem doesn't return the file type. It also almost certainly doesn't work on Windows, but if we like the benchmarks then that can be worked out.

aalekseyev edited reviewers, added: SimonSapin; removed: hg-reviewers.Nov 2 2021, 8:22 AM
baymax requested changes to this revision.Feb 2 2022, 8:06 PM

There seems to have been no activities on this Diff for the past 3 Months.

By policy, we are automatically moving it out of the need-review state.

Please, move it back to need-review without hesitation if this diff should still be discussed.

:baymax:need-review-idle:

This revision now requires changes to proceed.Feb 2 2022, 8:06 PM

Revision Contents
Changeset List

Diff 31011

rust/Cargo.lock

# This file is automatically @generated by Cargo. # This file is automatically @generated by Cargo.
# It is not intended for manual editing. # It is not intended for manual editing.
version = 3
[[package]] [[package]]
name = "adler" name = "adler"
version = "0.2.3" version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
[[package]] [[package]]
name = "aho-corasick" name = "aho-corasick"
"home", "home",
"im-rc", "im-rc",
"itertools", "itertools",
"lazy_static", "lazy_static",
"libc", "libc",
"log", "log",
"memmap2", "memmap2",
"micro-timer", "micro-timer",
"openat",
"pretty_assertions", "pretty_assertions",
"rand", "rand",
"rand_distr", "rand_distr",
"rand_pcg", "rand_pcg",
"rayon", "rayon",
"regex", "regex",
"same-file", "same-file",
"sha-1", "sha-1",
[[package]] [[package]]
name = "opaque-debug" name = "opaque-debug"
version = "0.3.0" version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]] [[package]]
name = "openat"
version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95aa7c05907b3ebde2610d602f4ddd992145cc6a84493647c30396f30ba83abe"
dependencies = [
"libc",
]
[[package]]
name = "output_vt100" name = "output_vt100"
version = "0.1.2" version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9" checksum = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9"
dependencies = [ dependencies = [
"winapi", "winapi",
] ]

rust/hg-core/Cargo.toml

stable_deref_trait = "1.2.0" stable_deref_trait = "1.2.0"
tempfile = "3.1.0" tempfile = "3.1.0"
crossbeam-channel = "0.4" crossbeam-channel = "0.4"
micro-timer = "0.3.0" micro-timer = "0.3.0"
log = "0.4.8" log = "0.4.8"
memmap2 = {version = "0.4", features = ["stable_deref_trait"]} memmap2 = {version = "0.4", features = ["stable_deref_trait"]}
zstd = "0.5.3" zstd = "0.5.3"
format-bytes = "0.2.2" format-bytes = "0.2.2"
openat = "0.1.21"
# We don't use the `miniz-oxide` backend to not change rhg benchmarks and until # We don't use the `miniz-oxide` backend to not change rhg benchmarks and until
# we have a clearer view of which backend is the fastest. # we have a clearer view of which backend is the fastest.
[dependencies.flate2] [dependencies.flate2]
version = "1.0.16" version = "1.0.16"
features = ["zlib"] features = ["zlib"]
default-features = false default-features = false
[dev-dependencies] [dev-dependencies]
clap = "*" clap = "*"
pretty_assertions = "0.6.1" pretty_assertions = "0.6.1"

rust/hg-core/src/dirstate/entry.rs

use crate::dirstate_tree::on_disk::DirstateV2ParseError; use crate::dirstate_tree::on_disk::DirstateV2ParseError;
use crate::errors::HgError; use crate::errors::HgError;
use bitflags::bitflags; use bitflags::bitflags;
use std::convert::{TryFrom, TryInto}; use std::convert::{TryFrom, TryInto};
use std::fs;
use std::io; use std::io;
use std::time::{SystemTime, UNIX_EPOCH}; use std::time::{SystemTime, UNIX_EPOCH};
#[derive(Copy, Clone, Debug, Eq, PartialEq)] #[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub enum EntryState { pub enum EntryState {
Normal, Normal,
Added, Added,
Removed, Removed,
truncated_seconds, truncated_seconds,
nanoseconds, nanoseconds,
}) })
} else { } else {
Err(DirstateV2ParseError) Err(DirstateV2ParseError)
} }
} }
pub fn for_mtime_of(metadata: &fs::Metadata) -> io::Result<Self> { pub fn for_mtime_of(metadata: &openat::Metadata) -> io::Result<Self> {
#[cfg(unix)] #[cfg(unix)]
{ {
use std::os::unix::fs::MetadataExt; let stat = metadata.stat();
let seconds = metadata.mtime(); let seconds = stat.st_mtime as libc::time_t;
// i64!-> u32 with value always in the `0 .. NSEC_PER_SEC` range let nanoseconds = stat.st_mtime_nsec as libc::c_long;
let nanoseconds = metadata.mtime_nsec().try_into().unwrap(); let nanoseconds : u32 = nanoseconds.try_into().unwrap();
Ok(Self::new_truncate(seconds, nanoseconds)) Ok(Self::new_truncate(seconds, nanoseconds))
} }
#[cfg(not(unix))] #[cfg(not(unix))]
{ {
metadata.modified().map(Self::from) panic!("not supported")
} }
} }
/// The lower 31 bits of the number of seconds since the epoch. /// The lower 31 bits of the number of seconds since the epoch.
pub fn truncated_seconds(&self) -> u32 { pub fn truncated_seconds(&self) -> u32 {
self.truncated_seconds self.truncated_seconds
} }
self.truncated_seconds == other.truncated_seconds self.truncated_seconds == other.truncated_seconds
&& (self.nanoseconds == other.nanoseconds && (self.nanoseconds == other.nanoseconds
|| self.nanoseconds == 0 || self.nanoseconds == 0
|| other.nanoseconds == 0) || other.nanoseconds == 0)
} }
pub fn likely_equal_to_mtime_of( pub fn likely_equal_to_mtime_of(
self, self,
metadata: &fs::Metadata, metadata: &openat::Metadata,
) -> io::Result<bool> { ) -> io::Result<bool> {
Ok(self.likely_equal(Self::for_mtime_of(metadata)?)) Ok(self.likely_equal(Self::for_mtime_of(metadata)?))
} }
} }
impl From<SystemTime> for TruncatedTimestamp { impl From<SystemTime> for TruncatedTimestamp {
fn from(system_time: SystemTime) -> Self { fn from(system_time: SystemTime) -> Self {
// On Unix, `SystemTime` is a wrapper for the `timespec` C struct: // On Unix, `SystemTime` is a wrapper for the `timespec` C struct:
// https://www.gnu.org/software/libc/manual/html_node/Time-Types.html#index-struct-timespec // https://www.gnu.org/software/libc/manual/html_node/Time-Types.html#index-struct-timespec
// We want to effectively access its fields, but the Rust standard // We want to effectively access its fields, but the Rust standard
// library does not expose them. The best we can do is: // library does not expose them. The best we can do is:
self.state() == EntryState::Normal self.state() == EntryState::Normal
&& self.size() == SIZE_FROM_OTHER_PARENT && self.size() == SIZE_FROM_OTHER_PARENT
} }
// TODO: other platforms // TODO: other platforms
#[cfg(unix)] #[cfg(unix)]
pub fn mode_changed( pub fn mode_changed(
&self, &self,
filesystem_metadata: &std::fs::Metadata, filesystem_metadata: &openat::Metadata,
) -> bool { ) -> bool {
use std::os::unix::fs::MetadataExt;
const EXEC_BIT_MASK: u32 = 0o100; const EXEC_BIT_MASK: u32 = 0o100;
let dirstate_exec_bit = (self.mode() as u32) & EXEC_BIT_MASK; let dirstate_exec_bit = (self.mode() as u32) & EXEC_BIT_MASK;
let fs_exec_bit = filesystem_metadata.mode() & EXEC_BIT_MASK; let fs_exec_bit = filesystem_metadata.stat().st_mode & EXEC_BIT_MASK;
dirstate_exec_bit != fs_exec_bit dirstate_exec_bit != fs_exec_bit
} }
/// Returns a `(state, mode, size, mtime)` tuple as for /// Returns a `(state, mode, size, mtime)` tuple as for
/// `DirstateMapMethods::debug_iter`. /// `DirstateMapMethods::debug_iter`.
pub fn debug_tuple(&self) -> (u8, i32, i32, i32) { pub fn debug_tuple(&self) -> (u8, i32, i32, i32) {
(self.state().into(), self.mode(), self.size(), self.mtime()) (self.state().into(), self.mode(), self.size(), self.mtime())
} }

rust/hg-core/src/dirstate_tree/status.rs

use crate::dirstate::entry::TruncatedTimestamp; use crate::dirstate::entry::TruncatedTimestamp;
use crate::dirstate::status::IgnoreFnType; use crate::dirstate::status::IgnoreFnType;
use crate::dirstate_tree::dirstate_map::BorrowedPath; use crate::dirstate_tree::dirstate_map::BorrowedPath;
use crate::dirstate_tree::dirstate_map::ChildNodesRef; use crate::dirstate_tree::dirstate_map::ChildNodesRef;
use crate::dirstate_tree::dirstate_map::DirstateMap; use crate::dirstate_tree::dirstate_map::DirstateMap;
use crate::dirstate_tree::dirstate_map::NodeData; use crate::dirstate_tree::dirstate_map::NodeData;
use crate::dirstate_tree::dirstate_map::NodeRef; use crate::dirstate_tree::dirstate_map::NodeRef;
use crate::dirstate_tree::on_disk::DirstateV2ParseError; use crate::dirstate_tree::on_disk::DirstateV2ParseError;
use crate::matchers::get_ignore_function; use crate::matchers::get_ignore_function;
use crate::matchers::Matcher; use crate::matchers::Matcher;
use crate::utils::files::get_bytes_from_os_string;
use crate::utils::files::get_path_from_bytes;
use crate::utils::hg_path::HgPath; use crate::utils::hg_path::HgPath;
use crate::BadMatch; use crate::BadMatch;
use crate::DirstateStatus; use crate::DirstateStatus;
use crate::EntryState; use crate::EntryState;
use crate::HgPathBuf;
use crate::PatternFileWarning; use crate::PatternFileWarning;
use crate::StatusError; use crate::StatusError;
use crate::StatusOptions; use crate::StatusOptions;
use micro_timer::timed; use micro_timer::timed;
use rayon::prelude::*; use rayon::prelude::*;
use sha1::{Digest, Sha1}; use sha1::{Digest, Sha1};
use std::borrow::Cow; use std::borrow::Cow;
use std::io; use std::io;
use std::path::Path; use std::path::Path;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::Mutex; use std::sync::Mutex;
use std::time::SystemTime; use std::time::SystemTime;
use openat::{Dir};
use std::ffi::CStr;
use std::os::unix::ffi::OsStrExt;
use std::ffi::CString;
use std::convert::TryInto;
/// Returns the status of the working directory compared to its parent /// Returns the status of the working directory compared to its parent
/// changeset. /// changeset.
/// ///
/// This algorithm is based on traversing the filesystem tree (`fs` in function /// This algorithm is based on traversing the filesystem tree (`fs` in function
/// and variable names) and dirstate tree at the same time. The core of this /// and variable names) and dirstate tree at the same time. The core of this
/// traversal is the recursive `traverse_fs_directory_and_dirstate` function /// traversal is the recursive `traverse_fs_directory_and_dirstate` function
/// and its use of `itertools::merge_join_by`. When reaching a path that only /// and its use of `itertools::merge_join_by`. When reaching a path that only
let is_at_repo_root = true; let is_at_repo_root = true;
let hg_path = &BorrowedPath::OnDisk(HgPath::new("")); let hg_path = &BorrowedPath::OnDisk(HgPath::new(""));
let has_ignored_ancestor = false; let has_ignored_ancestor = false;
let root_cached_mtime = None; let root_cached_mtime = None;
let root_dir_metadata = None; let root_dir_metadata = None;
// If the path we have for the repository root is a symlink, do follow it. // If the path we have for the repository root is a symlink, do follow it.
// (As opposed to symlinks within the working directory which are not // (As opposed to symlinks within the working directory which are not
// followed, using `std::fs::symlink_metadata`.) // followed, using `std::fs::symlink_metadata`.)
let root_dir = openat::Dir::open(&root_dir)?;
common.traverse_fs_directory_and_dirstate( common.traverse_fs_directory_and_dirstate(
has_ignored_ancestor, has_ignored_ancestor,
dmap.root.as_ref(), dmap.root.as_ref(),
hg_path, hg_path,
&root_dir, &root_dir,
root_dir_metadata, root_dir_metadata,
root_cached_mtime, root_cached_mtime,
is_at_repo_root, is_at_repo_root,
/// and therefore isn’t reading `.hgignore`. /// and therefore isn’t reading `.hgignore`.
ignore_patterns_have_changed: Option<bool>, ignore_patterns_have_changed: Option<bool>,
/// The current time at the start of the `status()` algorithm, as measured /// The current time at the start of the `status()` algorithm, as measured
/// and possibly truncated by the filesystem. /// and possibly truncated by the filesystem.
filesystem_time_at_status_start: Option<SystemTime>, filesystem_time_at_status_start: Option<SystemTime>,
} }
enum MetadataOrType<'a> {
Metadata(&'a openat::Metadata),
FileType(openat::SimpleType)
}
struct FilesystemEntry<'a> {
dir : &'a Dir,
name : &'a CStr,
metadata_or_type : MetadataOrType<'a>
}
enum TORREF <'a, T> {
Owned(T),
Borrowed(&'a T)
}
impl <'a, T> TORREF <'a, T> {
fn borrow(&self) -> &T {
match self {
TORREF::Owned(t) => &t,
TORREF::Borrowed(r) => r
}
}
}
impl <'a> FilesystemEntry<'a> {
fn of_read_dir(dir : &'a Dir, entry : &'a openat::Entry) -> Self {
FilesystemEntry {
dir, name : <& openat::Entry as openat::AsPath>::to_path(entry).unwrap(), metadata_or_type : MetadataOrType::FileType(entry.simple_type().unwrap()) // TODO: not unwrap
}
}
fn of_metadata(dir : &'a Dir, name : &'a CStr, metadata : &'a openat::Metadata) -> Self {
FilesystemEntry {
dir, name, metadata_or_type : MetadataOrType::Metadata(metadata)
}
}
fn file_type(&self) -> openat::SimpleType {
match self.metadata_or_type {
MetadataOrType::Metadata(metadata) => metadata.simple_type (),
MetadataOrType::FileType(file_type) => file_type
}
}
// Returns None if the entry is not a directory
fn as_directory(&self) -> io::Result<Option<(Dir, openat::Metadata)>> {
match self.file_type() {
openat::SimpleType::Dir => {
let dir = self.dir.sub_dir(self.name)?;
let metadata = dir.self_metadata()?;
Ok(Some((dir, metadata)))
},
_ => Ok(None)
}
}
fn metadata(&self) -> io::Result<TORREF<'a, openat::Metadata>> {
match self.metadata_or_type {
MetadataOrType::Metadata(m) => Ok(TORREF::Borrowed(m)),
MetadataOrType::FileType(_) => {
let metadata = self.dir.metadata(self.name)?;
Ok(TORREF::Owned(metadata)) }
}
}
}
fn stat_mtime(metadata : &openat::Metadata) -> SystemTime {
let stat = metadata.stat();
let nsec : i64 = stat.st_mtime_nsec as libc::c_long;
let nsec : u32 = nsec.try_into().unwrap();
let sec : i64 = stat.st_mtime as libc::time_t;
let sec : u64 = sec.try_into().unwrap();
let dur = std::time::Duration::new(sec, nsec);
SystemTime::UNIX_EPOCH.checked_add(dur).unwrap()
}
impl<'a, 'tree, 'on_disk> StatusCommon<'a, 'tree, 'on_disk> { impl<'a, 'tree, 'on_disk> StatusCommon<'a, 'tree, 'on_disk> {
fn read_dir( fn read_dir(
&self, &self,
hg_path: &HgPath, hg_path: &HgPath,
fs_path: &Path, fs_dir: &Dir,
is_at_repo_root: bool, is_at_repo_root: bool,
) -> Result<Vec<DirEntry>, ()> { ) -> Result<Vec<openat::Entry>, ()> {
DirEntry::read_dir(fs_path, is_at_repo_root) read_dir(fs_dir, is_at_repo_root)
.map_err(|error| self.io_error(error, hg_path)) .map_err(|error| self.io_error(error, hg_path))
} }
fn io_error(&self, error: std::io::Error, hg_path: &HgPath) { fn io_error(&self, error: std::io::Error, hg_path: &HgPath) {
let errno = error.raw_os_error().expect("expected real OS error"); let errno = error.raw_os_error().expect("expected real OS error");
self.outcome self.outcome
.lock() .lock()
.unwrap() .unwrap()
Ok(()) Ok(())
} }
/// If this returns true, we can get accurate results by only using /// If this returns true, we can get accurate results by only using
/// `symlink_metadata` for child nodes that exist in the dirstate and don’t /// `symlink_metadata` for child nodes that exist in the dirstate and don’t
/// need to call `read_dir`. /// need to call `read_dir`.
fn can_skip_fs_readdir( fn can_skip_fs_readdir(
&self, &self,
directory_metadata: Option<&std::fs::Metadata>, directory_metadata: Option<&openat::Metadata>,
cached_directory_mtime: Option<TruncatedTimestamp>, cached_directory_mtime: Option<TruncatedTimestamp>,
) -> bool { ) -> bool {
if !self.options.list_unknown && !self.options.list_ignored { if !self.options.list_unknown && !self.options.list_ignored {
// All states that we care about listing have corresponding // All states that we care about listing have corresponding
// dirstate entries. // dirstate entries.
// This happens for example with `hg status -mard`. // This happens for example with `hg status -mard`.
return true; return true;
} }
/// Returns whether all child entries of the filesystem directory have a /// Returns whether all child entries of the filesystem directory have a
/// corresponding dirstate node or are ignored. /// corresponding dirstate node or are ignored.
fn traverse_fs_directory_and_dirstate( fn traverse_fs_directory_and_dirstate(
&self, &self,
has_ignored_ancestor: bool, has_ignored_ancestor: bool,
dirstate_nodes: ChildNodesRef<'tree, 'on_disk>, dirstate_nodes: ChildNodesRef<'tree, 'on_disk>,
directory_hg_path: &BorrowedPath<'tree, 'on_disk>, directory_hg_path: &BorrowedPath<'tree, 'on_disk>,
directory_fs_path: &Path, directory_fd: &Dir,
directory_metadata: Option<&std::fs::Metadata>, directory_metadata: Option<&openat::Metadata>,
cached_directory_mtime: Option<TruncatedTimestamp>, cached_directory_mtime: Option<TruncatedTimestamp>,
is_at_repo_root: bool, is_at_repo_root: bool,
) -> Result<bool, DirstateV2ParseError> { ) -> Result<bool, DirstateV2ParseError> {
if self.can_skip_fs_readdir(directory_metadata, cached_directory_mtime) if self.can_skip_fs_readdir(directory_metadata, cached_directory_mtime)
{ {
dirstate_nodes dirstate_nodes
.par_iter() .par_iter()
.map(|dirstate_node| { .map(|dirstate_node| {
let fs_path = directory_fs_path.join(get_path_from_bytes( let base_name = dirstate_node.base_name(self.dmap.on_disk)?;
dirstate_node.base_name(self.dmap.on_disk)?.as_bytes(), let base_name_cstring = CString::new(base_name.as_bytes()).expect("NULL BYTE");
)); match directory_fd.metadata(base_name_cstring.as_ref()) {
match std::fs::symlink_metadata(&fs_path) {
Ok(fs_metadata) => self.traverse_fs_and_dirstate( Ok(fs_metadata) => self.traverse_fs_and_dirstate(
&fs_path, &FilesystemEntry::of_metadata(directory_fd, base_name_cstring.as_ref(), &fs_metadata),
&fs_metadata,
dirstate_node, dirstate_node,
has_ignored_ancestor, has_ignored_ancestor,
), ),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => { Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
self.traverse_dirstate_only(dirstate_node) self.traverse_dirstate_only(dirstate_node)
} }
Err(error) => { Err(error) => {
let hg_path = let hg_path =
dirstate_node.full_path(self.dmap.on_disk)?; dirstate_node.full_path(self.dmap.on_disk)?;
Ok(self.io_error(error, hg_path)) Ok(self.io_error(error, hg_path))
} }
} }
}) })
.collect::<Result<_, _>>()?; .collect::<Result<_, _>>()?;
// We don’t know, so conservatively say this isn’t the case // We don’t know, so conservatively say this isn’t the case
let children_all_have_dirstate_node_or_are_ignored = false; let children_all_have_dirstate_node_or_are_ignored = false;
return Ok(children_all_have_dirstate_node_or_are_ignored); return Ok(children_all_have_dirstate_node_or_are_ignored);
} }
let mut fs_entries = if let Ok(entries) = self.read_dir( let mut fs_entries = if let Ok(entries) = self.read_dir(
directory_hg_path, directory_hg_path,
directory_fs_path, directory_fd,
is_at_repo_root, is_at_repo_root,
) { ) {
entries entries
} else { } else {
// Treat an unreadable directory (typically because of insufficient // Treat an unreadable directory (typically because of insufficient
// permissions) like an empty directory. `self.read_dir` has // permissions) like an empty directory. `self.read_dir` has
// already called `self.io_error` so a warning will be emitted. // already called `self.io_error` so a warning will be emitted.
Vec::new() Vec::new()
}; };
// `merge_join_by` requires both its input iterators to be sorted: // `merge_join_by` requires both its input iterators to be sorted:
let dirstate_nodes = dirstate_nodes.sorted(); let dirstate_nodes = dirstate_nodes.sorted();
// `sort_unstable_by_key` doesn’t allow keys borrowing from the value: // `sort_unstable_by_key` doesn’t allow keys borrowing from the value:
// https://github.com/rust-lang/rust/issues/34162 // https://github.com/rust-lang/rust/issues/34162
fs_entries.sort_unstable_by(|e1, e2| e1.base_name.cmp(&e2.base_name)); fs_entries.sort_unstable_by(|e1, e2| e1.file_name().as_bytes().cmp(&e2.file_name().as_bytes()));
// Propagate here any error that would happen inside the comparison // Propagate here any error that would happen inside the comparison
// callback below // callback below
for dirstate_node in &dirstate_nodes { for dirstate_node in &dirstate_nodes {
dirstate_node.base_name(self.dmap.on_disk)?; dirstate_node.base_name(self.dmap.on_disk)?;
} }
itertools::merge_join_by( itertools::merge_join_by(
dirstate_nodes, dirstate_nodes,
&fs_entries, &fs_entries,
|dirstate_node, fs_entry| { |dirstate_node, fs_entry| {
// This `unwrap` never panics because we already propagated // This `unwrap` never panics because we already propagated
// those errors above // those errors above
dirstate_node dirstate_node
.base_name(self.dmap.on_disk) .base_name(self.dmap.on_disk)
.unwrap() .unwrap()
.cmp(&fs_entry.base_name) .as_bytes()
.cmp(&fs_entry.file_name().as_bytes())
}, },
) )
.par_bridge() .par_bridge()
.map(|pair| { .map(|pair| {
use itertools::EitherOrBoth::*; use itertools::EitherOrBoth::*;
let has_dirstate_node_or_is_ignored; let has_dirstate_node_or_is_ignored;
match pair { match pair {
Both(dirstate_node, fs_entry) => { Both(dirstate_node, fs_entry) => {
self.traverse_fs_and_dirstate( self.traverse_fs_and_dirstate(
&fs_entry.full_path, &FilesystemEntry::of_read_dir(directory_fd, &fs_entry),
&fs_entry.metadata,
dirstate_node, dirstate_node,
has_ignored_ancestor, has_ignored_ancestor,
)?; )?;
has_dirstate_node_or_is_ignored = true has_dirstate_node_or_is_ignored = true
} }
Left(dirstate_node) => { Left(dirstate_node) => {
self.traverse_dirstate_only(dirstate_node)?; self.traverse_dirstate_only(dirstate_node)?;
has_dirstate_node_or_is_ignored = true; has_dirstate_node_or_is_ignored = true;
} }
Right(fs_entry) => { Right(fs_entry) => {
has_dirstate_node_or_is_ignored = self.traverse_fs_only( has_dirstate_node_or_is_ignored = self.traverse_fs_only(
has_ignored_ancestor, has_ignored_ancestor,
directory_hg_path, directory_hg_path,
directory_fd,
fs_entry, fs_entry,
) )
} }
} }
Ok(has_dirstate_node_or_is_ignored) Ok(has_dirstate_node_or_is_ignored)
}) })
.try_reduce(|| true, |a, b| Ok(a && b)) .try_reduce(|| true, |a, b| Ok(a && b))
} }
fn traverse_fs_and_dirstate( fn traverse_fs_and_dirstate(
&self, &self,
fs_path: &Path, fs_entry: &FilesystemEntry,
fs_metadata: &std::fs::Metadata,
dirstate_node: NodeRef<'tree, 'on_disk>, dirstate_node: NodeRef<'tree, 'on_disk>,
has_ignored_ancestor: bool, has_ignored_ancestor: bool,
) -> Result<(), DirstateV2ParseError> { ) -> Result<(), DirstateV2ParseError> {
self.check_for_outdated_directory_cache(&dirstate_node)?; self.check_for_outdated_directory_cache(&dirstate_node)?;
let hg_path = &dirstate_node.full_path_borrowed(self.dmap.on_disk)?; let hg_path = &dirstate_node.full_path_borrowed(self.dmap.on_disk)?;
let file_type = fs_metadata.file_type(); let file_type = fs_entry.file_type();
let file_or_symlink = file_type.is_file() || file_type.is_symlink(); let file_or_symlink =
match file_type {
openat::SimpleType::File | openat::SimpleType::Symlink => true,
openat::SimpleType::Dir | openat::SimpleType::Other => false,
};
if !file_or_symlink { if !file_or_symlink {
// If we previously had a file here, it was removed (with // If we previously had a file here, it was removed (with
// `hg rm` or similar) or deleted before it could be // `hg rm` or similar) or deleted before it could be
// replaced by a directory or something else. // replaced by a directory or something else.
self.mark_removed_or_deleted_if_file( self.mark_removed_or_deleted_if_file(
&hg_path, &hg_path,
dirstate_node.state()?, dirstate_node.state()?,
); );
} }
if file_type.is_dir() { match fs_entry.as_directory().expect("TODO: handle") {
Some ((dir, fs_metadata)) => {
if self.options.collect_traversed_dirs { if self.options.collect_traversed_dirs {
self.outcome self.outcome
.lock() .lock()
.unwrap() .unwrap()
.traversed .traversed
.push(hg_path.detach_from_tree()) .push(hg_path.detach_from_tree())
} }
let is_ignored = has_ignored_ancestor || (self.ignore_fn)(hg_path); let is_ignored = has_ignored_ancestor || (self.ignore_fn)(hg_path);
let is_at_repo_root = false; let is_at_repo_root = false;
let children_all_have_dirstate_node_or_are_ignored = self let children_all_have_dirstate_node_or_are_ignored = self
.traverse_fs_directory_and_dirstate( .traverse_fs_directory_and_dirstate(
is_ignored, is_ignored,
dirstate_node.children(self.dmap.on_disk)?, dirstate_node.children(self.dmap.on_disk)?,
hg_path, hg_path,
fs_path, &dir,
Some(fs_metadata), Some(&fs_metadata),
dirstate_node.cached_directory_mtime()?, dirstate_node.cached_directory_mtime()?,
is_at_repo_root, is_at_repo_root,
)?; )?;
self.maybe_save_directory_mtime( self.maybe_save_directory_mtime(
children_all_have_dirstate_node_or_are_ignored, children_all_have_dirstate_node_or_are_ignored,
fs_metadata, &fs_metadata,
dirstate_node, dirstate_node,
)? )?
} else { },
None => {
if file_or_symlink && self.matcher.matches(hg_path) { if file_or_symlink && self.matcher.matches(hg_path) {
if let Some(state) = dirstate_node.state()? { if let Some(state) = dirstate_node.state()? {
match state { match state {
EntryState::Added => self EntryState::Added => self
.outcome .outcome
.lock() .lock()
.unwrap() .unwrap()
.added .added
.push(hg_path.detach_from_tree()), .push(hg_path.detach_from_tree()),
EntryState::Removed => self EntryState::Removed => self
.outcome .outcome
.lock() .lock()
.unwrap() .unwrap()
.removed .removed
.push(hg_path.detach_from_tree()), .push(hg_path.detach_from_tree()),
EntryState::Merged => self EntryState::Merged => self
.outcome .outcome
.lock() .lock()
.unwrap() .unwrap()
.modified .modified
.push(hg_path.detach_from_tree()), .push(hg_path.detach_from_tree()),
EntryState::Normal => self EntryState::Normal => {
.handle_normal_file(&dirstate_node, fs_metadata)?, let fs_metadata = fs_entry.metadata().expect("TODO");
self.handle_normal_file(
&dirstate_node,
fs_metadata.borrow(),
)?
}
} }
} else { } else {
// `node.entry.is_none()` indicates a "directory" // `node.entry.is_none()` indicates a "directory"
// node, but the filesystem has a file // node, but the filesystem has a file
self.mark_unknown_or_ignored( self.mark_unknown_or_ignored(
has_ignored_ancestor, has_ignored_ancestor,
hg_path, hg_path,
); );
} }
} }
for child_node in dirstate_node.children(self.dmap.on_disk)?.iter() for child_node in dirstate_node.children(self.dmap.on_disk)?.iter()
{ {
self.traverse_dirstate_only(child_node)? self.traverse_dirstate_only(child_node)?
} }
} },
};
Ok(()) Ok(())
} }
fn maybe_save_directory_mtime( fn maybe_save_directory_mtime(
&self, &self,
children_all_have_dirstate_node_or_are_ignored: bool, children_all_have_dirstate_node_or_are_ignored: bool,
directory_metadata: &std::fs::Metadata, directory_metadata: &openat::Metadata,
dirstate_node: NodeRef<'tree, 'on_disk>, dirstate_node: NodeRef<'tree, 'on_disk>,
) -> Result<(), DirstateV2ParseError> { ) -> Result<(), DirstateV2ParseError> {
if children_all_have_dirstate_node_or_are_ignored { if children_all_have_dirstate_node_or_are_ignored {
// All filesystem directory entries from `read_dir` have a // All filesystem directory entries from `read_dir` have a
// corresponding node in the dirstate, so we can reconstitute the // corresponding node in the dirstate, so we can reconstitute the
// names of those entries without calling `read_dir` again. // names of those entries without calling `read_dir` again.
if let (Some(status_start), Ok(directory_mtime)) = ( if let (Some(status_start), directory_mtime) = (
&self.filesystem_time_at_status_start, &self.filesystem_time_at_status_start,
directory_metadata.modified(), stat_mtime(directory_metadata),
) { ) {
// Although the Rust standard library’s `SystemTime` type // Although the Rust standard library’s `SystemTime` type
// has nanosecond precision, the times reported for a // has nanosecond precision, the times reported for a
// directory’s (or file’s) modified time may have lower // directory’s (or file’s) modified time may have lower
// resolution based on the filesystem (for example ext3 // resolution based on the filesystem (for example ext3
// only stores integer seconds), kernel (see // only stores integer seconds), kernel (see
// https://stackoverflow.com/a/14393315/1162888), etc. // https://stackoverflow.com/a/14393315/1162888), etc.
if &directory_mtime >= status_start { if &directory_mtime >= status_start {
Ok(()) Ok(())
} }
/// A file with `EntryState::Normal` in the dirstate was found in the /// A file with `EntryState::Normal` in the dirstate was found in the
/// filesystem /// filesystem
fn handle_normal_file( fn handle_normal_file(
&self, &self,
dirstate_node: &NodeRef<'tree, 'on_disk>, dirstate_node: &NodeRef<'tree, 'on_disk>,
fs_metadata: &std::fs::Metadata, fs_metadata: &openat::Metadata,
) -> Result<(), DirstateV2ParseError> { ) -> Result<(), DirstateV2ParseError> {
// Keep the low 31 bits // Keep the low 31 bits
fn truncate_u64(value: u64) -> i32 { fn truncate_u64(value: u64) -> i32 {
(value & 0x7FFF_FFFF) as i32 (value & 0x7FFF_FFFF) as i32
} }
let entry = dirstate_node let entry = dirstate_node
.entry()? .entry()?
.expect("handle_normal_file called with entry-less node"); .expect("handle_normal_file called with entry-less node");
let hg_path = &dirstate_node.full_path_borrowed(self.dmap.on_disk)?; let hg_path = &dirstate_node.full_path_borrowed(self.dmap.on_disk)?;
let mode_changed = let mode_changed =
|| self.options.check_exec && entry.mode_changed(fs_metadata); || self.options.check_exec && entry.mode_changed(fs_metadata);
let size = entry.size(); let size = entry.size();
let size_changed = size != truncate_u64(fs_metadata.len()); let size_changed = size != truncate_u64(fs_metadata.len());
if size >= 0 && size_changed && fs_metadata.file_type().is_symlink() { if size >= 0 && size_changed && fs_metadata.simple_type() == openat::SimpleType::Symlink {
// issue6456: Size returned may be longer due to encryption // issue6456: Size returned may be longer due to encryption
// on EXT-4 fscrypt. TODO maybe only do it on EXT4? // on EXT-4 fscrypt. TODO maybe only do it on EXT4?
self.outcome self.outcome
.lock() .lock()
.unwrap() .unwrap()
.unsure .unsure
.push(hg_path.detach_from_tree()) .push(hg_path.detach_from_tree())
} else if dirstate_node.has_copy_source() } else if dirstate_node.has_copy_source()
/// Something in the filesystem has no corresponding dirstate node /// Something in the filesystem has no corresponding dirstate node
/// ///
/// Returns whether that path is ignored /// Returns whether that path is ignored
fn traverse_fs_only( fn traverse_fs_only(
&self, &self,
has_ignored_ancestor: bool, has_ignored_ancestor: bool,
directory_hg_path: &HgPath, directory_hg_path: &HgPath,
fs_entry: &DirEntry, parent_dir : &Dir,
fs_entry: &openat::Entry,
) -> bool { ) -> bool {
let hg_path = directory_hg_path.join(&fs_entry.base_name); let hg_path = directory_hg_path.join(HgPath::new(&fs_entry.file_name ().as_bytes()));
let file_type = fs_entry.metadata.file_type(); let file_type = fs_entry.simple_type().unwrap(); // TODO: fix unwrap
let file_or_symlink = file_type.is_file() || file_type.is_symlink(); let file_or_symlink = match file_type {
if file_type.is_dir() { openat::SimpleType::File | openat::SimpleType::Symlink => true,
_ => false };
if openat::SimpleType::Dir == file_type {
let is_ignored = let is_ignored =
has_ignored_ancestor || (self.ignore_fn)(&hg_path); has_ignored_ancestor || (self.ignore_fn)(&hg_path);
let traverse_children = if is_ignored { let traverse_children = if is_ignored {
// Descendants of an ignored directory are all ignored // Descendants of an ignored directory are all ignored
self.options.list_ignored self.options.list_ignored
} else { } else {
// Descendants of an unknown directory may be either unknown or // Descendants of an unknown directory may be either unknown or
// ignored // ignored
self.options.list_unknown || self.options.list_ignored self.options.list_unknown || self.options.list_ignored
}; };
if traverse_children { if traverse_children {
let is_at_repo_root = false; let is_at_repo_root = false;
let dir = parent_dir.sub_dir(fs_entry.file_name()).expect("TODO");
if let Ok(children_fs_entries) = self.read_dir( if let Ok(children_fs_entries) = self.read_dir(
&hg_path, &hg_path,
&fs_entry.full_path, &dir,
is_at_repo_root, is_at_repo_root,
) { ) {
children_fs_entries.par_iter().for_each(|child_fs_entry| { children_fs_entries.par_iter().for_each(|child_fs_entry| {
self.traverse_fs_only( self.traverse_fs_only(
is_ignored, is_ignored,
&hg_path, &hg_path,
&dir,
child_fs_entry, child_fs_entry,
); );
}) })
} }
} }
if self.options.collect_traversed_dirs { if self.options.collect_traversed_dirs {
self.outcome.lock().unwrap().traversed.push(hg_path.into()) self.outcome.lock().unwrap().traversed.push(hg_path.into())
} }
.unknown .unknown
.push(hg_path.detach_from_tree()) .push(hg_path.detach_from_tree())
} }
} }
is_ignored is_ignored
} }
} }
struct DirEntry {
base_name: HgPathBuf,
full_path: PathBuf,
metadata: std::fs::Metadata,
}
impl DirEntry {
/// Returns **unsorted** entries in the given directory, with name and /// Returns **unsorted** entries in the given directory, with name and
/// metadata. /// metadata.
/// ///
/// If a `.hg` sub-directory is encountered: /// If a `.hg` sub-directory is encountered:
/// ///
/// * At the repository root, ignore that sub-directory /// * At the repository root, ignore that sub-directory
/// * Elsewhere, we’re listing the content of a sub-repo. Return an empty /// * Elsewhere, we’re listing the content of a sub-repo. Return an empty
/// list instead. /// list instead.
fn read_dir(path: &Path, is_at_repo_root: bool) -> io::Result<Vec<Self>> { fn read_dir(dir: &openat::Dir, is_at_repo_root: bool) -> io::Result<Vec<openat::Entry>> {
let mut results = Vec::new(); let mut results = Vec::new();
for entry in path.read_dir()? { for entry in dir.list_dir(".")? {
let entry = entry?; let entry = entry?;
let metadata = entry.metadata()?; let name = entry.file_name();
let name = get_bytes_from_os_string(entry.file_name());
// FIXME don't do this when cached // FIXME don't do this when cached
if name == b".hg" { if name.as_bytes() == b".hg" {
if is_at_repo_root { if is_at_repo_root {
// Skip the repo’s own .hg (might be a symlink) // Skip the repo’s own .hg (might be a symlink)
continue; continue;
} else if metadata.is_dir() { // TODO: handle simple_type returning None
} else if Some(openat::SimpleType::Dir) == entry.simple_type() {
// A .hg sub-directory at another location means a subrepo, // A .hg sub-directory at another location means a subrepo,
// skip it entirely. // skip it entirely.
return Ok(Vec::new()); return Ok(Vec::new());
} }
} }
results.push(DirEntry { results.push(entry)
base_name: name.into(),
full_path: entry.path(),
metadata,
})
} }
Ok(results) Ok(results)
} }
}
/// Return the `mtime` of a temporary file newly-created in the `.hg` directory /// Return the `mtime` of a temporary file newly-created in the `.hg` directory
/// of the give repository. /// of the give repository.
/// ///
/// This is similar to `SystemTime::now()`, with the result truncated to the /// This is similar to `SystemTime::now()`, with the result truncated to the
/// same time resolution as other files’ modification times. Using `.hg` /// same time resolution as other files’ modification times. Using `.hg`
/// instead of the system’s default temporary directory (such as `/tmp`) makes /// instead of the system’s default temporary directory (such as `/tmp`) makes
/// it more likely the temporary file is in the same disk partition as contents /// it more likely the temporary file is in the same disk partition as contents