This is an archive of the discontinued Mercurial Phabricator instance.

rhg: lazily get filesystem metadata
Needs RevisionPublic

Authored by aalekseyev on Nov 2 2021, 8:06 AM.

Download Raw Diff

Details

Reviewers

SimonSapin
baymax

Summary

rhg: use openat library for more efficient filesystem access

Diff Detail

Repository

rHG Mercurial

Branch

default

Lint

No Linters Available

Unit

No Unit Test Coverage

Event Timeline

aalekseyev created this revision.Nov 2 2021, 8:06 AM

Herald added a reviewer: hg-reviewers. · View Herald TranscriptNov 2 2021, 8:06 AM

Herald added a subscriber: mercurial-patches. · View Herald Transcript

@SimonSapin, this is a proof-of-concept patch I mentioned to you. It provides a large performance improvement in my benchmarks (in particular when running stat -mard), although I'm not entirely sure it's covered with tests.
It's not quite finished, in particular it doesn't handle the case when the filesystem doesn't return the file type. It also almost certainly doesn't work on Windows, but if we like the benchmarks then that can be worked out.

aalekseyev edited reviewers, added: SimonSapin; removed: hg-reviewers.Nov 2 2021, 8:22 AM

There seems to have been no activities on this Diff for the past 3 Months.

By policy, we are automatically moving it out of the need-review state.

Please, move it back to need-review without hesitation if this diff should still be discussed.

:baymax:need-review-idle:

This revision now requires changes to proceed.Feb 2 2022, 8:06 PM

Revision Contents
Changeset List

			Path	Packages
M			rust/Cargo.lock (12 lines)
M			rust/hg-core/Cargo.toml (1 line)
M			rust/hg-core/src/dirstate/entry.rs (21 lines)
M			rust/hg-core/src/dirstate_tree/status.rs (247 lines)

Commit	Parents	Author	Summary	Date
923fa73e5c85	9085a3944077	Arseniy Alekseyev		Oct 29 2021, 7:37 AM
e75f220f374e	923fa73e5c85	Arseniy Alekseyev		Nov 1 2021, 4:01 PM

Diff 31011

rust/Cargo.lock

	# This file is automatically @generated by Cargo.			# This file is automatically @generated by Cargo.
	# It is not intended for manual editing.			# It is not intended for manual editing.
				version = 3

	[[package]]			[[package]]
	name = "adler"			name = "adler"
	version = "0.2.3"			version = "0.2.3"
	source = "registry+https://github.com/rust-lang/crates.io-index"			source = "registry+https://github.com/rust-lang/crates.io-index"
	checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"			checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"

	[[package]]			[[package]]
	name = "aho-corasick"			name = "aho-corasick"
	"home",			"home",
	"im-rc",			"im-rc",
	"itertools",			"itertools",
	"lazy_static",			"lazy_static",
	"libc",			"libc",
	"log",			"log",
	"memmap2",			"memmap2",
	"micro-timer",			"micro-timer",
				"openat",
	"pretty_assertions",			"pretty_assertions",
	"rand",			"rand",
	"rand_distr",			"rand_distr",
	"rand_pcg",			"rand_pcg",
	"rayon",			"rayon",
	"regex",			"regex",
	"same-file",			"same-file",
	"sha-1",			"sha-1",

	[[package]]			[[package]]
	name = "opaque-debug"			name = "opaque-debug"
	version = "0.3.0"			version = "0.3.0"
	source = "registry+https://github.com/rust-lang/crates.io-index"			source = "registry+https://github.com/rust-lang/crates.io-index"
	checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"			checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"

	[[package]]			[[package]]
				name = "openat"
				version = "0.1.21"
				source = "registry+https://github.com/rust-lang/crates.io-index"
				checksum = "95aa7c05907b3ebde2610d602f4ddd992145cc6a84493647c30396f30ba83abe"
				dependencies = [
				"libc",
				]

				[[package]]
	name = "output_vt100"			name = "output_vt100"
	version = "0.1.2"			version = "0.1.2"
	source = "registry+https://github.com/rust-lang/crates.io-index"			source = "registry+https://github.com/rust-lang/crates.io-index"
	checksum = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9"			checksum = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9"
	dependencies = [			dependencies = [
	"winapi",			"winapi",
	]			]

rust/hg-core/Cargo.toml

	stable_deref_trait = "1.2.0"			stable_deref_trait = "1.2.0"
	tempfile = "3.1.0"			tempfile = "3.1.0"
	crossbeam-channel = "0.4"			crossbeam-channel = "0.4"
	micro-timer = "0.3.0"			micro-timer = "0.3.0"
	log = "0.4.8"			log = "0.4.8"
	memmap2 = {version = "0.4", features = ["stable_deref_trait"]}			memmap2 = {version = "0.4", features = ["stable_deref_trait"]}
	zstd = "0.5.3"			zstd = "0.5.3"
	format-bytes = "0.2.2"			format-bytes = "0.2.2"
				openat = "0.1.21"

	# We don't use the `miniz-oxide` backend to not change rhg benchmarks and until			# We don't use the `miniz-oxide` backend to not change rhg benchmarks and until
	# we have a clearer view of which backend is the fastest.			# we have a clearer view of which backend is the fastest.
	[dependencies.flate2]			[dependencies.flate2]
	version = "1.0.16"			version = "1.0.16"
	features = ["zlib"]			features = ["zlib"]
	default-features = false			default-features = false

	[dev-dependencies]			[dev-dependencies]
	clap = "*"			clap = "*"
	pretty_assertions = "0.6.1"			pretty_assertions = "0.6.1"

rust/hg-core/src/dirstate/entry.rs

	use crate::dirstate_tree::on_disk::DirstateV2ParseError;			use crate::dirstate_tree::on_disk::DirstateV2ParseError;
	use crate::errors::HgError;			use crate::errors::HgError;
	use bitflags::bitflags;			use bitflags::bitflags;
	use std::convert::{TryFrom, TryInto};			use std::convert::{TryFrom, TryInto};
	use std::fs;
	use std::io;			use std::io;
	use std::time::{SystemTime, UNIX_EPOCH};			use std::time::{SystemTime, UNIX_EPOCH};

	#[derive(Copy, Clone, Debug, Eq, PartialEq)]			#[derive(Copy, Clone, Debug, Eq, PartialEq)]
	pub enum EntryState {			pub enum EntryState {
	Normal,			Normal,
	Added,			Added,
	Removed,			Removed,
	truncated_seconds,			truncated_seconds,
	nanoseconds,			nanoseconds,
	})			})
	} else {			} else {
	Err(DirstateV2ParseError)			Err(DirstateV2ParseError)
	}			}
	}			}

	pub fn for_mtime_of(metadata: &fs::Metadata) -> io::Result<Self> {			pub fn for_mtime_of(metadata: &openat::Metadata) -> io::Result<Self> {
	#[cfg(unix)]			#[cfg(unix)]
	{			{
	use std::os::unix::fs::MetadataExt;			let stat = metadata.stat();
	let seconds = metadata.mtime();			let seconds = stat.st_mtime as libc::time_t;
	// i64!-> u32 with value always in the `0 .. NSEC_PER_SEC` range			let nanoseconds = stat.st_mtime_nsec as libc::c_long;
	let nanoseconds = metadata.mtime_nsec().try_into().unwrap();			let nanoseconds : u32 = nanoseconds.try_into().unwrap();
	Ok(Self::new_truncate(seconds, nanoseconds))			Ok(Self::new_truncate(seconds, nanoseconds))
	}			}
	#[cfg(not(unix))]			#[cfg(not(unix))]
	{			{
	metadata.modified().map(Self::from)			panic!("not supported")
	}			}
	}			}

	/// The lower 31 bits of the number of seconds since the epoch.			/// The lower 31 bits of the number of seconds since the epoch.
	pub fn truncated_seconds(&self) -> u32 {			pub fn truncated_seconds(&self) -> u32 {
	self.truncated_seconds			self.truncated_seconds
	}			}

	self.truncated_seconds == other.truncated_seconds			self.truncated_seconds == other.truncated_seconds
	&& (self.nanoseconds == other.nanoseconds			&& (self.nanoseconds == other.nanoseconds
	\|\| self.nanoseconds == 0			\|\| self.nanoseconds == 0
	\|\| other.nanoseconds == 0)			\|\| other.nanoseconds == 0)
	}			}

	pub fn likely_equal_to_mtime_of(			pub fn likely_equal_to_mtime_of(
	self,			self,
	metadata: &fs::Metadata,			metadata: &openat::Metadata,
	) -> io::Result<bool> {			) -> io::Result<bool> {
	Ok(self.likely_equal(Self::for_mtime_of(metadata)?))			Ok(self.likely_equal(Self::for_mtime_of(metadata)?))
	}			}

	}			}

	impl From<SystemTime> for TruncatedTimestamp {			impl From<SystemTime> for TruncatedTimestamp {
	fn from(system_time: SystemTime) -> Self {			fn from(system_time: SystemTime) -> Self {
	// On Unix, `SystemTime` is a wrapper for the `timespec` C struct:			// On Unix, `SystemTime` is a wrapper for the `timespec` C struct:
	// https://www.gnu.org/software/libc/manual/html_node/Time-Types.html#index-struct-timespec			// https://www.gnu.org/software/libc/manual/html_node/Time-Types.html#index-struct-timespec
	// We want to effectively access its fields, but the Rust standard			// We want to effectively access its fields, but the Rust standard
	// library does not expose them. The best we can do is:			// library does not expose them. The best we can do is:
	self.state() == EntryState::Normal			self.state() == EntryState::Normal
	&& self.size() == SIZE_FROM_OTHER_PARENT			&& self.size() == SIZE_FROM_OTHER_PARENT
	}			}

	// TODO: other platforms			// TODO: other platforms
	#[cfg(unix)]			#[cfg(unix)]
	pub fn mode_changed(			pub fn mode_changed(
	&self,			&self,
	filesystem_metadata: &std::fs::Metadata,			filesystem_metadata: &openat::Metadata,
	) -> bool {			) -> bool {
	use std::os::unix::fs::MetadataExt;
	const EXEC_BIT_MASK: u32 = 0o100;			const EXEC_BIT_MASK: u32 = 0o100;
	let dirstate_exec_bit = (self.mode() as u32) & EXEC_BIT_MASK;			let dirstate_exec_bit = (self.mode() as u32) & EXEC_BIT_MASK;
	let fs_exec_bit = filesystem_metadata.mode() & EXEC_BIT_MASK;			let fs_exec_bit = filesystem_metadata.stat().st_mode & EXEC_BIT_MASK;
	dirstate_exec_bit != fs_exec_bit			dirstate_exec_bit != fs_exec_bit
	}			}

	/// Returns a `(state, mode, size, mtime)` tuple as for			/// Returns a `(state, mode, size, mtime)` tuple as for
	/// `DirstateMapMethods::debug_iter`.			/// `DirstateMapMethods::debug_iter`.
	pub fn debug_tuple(&self) -> (u8, i32, i32, i32) {			pub fn debug_tuple(&self) -> (u8, i32, i32, i32) {
	(self.state().into(), self.mode(), self.size(), self.mtime())			(self.state().into(), self.mode(), self.size(), self.mtime())
	}			}

rust/hg-core/src/dirstate_tree/status.rs

	use crate::dirstate::entry::TruncatedTimestamp;			use crate::dirstate::entry::TruncatedTimestamp;
	use crate::dirstate::status::IgnoreFnType;			use crate::dirstate::status::IgnoreFnType;
	use crate::dirstate_tree::dirstate_map::BorrowedPath;			use crate::dirstate_tree::dirstate_map::BorrowedPath;
	use crate::dirstate_tree::dirstate_map::ChildNodesRef;			use crate::dirstate_tree::dirstate_map::ChildNodesRef;
	use crate::dirstate_tree::dirstate_map::DirstateMap;			use crate::dirstate_tree::dirstate_map::DirstateMap;
	use crate::dirstate_tree::dirstate_map::NodeData;			use crate::dirstate_tree::dirstate_map::NodeData;
	use crate::dirstate_tree::dirstate_map::NodeRef;			use crate::dirstate_tree::dirstate_map::NodeRef;
	use crate::dirstate_tree::on_disk::DirstateV2ParseError;			use crate::dirstate_tree::on_disk::DirstateV2ParseError;
	use crate::matchers::get_ignore_function;			use crate::matchers::get_ignore_function;
	use crate::matchers::Matcher;			use crate::matchers::Matcher;
	use crate::utils::files::get_bytes_from_os_string;
	use crate::utils::files::get_path_from_bytes;
	use crate::utils::hg_path::HgPath;			use crate::utils::hg_path::HgPath;
	use crate::BadMatch;			use crate::BadMatch;
	use crate::DirstateStatus;			use crate::DirstateStatus;
	use crate::EntryState;			use crate::EntryState;
	use crate::HgPathBuf;
	use crate::PatternFileWarning;			use crate::PatternFileWarning;
	use crate::StatusError;			use crate::StatusError;
	use crate::StatusOptions;			use crate::StatusOptions;
	use micro_timer::timed;			use micro_timer::timed;
	use rayon::prelude::*;			use rayon::prelude::*;
	use sha1::{Digest, Sha1};			use sha1::{Digest, Sha1};
	use std::borrow::Cow;			use std::borrow::Cow;
	use std::io;			use std::io;
	use std::path::Path;			use std::path::Path;
	use std::path::PathBuf;			use std::path::PathBuf;
	use std::sync::Mutex;			use std::sync::Mutex;
	use std::time::SystemTime;			use std::time::SystemTime;
				use openat::{Dir};
				use std::ffi::CStr;
				use std::os::unix::ffi::OsStrExt;
				use std::ffi::CString;
				use std::convert::TryInto;

	/// Returns the status of the working directory compared to its parent			/// Returns the status of the working directory compared to its parent
	/// changeset.			/// changeset.
	///			///
	/// This algorithm is based on traversing the filesystem tree (`fs` in function			/// This algorithm is based on traversing the filesystem tree (`fs` in function
	/// and variable names) and dirstate tree at the same time. The core of this			/// and variable names) and dirstate tree at the same time. The core of this
	/// traversal is the recursive `traverse_fs_directory_and_dirstate` function			/// traversal is the recursive `traverse_fs_directory_and_dirstate` function
	/// and its use of `itertools::merge_join_by`. When reaching a path that only			/// and its use of `itertools::merge_join_by`. When reaching a path that only
	let is_at_repo_root = true;			let is_at_repo_root = true;
	let hg_path = &BorrowedPath::OnDisk(HgPath::new(""));			let hg_path = &BorrowedPath::OnDisk(HgPath::new(""));
	let has_ignored_ancestor = false;			let has_ignored_ancestor = false;
	let root_cached_mtime = None;			let root_cached_mtime = None;
	let root_dir_metadata = None;			let root_dir_metadata = None;
	// If the path we have for the repository root is a symlink, do follow it.			// If the path we have for the repository root is a symlink, do follow it.
	// (As opposed to symlinks within the working directory which are not			// (As opposed to symlinks within the working directory which are not
	// followed, using `std::fs::symlink_metadata`.)			// followed, using `std::fs::symlink_metadata`.)
				let root_dir = openat::Dir::open(&root_dir)?;
	common.traverse_fs_directory_and_dirstate(			common.traverse_fs_directory_and_dirstate(
	has_ignored_ancestor,			has_ignored_ancestor,
	dmap.root.as_ref(),			dmap.root.as_ref(),
	hg_path,			hg_path,
	&root_dir,			&root_dir,
	root_dir_metadata,			root_dir_metadata,
	root_cached_mtime,			root_cached_mtime,
	is_at_repo_root,			is_at_repo_root,
	/// and therefore isn’t reading `.hgignore`.			/// and therefore isn’t reading `.hgignore`.
	ignore_patterns_have_changed: Option<bool>,			ignore_patterns_have_changed: Option<bool>,

	/// The current time at the start of the `status()` algorithm, as measured			/// The current time at the start of the `status()` algorithm, as measured
	/// and possibly truncated by the filesystem.			/// and possibly truncated by the filesystem.
	filesystem_time_at_status_start: Option<SystemTime>,			filesystem_time_at_status_start: Option<SystemTime>,
	}			}

				enum MetadataOrType<'a> {
				Metadata(&'a openat::Metadata),
				FileType(openat::SimpleType)
				}

				struct FilesystemEntry<'a> {
				dir : &'a Dir,
				name : &'a CStr,
				metadata_or_type : MetadataOrType<'a>
				}

				enum TORREF <'a, T> {
				Owned(T),
				Borrowed(&'a T)
				}

				impl <'a, T> TORREF <'a, T> {
				fn borrow(&self) -> &T {
				match self {
				TORREF::Owned(t) => &t,
				TORREF::Borrowed(r) => r
				}
				}
				}

				impl <'a> FilesystemEntry<'a> {

				fn of_read_dir(dir : &'a Dir, entry : &'a openat::Entry) -> Self {
				FilesystemEntry {
				dir, name : <& openat::Entry as openat::AsPath>::to_path(entry).unwrap(), metadata_or_type : MetadataOrType::FileType(entry.simple_type().unwrap()) // TODO: not unwrap
				}
				}

				fn of_metadata(dir : &'a Dir, name : &'a CStr, metadata : &'a openat::Metadata) -> Self {
				FilesystemEntry {
				dir, name, metadata_or_type : MetadataOrType::Metadata(metadata)
				}
				}

				fn file_type(&self) -> openat::SimpleType {
				match self.metadata_or_type {
				MetadataOrType::Metadata(metadata) => metadata.simple_type (),
				MetadataOrType::FileType(file_type) => file_type
				}
				}

				// Returns None if the entry is not a directory
				fn as_directory(&self) -> io::Result<Option<(Dir, openat::Metadata)>> {
				match self.file_type() {
				openat::SimpleType::Dir => {
				let dir = self.dir.sub_dir(self.name)?;
				let metadata = dir.self_metadata()?;
				Ok(Some((dir, metadata)))
				},
				_ => Ok(None)
				}
				}

				fn metadata(&self) -> io::Result<TORREF<'a, openat::Metadata>> {
				match self.metadata_or_type {
				MetadataOrType::Metadata(m) => Ok(TORREF::Borrowed(m)),
				MetadataOrType::FileType(_) => {
				let metadata = self.dir.metadata(self.name)?;
				Ok(TORREF::Owned(metadata)) }
				}
				}
				}

				fn stat_mtime(metadata : &openat::Metadata) -> SystemTime {
				let stat = metadata.stat();
				let nsec : i64 = stat.st_mtime_nsec as libc::c_long;
				let nsec : u32 = nsec.try_into().unwrap();
				let sec : i64 = stat.st_mtime as libc::time_t;
				let sec : u64 = sec.try_into().unwrap();
				let dur = std::time::Duration::new(sec, nsec);
				SystemTime::UNIX_EPOCH.checked_add(dur).unwrap()
				}



	impl<'a, 'tree, 'on_disk> StatusCommon<'a, 'tree, 'on_disk> {			impl<'a, 'tree, 'on_disk> StatusCommon<'a, 'tree, 'on_disk> {
	fn read_dir(			fn read_dir(
	&self,			&self,
	hg_path: &HgPath,			hg_path: &HgPath,
	fs_path: &Path,			fs_dir: &Dir,
	is_at_repo_root: bool,			is_at_repo_root: bool,
	) -> Result<Vec<DirEntry>, ()> {			) -> Result<Vec<openat::Entry>, ()> {
	DirEntry::read_dir(fs_path, is_at_repo_root)			read_dir(fs_dir, is_at_repo_root)
	.map_err(\|error\| self.io_error(error, hg_path))			.map_err(\|error\| self.io_error(error, hg_path))
	}			}

	fn io_error(&self, error: std::io::Error, hg_path: &HgPath) {			fn io_error(&self, error: std::io::Error, hg_path: &HgPath) {
	let errno = error.raw_os_error().expect("expected real OS error");			let errno = error.raw_os_error().expect("expected real OS error");
	self.outcome			self.outcome
	.lock()			.lock()
	.unwrap()			.unwrap()
	Ok(())			Ok(())
	}			}

	/// If this returns true, we can get accurate results by only using			/// If this returns true, we can get accurate results by only using
	/// `symlink_metadata` for child nodes that exist in the dirstate and don’t			/// `symlink_metadata` for child nodes that exist in the dirstate and don’t
	/// need to call `read_dir`.			/// need to call `read_dir`.
	fn can_skip_fs_readdir(			fn can_skip_fs_readdir(
	&self,			&self,
	directory_metadata: Option<&std::fs::Metadata>,			directory_metadata: Option<&openat::Metadata>,
	cached_directory_mtime: Option<TruncatedTimestamp>,			cached_directory_mtime: Option<TruncatedTimestamp>,
	) -> bool {			) -> bool {
	if !self.options.list_unknown && !self.options.list_ignored {			if !self.options.list_unknown && !self.options.list_ignored {
	// All states that we care about listing have corresponding			// All states that we care about listing have corresponding
	// dirstate entries.			// dirstate entries.
	// This happens for example with `hg status -mard`.			// This happens for example with `hg status -mard`.
	return true;			return true;
	}			}

	/// Returns whether all child entries of the filesystem directory have a			/// Returns whether all child entries of the filesystem directory have a
	/// corresponding dirstate node or are ignored.			/// corresponding dirstate node or are ignored.
	fn traverse_fs_directory_and_dirstate(			fn traverse_fs_directory_and_dirstate(
	&self,			&self,
	has_ignored_ancestor: bool,			has_ignored_ancestor: bool,
	dirstate_nodes: ChildNodesRef<'tree, 'on_disk>,			dirstate_nodes: ChildNodesRef<'tree, 'on_disk>,
	directory_hg_path: &BorrowedPath<'tree, 'on_disk>,			directory_hg_path: &BorrowedPath<'tree, 'on_disk>,
	directory_fs_path: &Path,			directory_fd: &Dir,
	directory_metadata: Option<&std::fs::Metadata>,			directory_metadata: Option<&openat::Metadata>,
	cached_directory_mtime: Option<TruncatedTimestamp>,			cached_directory_mtime: Option<TruncatedTimestamp>,
	is_at_repo_root: bool,			is_at_repo_root: bool,
	) -> Result<bool, DirstateV2ParseError> {			) -> Result<bool, DirstateV2ParseError> {
	if self.can_skip_fs_readdir(directory_metadata, cached_directory_mtime)			if self.can_skip_fs_readdir(directory_metadata, cached_directory_mtime)
	{			{
	dirstate_nodes			dirstate_nodes
	.par_iter()			.par_iter()
	.map(\|dirstate_node\| {			.map(\|dirstate_node\| {
	let fs_path = directory_fs_path.join(get_path_from_bytes(			let base_name = dirstate_node.base_name(self.dmap.on_disk)?;
	dirstate_node.base_name(self.dmap.on_disk)?.as_bytes(),			let base_name_cstring = CString::new(base_name.as_bytes()).expect("NULL BYTE");
	));			match directory_fd.metadata(base_name_cstring.as_ref()) {
	match std::fs::symlink_metadata(&fs_path) {
	Ok(fs_metadata) => self.traverse_fs_and_dirstate(			Ok(fs_metadata) => self.traverse_fs_and_dirstate(
	&fs_path,			&FilesystemEntry::of_metadata(directory_fd, base_name_cstring.as_ref(), &fs_metadata),
	&fs_metadata,
	dirstate_node,			dirstate_node,
	has_ignored_ancestor,			has_ignored_ancestor,
	),			),
	Err(e) if e.kind() == std::io::ErrorKind::NotFound => {			Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
	self.traverse_dirstate_only(dirstate_node)			self.traverse_dirstate_only(dirstate_node)
	}			}
	Err(error) => {			Err(error) => {
	let hg_path =			let hg_path =
	dirstate_node.full_path(self.dmap.on_disk)?;			dirstate_node.full_path(self.dmap.on_disk)?;
	Ok(self.io_error(error, hg_path))			Ok(self.io_error(error, hg_path))
	}			}
	}			}
	})			})
	.collect::<Result<_, _>>()?;			.collect::<Result<_, _>>()?;

	// We don’t know, so conservatively say this isn’t the case			// We don’t know, so conservatively say this isn’t the case
	let children_all_have_dirstate_node_or_are_ignored = false;			let children_all_have_dirstate_node_or_are_ignored = false;

	return Ok(children_all_have_dirstate_node_or_are_ignored);			return Ok(children_all_have_dirstate_node_or_are_ignored);
	}			}

	let mut fs_entries = if let Ok(entries) = self.read_dir(			let mut fs_entries = if let Ok(entries) = self.read_dir(
	directory_hg_path,			directory_hg_path,
	directory_fs_path,			directory_fd,
	is_at_repo_root,			is_at_repo_root,
	) {			) {
	entries			entries
	} else {			} else {
	// Treat an unreadable directory (typically because of insufficient			// Treat an unreadable directory (typically because of insufficient
	// permissions) like an empty directory. `self.read_dir` has			// permissions) like an empty directory. `self.read_dir` has
	// already called `self.io_error` so a warning will be emitted.			// already called `self.io_error` so a warning will be emitted.
	Vec::new()			Vec::new()
	};			};

	// `merge_join_by` requires both its input iterators to be sorted:			// `merge_join_by` requires both its input iterators to be sorted:

	let dirstate_nodes = dirstate_nodes.sorted();			let dirstate_nodes = dirstate_nodes.sorted();
	// `sort_unstable_by_key` doesn’t allow keys borrowing from the value:			// `sort_unstable_by_key` doesn’t allow keys borrowing from the value:
	// https://github.com/rust-lang/rust/issues/34162			// https://github.com/rust-lang/rust/issues/34162
	fs_entries.sort_unstable_by(\|e1, e2\| e1.base_name.cmp(&e2.base_name));			fs_entries.sort_unstable_by(\|e1, e2\| e1.file_name().as_bytes().cmp(&e2.file_name().as_bytes()));

	// Propagate here any error that would happen inside the comparison			// Propagate here any error that would happen inside the comparison
	// callback below			// callback below
	for dirstate_node in &dirstate_nodes {			for dirstate_node in &dirstate_nodes {
	dirstate_node.base_name(self.dmap.on_disk)?;			dirstate_node.base_name(self.dmap.on_disk)?;
	}			}
	itertools::merge_join_by(			itertools::merge_join_by(
	dirstate_nodes,			dirstate_nodes,
	&fs_entries,			&fs_entries,
	\|dirstate_node, fs_entry\| {			\|dirstate_node, fs_entry\| {
	// This `unwrap` never panics because we already propagated			// This `unwrap` never panics because we already propagated
	// those errors above			// those errors above
	dirstate_node			dirstate_node
	.base_name(self.dmap.on_disk)			.base_name(self.dmap.on_disk)
	.unwrap()			.unwrap()
	.cmp(&fs_entry.base_name)			.as_bytes()
				.cmp(&fs_entry.file_name().as_bytes())
	},			},
	)			)
	.par_bridge()			.par_bridge()
	.map(\|pair\| {			.map(\|pair\| {
	use itertools::EitherOrBoth::*;			use itertools::EitherOrBoth::*;
	let has_dirstate_node_or_is_ignored;			let has_dirstate_node_or_is_ignored;
	match pair {			match pair {
	Both(dirstate_node, fs_entry) => {			Both(dirstate_node, fs_entry) => {
	self.traverse_fs_and_dirstate(			self.traverse_fs_and_dirstate(
	&fs_entry.full_path,			&FilesystemEntry::of_read_dir(directory_fd, &fs_entry),
	&fs_entry.metadata,
	dirstate_node,			dirstate_node,
	has_ignored_ancestor,			has_ignored_ancestor,
	)?;			)?;
	has_dirstate_node_or_is_ignored = true			has_dirstate_node_or_is_ignored = true
	}			}
	Left(dirstate_node) => {			Left(dirstate_node) => {
	self.traverse_dirstate_only(dirstate_node)?;			self.traverse_dirstate_only(dirstate_node)?;
	has_dirstate_node_or_is_ignored = true;			has_dirstate_node_or_is_ignored = true;
	}			}
	Right(fs_entry) => {			Right(fs_entry) => {
	has_dirstate_node_or_is_ignored = self.traverse_fs_only(			has_dirstate_node_or_is_ignored = self.traverse_fs_only(
	has_ignored_ancestor,			has_ignored_ancestor,
	directory_hg_path,			directory_hg_path,
				directory_fd,
	fs_entry,			fs_entry,
	)			)
	}			}
	}			}
	Ok(has_dirstate_node_or_is_ignored)			Ok(has_dirstate_node_or_is_ignored)
	})			})
	.try_reduce(\|\| true, \|a, b\| Ok(a && b))			.try_reduce(\|\| true, \|a, b\| Ok(a && b))
	}			}

	fn traverse_fs_and_dirstate(			fn traverse_fs_and_dirstate(
	&self,			&self,
	fs_path: &Path,			fs_entry: &FilesystemEntry,
	fs_metadata: &std::fs::Metadata,
	dirstate_node: NodeRef<'tree, 'on_disk>,			dirstate_node: NodeRef<'tree, 'on_disk>,
	has_ignored_ancestor: bool,			has_ignored_ancestor: bool,
	) -> Result<(), DirstateV2ParseError> {			) -> Result<(), DirstateV2ParseError> {
	self.check_for_outdated_directory_cache(&dirstate_node)?;			self.check_for_outdated_directory_cache(&dirstate_node)?;
	let hg_path = &dirstate_node.full_path_borrowed(self.dmap.on_disk)?;			let hg_path = &dirstate_node.full_path_borrowed(self.dmap.on_disk)?;
	let file_type = fs_metadata.file_type();			let file_type = fs_entry.file_type();
	let file_or_symlink = file_type.is_file() \|\| file_type.is_symlink();			let file_or_symlink =
				match file_type {
				openat::SimpleType::File \| openat::SimpleType::Symlink => true,
				openat::SimpleType::Dir \| openat::SimpleType::Other => false,
				};
	if !file_or_symlink {			if !file_or_symlink {
	// If we previously had a file here, it was removed (with			// If we previously had a file here, it was removed (with
	// `hg rm` or similar) or deleted before it could be			// `hg rm` or similar) or deleted before it could be
	// replaced by a directory or something else.			// replaced by a directory or something else.
	self.mark_removed_or_deleted_if_file(			self.mark_removed_or_deleted_if_file(
	&hg_path,			&hg_path,
	dirstate_node.state()?,			dirstate_node.state()?,
	);			);
	}			}
	if file_type.is_dir() {			match fs_entry.as_directory().expect("TODO: handle") {
				Some ((dir, fs_metadata)) => {
	if self.options.collect_traversed_dirs {			if self.options.collect_traversed_dirs {
	self.outcome			self.outcome
	.lock()			.lock()
	.unwrap()			.unwrap()
	.traversed			.traversed
	.push(hg_path.detach_from_tree())			.push(hg_path.detach_from_tree())
	}			}
	let is_ignored = has_ignored_ancestor \|\| (self.ignore_fn)(hg_path);			let is_ignored = has_ignored_ancestor \|\| (self.ignore_fn)(hg_path);
	let is_at_repo_root = false;			let is_at_repo_root = false;
	let children_all_have_dirstate_node_or_are_ignored = self			let children_all_have_dirstate_node_or_are_ignored = self
	.traverse_fs_directory_and_dirstate(			.traverse_fs_directory_and_dirstate(
	is_ignored,			is_ignored,
	dirstate_node.children(self.dmap.on_disk)?,			dirstate_node.children(self.dmap.on_disk)?,
	hg_path,			hg_path,
	fs_path,			&dir,
	Some(fs_metadata),			Some(&fs_metadata),
	dirstate_node.cached_directory_mtime()?,			dirstate_node.cached_directory_mtime()?,
	is_at_repo_root,			is_at_repo_root,
	)?;			)?;
	self.maybe_save_directory_mtime(			self.maybe_save_directory_mtime(
	children_all_have_dirstate_node_or_are_ignored,			children_all_have_dirstate_node_or_are_ignored,
	fs_metadata,			&fs_metadata,
	dirstate_node,			dirstate_node,
	)?			)?
	} else {			},
				None => {
	if file_or_symlink && self.matcher.matches(hg_path) {			if file_or_symlink && self.matcher.matches(hg_path) {
	if let Some(state) = dirstate_node.state()? {			if let Some(state) = dirstate_node.state()? {
	match state {			match state {
	EntryState::Added => self			EntryState::Added => self
	.outcome			.outcome
	.lock()			.lock()
	.unwrap()			.unwrap()
	.added			.added
	.push(hg_path.detach_from_tree()),			.push(hg_path.detach_from_tree()),
	EntryState::Removed => self			EntryState::Removed => self
	.outcome			.outcome
	.lock()			.lock()
	.unwrap()			.unwrap()
	.removed			.removed
	.push(hg_path.detach_from_tree()),			.push(hg_path.detach_from_tree()),
	EntryState::Merged => self			EntryState::Merged => self
	.outcome			.outcome
	.lock()			.lock()
	.unwrap()			.unwrap()
	.modified			.modified
	.push(hg_path.detach_from_tree()),			.push(hg_path.detach_from_tree()),
	EntryState::Normal => self			EntryState::Normal => {
	.handle_normal_file(&dirstate_node, fs_metadata)?,			let fs_metadata = fs_entry.metadata().expect("TODO");
				self.handle_normal_file(
				&dirstate_node,
				fs_metadata.borrow(),
				)?
				}
	}			}
	} else {			} else {
	// `node.entry.is_none()` indicates a "directory"			// `node.entry.is_none()` indicates a "directory"
	// node, but the filesystem has a file			// node, but the filesystem has a file
	self.mark_unknown_or_ignored(			self.mark_unknown_or_ignored(
	has_ignored_ancestor,			has_ignored_ancestor,
	hg_path,			hg_path,
	);			);
	}			}
	}			}

	for child_node in dirstate_node.children(self.dmap.on_disk)?.iter()			for child_node in dirstate_node.children(self.dmap.on_disk)?.iter()
	{			{
	self.traverse_dirstate_only(child_node)?			self.traverse_dirstate_only(child_node)?
	}			}
	}			},
				};
	Ok(())			Ok(())
	}			}


	fn maybe_save_directory_mtime(			fn maybe_save_directory_mtime(
	&self,			&self,
	children_all_have_dirstate_node_or_are_ignored: bool,			children_all_have_dirstate_node_or_are_ignored: bool,
	directory_metadata: &std::fs::Metadata,			directory_metadata: &openat::Metadata,
	dirstate_node: NodeRef<'tree, 'on_disk>,			dirstate_node: NodeRef<'tree, 'on_disk>,
	) -> Result<(), DirstateV2ParseError> {			) -> Result<(), DirstateV2ParseError> {
	if children_all_have_dirstate_node_or_are_ignored {			if children_all_have_dirstate_node_or_are_ignored {
	// All filesystem directory entries from `read_dir` have a			// All filesystem directory entries from `read_dir` have a
	// corresponding node in the dirstate, so we can reconstitute the			// corresponding node in the dirstate, so we can reconstitute the
	// names of those entries without calling `read_dir` again.			// names of those entries without calling `read_dir` again.
	if let (Some(status_start), Ok(directory_mtime)) = (			if let (Some(status_start), directory_mtime) = (
	&self.filesystem_time_at_status_start,			&self.filesystem_time_at_status_start,
	directory_metadata.modified(),			stat_mtime(directory_metadata),
	) {			) {
	// Although the Rust standard library’s `SystemTime` type			// Although the Rust standard library’s `SystemTime` type
	// has nanosecond precision, the times reported for a			// has nanosecond precision, the times reported for a
	// directory’s (or file’s) modified time may have lower			// directory’s (or file’s) modified time may have lower
	// resolution based on the filesystem (for example ext3			// resolution based on the filesystem (for example ext3
	// only stores integer seconds), kernel (see			// only stores integer seconds), kernel (see
	// https://stackoverflow.com/a/14393315/1162888), etc.			// https://stackoverflow.com/a/14393315/1162888), etc.
	if &directory_mtime >= status_start {			if &directory_mtime >= status_start {
	Ok(())			Ok(())
	}			}

	/// A file with `EntryState::Normal` in the dirstate was found in the			/// A file with `EntryState::Normal` in the dirstate was found in the
	/// filesystem			/// filesystem
	fn handle_normal_file(			fn handle_normal_file(
	&self,			&self,
	dirstate_node: &NodeRef<'tree, 'on_disk>,			dirstate_node: &NodeRef<'tree, 'on_disk>,
	fs_metadata: &std::fs::Metadata,			fs_metadata: &openat::Metadata,
	) -> Result<(), DirstateV2ParseError> {			) -> Result<(), DirstateV2ParseError> {
	// Keep the low 31 bits			// Keep the low 31 bits
	fn truncate_u64(value: u64) -> i32 {			fn truncate_u64(value: u64) -> i32 {
	(value & 0x7FFF_FFFF) as i32			(value & 0x7FFF_FFFF) as i32
	}			}

	let entry = dirstate_node			let entry = dirstate_node
	.entry()?			.entry()?
	.expect("handle_normal_file called with entry-less node");			.expect("handle_normal_file called with entry-less node");
	let hg_path = &dirstate_node.full_path_borrowed(self.dmap.on_disk)?;			let hg_path = &dirstate_node.full_path_borrowed(self.dmap.on_disk)?;
	let mode_changed =			let mode_changed =
	\|\| self.options.check_exec && entry.mode_changed(fs_metadata);			\|\| self.options.check_exec && entry.mode_changed(fs_metadata);
	let size = entry.size();			let size = entry.size();
	let size_changed = size != truncate_u64(fs_metadata.len());			let size_changed = size != truncate_u64(fs_metadata.len());
	if size >= 0 && size_changed && fs_metadata.file_type().is_symlink() {			if size >= 0 && size_changed && fs_metadata.simple_type() == openat::SimpleType::Symlink {
	// issue6456: Size returned may be longer due to encryption			// issue6456: Size returned may be longer due to encryption
	// on EXT-4 fscrypt. TODO maybe only do it on EXT4?			// on EXT-4 fscrypt. TODO maybe only do it on EXT4?
	self.outcome			self.outcome
	.lock()			.lock()
	.unwrap()			.unwrap()
	.unsure			.unsure
	.push(hg_path.detach_from_tree())			.push(hg_path.detach_from_tree())
	} else if dirstate_node.has_copy_source()			} else if dirstate_node.has_copy_source()

	/// Something in the filesystem has no corresponding dirstate node			/// Something in the filesystem has no corresponding dirstate node
	///			///
	/// Returns whether that path is ignored			/// Returns whether that path is ignored
	fn traverse_fs_only(			fn traverse_fs_only(
	&self,			&self,
	has_ignored_ancestor: bool,			has_ignored_ancestor: bool,
	directory_hg_path: &HgPath,			directory_hg_path: &HgPath,
	fs_entry: &DirEntry,			parent_dir : &Dir,
				fs_entry: &openat::Entry,
	) -> bool {			) -> bool {
	let hg_path = directory_hg_path.join(&fs_entry.base_name);			let hg_path = directory_hg_path.join(HgPath::new(&fs_entry.file_name ().as_bytes()));
	let file_type = fs_entry.metadata.file_type();			let file_type = fs_entry.simple_type().unwrap(); // TODO: fix unwrap
	let file_or_symlink = file_type.is_file() \|\| file_type.is_symlink();			let file_or_symlink = match file_type {
	if file_type.is_dir() {			openat::SimpleType::File \| openat::SimpleType::Symlink => true,
				_ => false };
				if openat::SimpleType::Dir == file_type {
	let is_ignored =			let is_ignored =
	has_ignored_ancestor \|\| (self.ignore_fn)(&hg_path);			has_ignored_ancestor \|\| (self.ignore_fn)(&hg_path);
	let traverse_children = if is_ignored {			let traverse_children = if is_ignored {
	// Descendants of an ignored directory are all ignored			// Descendants of an ignored directory are all ignored
	self.options.list_ignored			self.options.list_ignored
	} else {			} else {
	// Descendants of an unknown directory may be either unknown or			// Descendants of an unknown directory may be either unknown or
	// ignored			// ignored
	self.options.list_unknown \|\| self.options.list_ignored			self.options.list_unknown \|\| self.options.list_ignored
	};			};
	if traverse_children {			if traverse_children {
	let is_at_repo_root = false;			let is_at_repo_root = false;
				let dir = parent_dir.sub_dir(fs_entry.file_name()).expect("TODO");
	if let Ok(children_fs_entries) = self.read_dir(			if let Ok(children_fs_entries) = self.read_dir(
	&hg_path,			&hg_path,
	&fs_entry.full_path,			&dir,
	is_at_repo_root,			is_at_repo_root,
	) {			) {
	children_fs_entries.par_iter().for_each(\|child_fs_entry\| {			children_fs_entries.par_iter().for_each(\|child_fs_entry\| {
	self.traverse_fs_only(			self.traverse_fs_only(
	is_ignored,			is_ignored,
	&hg_path,			&hg_path,
				&dir,
	child_fs_entry,			child_fs_entry,
	);			);
	})			})
	}			}
	}			}
	if self.options.collect_traversed_dirs {			if self.options.collect_traversed_dirs {
	self.outcome.lock().unwrap().traversed.push(hg_path.into())			self.outcome.lock().unwrap().traversed.push(hg_path.into())
	}			}
	.unknown			.unknown
	.push(hg_path.detach_from_tree())			.push(hg_path.detach_from_tree())
	}			}
	}			}
	is_ignored			is_ignored
	}			}
	}			}

	struct DirEntry {
	base_name: HgPathBuf,
	full_path: PathBuf,
	metadata: std::fs::Metadata,
	}

	impl DirEntry {
	/// Returns unsorted entries in the given directory, with name and			/// Returns unsorted entries in the given directory, with name and
	/// metadata.			/// metadata.
	///			///
	/// If a `.hg` sub-directory is encountered:			/// If a `.hg` sub-directory is encountered:
	///			///
	/// * At the repository root, ignore that sub-directory			/// * At the repository root, ignore that sub-directory
	/// * Elsewhere, we’re listing the content of a sub-repo. Return an empty			/// * Elsewhere, we’re listing the content of a sub-repo. Return an empty
	/// list instead.			/// list instead.
	fn read_dir(path: &Path, is_at_repo_root: bool) -> io::Result<Vec<Self>> {			fn read_dir(dir: &openat::Dir, is_at_repo_root: bool) -> io::Result<Vec<openat::Entry>> {
	let mut results = Vec::new();			let mut results = Vec::new();
	for entry in path.read_dir()? {			for entry in dir.list_dir(".")? {
	let entry = entry?;			let entry = entry?;
	let metadata = entry.metadata()?;			let name = entry.file_name();
	let name = get_bytes_from_os_string(entry.file_name());
	// FIXME don't do this when cached			// FIXME don't do this when cached
	if name == b".hg" {			if name.as_bytes() == b".hg" {
	if is_at_repo_root {			if is_at_repo_root {
	// Skip the repo’s own .hg (might be a symlink)			// Skip the repo’s own .hg (might be a symlink)
	continue;			continue;
	} else if metadata.is_dir() {			// TODO: handle simple_type returning None
				} else if Some(openat::SimpleType::Dir) == entry.simple_type() {
	// A .hg sub-directory at another location means a subrepo,			// A .hg sub-directory at another location means a subrepo,
	// skip it entirely.			// skip it entirely.
	return Ok(Vec::new());			return Ok(Vec::new());
	}			}
	}			}
	results.push(DirEntry {			results.push(entry)
	base_name: name.into(),
	full_path: entry.path(),
	metadata,
	})
	}			}
	Ok(results)			Ok(results)
	}			}
	}

	/// Return the `mtime` of a temporary file newly-created in the `.hg` directory			/// Return the `mtime` of a temporary file newly-created in the `.hg` directory
	/// of the give repository.			/// of the give repository.
	///			///
	/// This is similar to `SystemTime::now()`, with the result truncated to the			/// This is similar to `SystemTime::now()`, with the result truncated to the
	/// same time resolution as other files’ modification times. Using `.hg`			/// same time resolution as other files’ modification times. Using `.hg`
	/// instead of the system’s default temporary directory (such as `/tmp`) makes			/// instead of the system’s default temporary directory (such as `/tmp`) makes
	/// it more likely the temporary file is in the same disk partition as contents			/// it more likely the temporary file is in the same disk partition as contents

Diff	ID	Base	Description	Created	Lint	Unit
Base			Base
Diff 1	31011			Nov 2 2021, 8:06 AM	★	★