This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10804

rhg: Add support for dirstate-v2
ClosedPublic

Authored by SimonSapin on May 31 2021, 8:27 AM.

Details

Reviewers
Alphare
pulkit
Group Reviewers
hg-reviewers
Commits
rHGbd88b6bfd8da: rhg: Add support for dirstate-v2

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

SimonSapin created this revision.May 31 2021, 8:27 AM
Alphare accepted this revision.Jun 3 2021, 4:51 AM
This revision is now accepted and ready to land.Jun 3 2021, 4:51 AM
pulkit accepted this revision.Jun 4 2021, 5:58 AM
SimonSapin added a commit: rHGbd88b6bfd8da: rhg: Add support for dirstate-v2.Jun 7 2021, 3:44 AM
Closed by commit rHGbd88b6bfd8da: rhg: Add support for dirstate-v2 (authored by SimonSapin). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 28462

rust/hg-core/src/dirstate.rs

// dirstate module // dirstate module
// //
// Copyright 2019 Raphaël Gomès <rgomes@octobus.net> // Copyright 2019 Raphaël Gomès <rgomes@octobus.net>
// //
// This software may be used and distributed according to the terms of the // This software may be used and distributed according to the terms of the
// GNU General Public License version 2 or any later version. // GNU General Public License version 2 or any later version.
use crate::dirstate_tree::on_disk::DirstateV2ParseError; use crate::dirstate_tree::on_disk::DirstateV2ParseError;
use crate::errors::HgError; use crate::errors::HgError;
use crate::revlog::node::NULL_NODE;
use crate::revlog::Node; use crate::revlog::Node;
use crate::utils::hg_path::{HgPath, HgPathBuf}; use crate::utils::hg_path::{HgPath, HgPathBuf};
use crate::FastHashMap; use crate::FastHashMap;
use bytes_cast::{unaligned, BytesCast}; use bytes_cast::{unaligned, BytesCast};
use std::convert::TryFrom; use std::convert::TryFrom;
pub mod dirs_multiset; pub mod dirs_multiset;
pub mod dirstate_map; pub mod dirstate_map;
pub mod parsers; pub mod parsers;
pub mod status; pub mod status;
#[derive(Debug, PartialEq, Clone, BytesCast)] #[derive(Debug, PartialEq, Clone, BytesCast)]
#[repr(C)] #[repr(C)]
pub struct DirstateParents { pub struct DirstateParents {
pub p1: Node, pub p1: Node,
pub p2: Node, pub p2: Node,
} }
impl DirstateParents {
pub const NULL: Self = Self {
p1: NULL_NODE,
p2: NULL_NODE,
};
}
/// The C implementation uses all signed types. This will be an issue /// The C implementation uses all signed types. This will be an issue
/// either when 4GB+ source files are commonplace or in 2038, whichever /// either when 4GB+ source files are commonplace or in 2038, whichever
/// comes first. /// comes first.
#[derive(Debug, PartialEq, Copy, Clone)] #[derive(Debug, PartialEq, Copy, Clone)]
pub struct DirstateEntry { pub struct DirstateEntry {
pub state: EntryState, pub state: EntryState,
pub mode: i32, pub mode: i32,
pub mtime: i32, pub mtime: i32,

rust/hg-core/src/dirstate_tree.rs

pub mod dirstate_map; pub mod dirstate_map;
pub mod dispatch; pub mod dispatch;
pub mod on_disk; pub mod on_disk;
pub mod path_with_basename; pub mod path_with_basename;
mod status; pub mod status;

rust/hg-core/src/dirstate_tree/on_disk.rs

} }
impl From<DirstateV2ParseError> for crate::DirstateError { impl From<DirstateV2ParseError> for crate::DirstateError {
fn from(error: DirstateV2ParseError) -> Self { fn from(error: DirstateV2ParseError) -> Self {
HgError::from(error).into() HgError::from(error).into()
} }
} }
fn read_header(on_disk: &[u8]) -> Result<&Header, DirstateV2ParseError> {
let (header, _) =
Header::from_bytes(on_disk).map_err(|_| DirstateV2ParseError)?;
if header.marker == *V2_FORMAT_MARKER {
Ok(header)
} else {
Err(DirstateV2ParseError)
}
}
pub(super) fn read<'on_disk>( pub(super) fn read<'on_disk>(
on_disk: &'on_disk [u8], on_disk: &'on_disk [u8],
) -> Result< ) -> Result<
(DirstateMap<'on_disk>, Option<DirstateParents>), (DirstateMap<'on_disk>, Option<DirstateParents>),
DirstateV2ParseError, DirstateV2ParseError,
> { > {
if on_disk.is_empty() { if on_disk.is_empty() {
return Ok((DirstateMap::empty(on_disk), None)); return Ok((DirstateMap::empty(on_disk), None));
} }
let (header, _) = let header = read_header(on_disk)?;
Header::from_bytes(on_disk).map_err(|_| DirstateV2ParseError)?;
let Header {
marker,
parents,
root,
nodes_with_entry_count,
nodes_with_copy_source_count,
} = header;
if marker != V2_FORMAT_MARKER {
return Err(DirstateV2ParseError);
}
let dirstate_map = DirstateMap { let dirstate_map = DirstateMap {
on_disk, on_disk,
root: dirstate_map::ChildNodes::OnDisk(read_slice::<Node>( root: dirstate_map::ChildNodes::OnDisk(read_slice::<Node>(
on_disk, *root, on_disk,
header.root,
)?), )?),
nodes_with_entry_count: nodes_with_entry_count.get(), nodes_with_entry_count: header.nodes_with_entry_count.get(),
nodes_with_copy_source_count: nodes_with_copy_source_count.get(), nodes_with_copy_source_count: header
.nodes_with_copy_source_count
.get(),
}; };
let parents = Some(parents.clone()); let parents = Some(header.parents.clone());
Ok((dirstate_map, parents)) Ok((dirstate_map, parents))
} }
impl Node { impl Node {
pub(super) fn full_path<'on_disk>( pub(super) fn full_path<'on_disk>(
&self, &self,
on_disk: &'on_disk [u8], on_disk: &'on_disk [u8],
) -> Result<&'on_disk HgPath, DirstateV2ParseError> { ) -> Result<&'on_disk HgPath, DirstateV2ParseError> {
let len = usize::try_from(slice.len.get()).unwrap_or(std::usize::MAX); let len = usize::try_from(slice.len.get()).unwrap_or(std::usize::MAX);
on_disk on_disk
.get(start..) .get(start..)
.and_then(|bytes| T::slice_from_bytes(bytes, len).ok()) .and_then(|bytes| T::slice_from_bytes(bytes, len).ok())
.map(|(slice, _rest)| slice) .map(|(slice, _rest)| slice)
.ok_or_else(|| DirstateV2ParseError) .ok_or_else(|| DirstateV2ParseError)
} }
pub(crate) fn parse_dirstate_parents(
on_disk: &[u8],
) -> Result<&DirstateParents, HgError> {
Ok(&read_header(on_disk)?.parents)
}
pub(crate) fn for_each_tracked_path<'on_disk>(
on_disk: &'on_disk [u8],
mut f: impl FnMut(&'on_disk HgPath),
) -> Result<(), DirstateV2ParseError> {
let header = read_header(on_disk)?;
fn recur<'on_disk>(
on_disk: &'on_disk [u8],
nodes: Slice,
f: &mut impl FnMut(&'on_disk HgPath),
) -> Result<(), DirstateV2ParseError> {
for node in read_slice::<Node>(on_disk, nodes)? {
if let Some(state) = node.state()? {
if state.is_tracked() {
f(node.full_path(on_disk)?)
}
}
recur(on_disk, node.children, f)?
}
Ok(())
}
recur(on_disk, header.root, &mut f)
}
pub(super) fn write( pub(super) fn write(
dirstate_map: &mut DirstateMap, dirstate_map: &mut DirstateMap,
parents: DirstateParents, parents: DirstateParents,
) -> Result<Vec<u8>, DirstateError> { ) -> Result<Vec<u8>, DirstateError> {
let header_len = std::mem::size_of::<Header>(); let header_len = std::mem::size_of::<Header>();
// This ignores the space for paths, and for nodes without an entry. // This ignores the space for paths, and for nodes without an entry.
// TODO: better estimate? Skip the `Vec` and write to a file directly? // TODO: better estimate? Skip the `Vec` and write to a file directly?

rust/hg-core/src/operations/list_tracked_files.rs

// list_tracked_files.rs // list_tracked_files.rs
// //
// Copyright 2020 Antoine Cezar <antoine.cezar@octobus.net> // Copyright 2020 Antoine Cezar <antoine.cezar@octobus.net>
// //
// This software may be used and distributed according to the terms of the // This software may be used and distributed according to the terms of the
// GNU General Public License version 2 or any later version. // GNU General Public License version 2 or any later version.
use crate::dirstate::parsers::parse_dirstate_entries; use crate::dirstate::parsers::parse_dirstate_entries;
use crate::dirstate_tree::on_disk::for_each_tracked_path;
use crate::errors::HgError; use crate::errors::HgError;
use crate::repo::Repo; use crate::repo::Repo;
use crate::revlog::changelog::Changelog; use crate::revlog::changelog::Changelog;
use crate::revlog::manifest::{Manifest, ManifestEntry}; use crate::revlog::manifest::{Manifest, ManifestEntry};
use crate::revlog::node::Node; use crate::revlog::node::Node;
use crate::revlog::revlog::RevlogError; use crate::revlog::revlog::RevlogError;
use crate::utils::hg_path::HgPath; use crate::utils::hg_path::HgPath;
use crate::DirstateError;
use rayon::prelude::*; use rayon::prelude::*;
/// List files under Mercurial control in the working directory /// List files under Mercurial control in the working directory
/// by reading the dirstate /// by reading the dirstate
pub struct Dirstate { pub struct Dirstate {
/// The `dirstate` content. /// The `dirstate` content.
content: Vec<u8>, content: Vec<u8>,
dirstate_v2: bool,
} }
impl Dirstate { impl Dirstate {
pub fn new(repo: &Repo) -> Result<Self, HgError> { pub fn new(repo: &Repo) -> Result<Self, HgError> {
let content = repo.hg_vfs().read("dirstate")?; Ok(Self {
Ok(Self { content }) content: repo.hg_vfs().read("dirstate")?,
dirstate_v2: repo.has_dirstate_v2(),
})
} }
pub fn tracked_files(&self) -> Result<Vec<&HgPath>, HgError> { pub fn tracked_files(&self) -> Result<Vec<&HgPath>, DirstateError> {
let mut files = Vec::new(); let mut files = Vec::new();
if !self.content.is_empty() {
if self.dirstate_v2 {
for_each_tracked_path(&self.content, |path| files.push(path))?
} else {
let _parents = parse_dirstate_entries( let _parents = parse_dirstate_entries(
&self.content, &self.content,
|path, entry, _copy_source| { |path, entry, _copy_source| {
if entry.state.is_tracked() { if entry.state.is_tracked() {
files.push(path) files.push(path)
} }
Ok(()) Ok(())
}, },
)?; )?;
}
}
files.par_sort_unstable(); files.par_sort_unstable();
Ok(files) Ok(files)
} }
} }
/// List files under Mercurial control at a given revision. /// List files under Mercurial control at a given revision.
pub fn list_rev_tracked_files( pub fn list_rev_tracked_files(
repo: &Repo, repo: &Repo,

rust/hg-core/src/repo.rs

/// For accessing the working copy /// For accessing the working copy
pub fn working_directory_vfs(&self) -> Vfs<'_> { pub fn working_directory_vfs(&self) -> Vfs<'_> {
Vfs { Vfs {
base: &self.working_directory, base: &self.working_directory,
} }
} }
pub fn has_dirstate_v2(&self) -> bool {
self.requirements
.contains(requirements::DIRSTATE_V2_REQUIREMENT)
}
pub fn dirstate_parents( pub fn dirstate_parents(
&self, &self,
) -> Result<crate::dirstate::DirstateParents, HgError> { ) -> Result<crate::dirstate::DirstateParents, HgError> {
let dirstate = self.hg_vfs().mmap_open("dirstate")?; let dirstate = self.hg_vfs().mmap_open("dirstate")?;
let parents = if dirstate.is_empty() {
crate::dirstate::parsers::parse_dirstate_parents(&dirstate)?; return Ok(crate::dirstate::DirstateParents::NULL);
}
let parents = if self.has_dirstate_v2() {
crate::dirstate_tree::on_disk::parse_dirstate_parents(&dirstate)?
} else {
crate::dirstate::parsers::parse_dirstate_parents(&dirstate)?
};
Ok(parents.clone()) Ok(parents.clone())
} }
} }
impl Vfs<'_> { impl Vfs<'_> {
pub fn join(&self, relative_path: impl AsRef<Path>) -> PathBuf { pub fn join(&self, relative_path: impl AsRef<Path>) -> PathBuf {
self.base.join(relative_path) self.base.join(relative_path)
} }

rust/hg-core/src/requirements.rs

/// rhg supports repository with or without these /// rhg supports repository with or without these
const SUPPORTED: &[&str] = &[ const SUPPORTED: &[&str] = &[
"generaldelta", "generaldelta",
SHARED_REQUIREMENT, SHARED_REQUIREMENT,
SHARESAFE_REQUIREMENT, SHARESAFE_REQUIREMENT,
SPARSEREVLOG_REQUIREMENT, SPARSEREVLOG_REQUIREMENT,
RELATIVE_SHARED_REQUIREMENT, RELATIVE_SHARED_REQUIREMENT,
REVLOG_COMPRESSION_ZSTD, REVLOG_COMPRESSION_ZSTD,
DIRSTATE_V2_REQUIREMENT,
// As of this writing everything rhg does is read-only. // As of this writing everything rhg does is read-only.
// When it starts writing to the repository, it’ll need to either keep the // When it starts writing to the repository, it’ll need to either keep the
// persistent nodemap up to date or remove this entry: // persistent nodemap up to date or remove this entry:
NODEMAP_REQUIREMENT, NODEMAP_REQUIREMENT,
]; ];
// Copied from mercurial/requirements.py: // Copied from mercurial/requirements.py:
pub(crate) const DIRSTATE_V2_REQUIREMENT: &str = "exp-dirstate-v2";
/// When narrowing is finalized and no longer subject to format changes, /// When narrowing is finalized and no longer subject to format changes,
/// we should move this to just "narrow" or similar. /// we should move this to just "narrow" or similar.
#[allow(unused)] #[allow(unused)]
pub(crate) const NARROW_REQUIREMENT: &str = "narrowhg-experimental"; pub(crate) const NARROW_REQUIREMENT: &str = "narrowhg-experimental";
/// Enables sparse working directory usage /// Enables sparse working directory usage
#[allow(unused)] #[allow(unused)]
pub(crate) const SPARSE_REQUIREMENT: &str = "exp-sparse"; pub(crate) const SPARSE_REQUIREMENT: &str = "exp-sparse";

rust/rhg/src/commands/status.rs

// status.rs // status.rs
// //
// Copyright 2020, Georges Racinet <georges.racinets@octobus.net> // Copyright 2020, Georges Racinet <georges.racinets@octobus.net>
// //
// This software may be used and distributed according to the terms of the // This software may be used and distributed according to the terms of the
// GNU General Public License version 2 or any later version. // GNU General Public License version 2 or any later version.
use crate::error::CommandError; use crate::error::CommandError;
use crate::ui::Ui; use crate::ui::Ui;
use clap::{Arg, SubCommand}; use clap::{Arg, SubCommand};
use hg; use hg;
use hg::dirstate_tree::dirstate_map::DirstateMap;
use hg::errors::HgResultExt; use hg::errors::HgResultExt;
use hg::errors::IoResultExt; use hg::errors::IoResultExt;
use hg::matchers::AlwaysMatcher; use hg::matchers::AlwaysMatcher;
use hg::operations::cat; use hg::operations::cat;
use hg::repo::Repo; use hg::repo::Repo;
use hg::revlog::node::Node; use hg::revlog::node::Node;
use hg::utils::hg_path::{hg_path_to_os_string, HgPath}; use hg::utils::hg_path::{hg_path_to_os_string, HgPath};
use hg::{DirstateMap, StatusError}; use hg::StatusError;
use hg::{HgPathCow, StatusOptions}; use hg::{HgPathCow, StatusOptions};
use log::{info, warn}; use log::{info, warn};
use std::convert::TryInto; use std::convert::TryInto;
use std::fs; use std::fs;
use std::io::BufReader; use std::io::BufReader;
use std::io::Read; use std::io::Read;
pub const HELP_TEXT: &str = " pub const HELP_TEXT: &str = "
if requested.is_empty() { if requested.is_empty() {
DEFAULT_DISPLAY_STATES DEFAULT_DISPLAY_STATES
} else { } else {
requested requested
} }
}; };
let repo = invocation.repo?; let repo = invocation.repo?;
let mut dmap = DirstateMap::new();
let dirstate_data = let dirstate_data =
repo.hg_vfs().mmap_open("dirstate").io_not_found_as_none()?; repo.hg_vfs().mmap_open("dirstate").io_not_found_as_none()?;
let dirstate_data = match &dirstate_data { let dirstate_data = match &dirstate_data {
Some(mmap) => &**mmap, Some(mmap) => &**mmap,
None => b"", None => b"",
}; };
let parents = dmap.read(dirstate_data)?; let (mut dmap, parents) = if repo.has_dirstate_v2() {
DirstateMap::new_v2(dirstate_data)?
} else {
DirstateMap::new_v1(dirstate_data)?
};
let options = StatusOptions { let options = StatusOptions {
// TODO should be provided by the dirstate parsing and // TODO should be provided by the dirstate parsing and
// hence be stored on dmap. Using a value that assumes we aren't // hence be stored on dmap. Using a value that assumes we aren't
// below the time resolution granularity of the FS and the // below the time resolution granularity of the FS and the
// dirstate. // dirstate.
last_normal_time: 0, last_normal_time: 0,
// we're currently supporting file systems with exec flags only // we're currently supporting file systems with exec flags only
// anyway // anyway
check_exec: true, check_exec: true,
list_clean: display_states.clean, list_clean: display_states.clean,
list_unknown: display_states.unknown, list_unknown: display_states.unknown,
list_ignored: display_states.ignored, list_ignored: display_states.ignored,
collect_traversed_dirs: false, collect_traversed_dirs: false,
}; };
let ignore_file = repo.working_directory_vfs().join(".hgignore"); // TODO hardcoded let ignore_file = repo.working_directory_vfs().join(".hgignore"); // TODO hardcoded
let (mut ds_status, pattern_warnings) = hg::status( let (mut ds_status, pattern_warnings) = hg::dirstate_tree::status::status(
&dmap, &mut dmap,
&AlwaysMatcher, &AlwaysMatcher,
repo.working_directory_path().to_owned(), repo.working_directory_path().to_owned(),
vec![ignore_file], vec![ignore_file],
options, options,
)?; )?;
if !pattern_warnings.is_empty() { if !pattern_warnings.is_empty() {
warn!("Pattern warnings: {:?}", &pattern_warnings); warn!("Pattern warnings: {:?}", &pattern_warnings);
} }