( )⚙ D11961 rhg: Expose FilelogEntry that wraps RevlogEntry

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D11961

rhg: Expose FilelogEntry that wraps RevlogEntry
ClosedPublic

Authored by SimonSapin on Jan 6 2022, 1:56 PM.

Details

Reviewers
Alphare
Group Reviewers
hg-reviewers
Commits
rHG35c47015b9b7: rhg: Expose FilelogEntry that wraps RevlogEntry
Summary

This can be later extended to access metadata such as uncompressed_len without
necessarily resolving deltas.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

SimonSapin created this revision.Jan 6 2022, 1:56 PM
Alphare accepted this revision.Jan 7 2022, 4:32 AM
This revision is now accepted and ready to land.Jan 7 2022, 4:32 AM
SimonSapin added a commit: rHG35c47015b9b7: rhg: Expose FilelogEntry that wraps RevlogEntry.Jan 7 2022, 4:52 AM
Closed by commit rHG35c47015b9b7: rhg: Expose FilelogEntry that wraps RevlogEntry (authored by SimonSapin). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 31613

rust/hg-core/src/revlog/filelog.rs

use crate::errors::HgError; use crate::errors::HgError;
use crate::repo::Repo; use crate::repo::Repo;
use crate::revlog::path_encode::path_encode; use crate::revlog::path_encode::path_encode;
use crate::revlog::revlog::RevlogEntry;
use crate::revlog::revlog::{Revlog, RevlogError}; use crate::revlog::revlog::{Revlog, RevlogError};
use crate::revlog::NodePrefix; use crate::revlog::NodePrefix;
use crate::revlog::Revision; use crate::revlog::Revision;
use crate::utils::files::get_path_from_bytes; use crate::utils::files::get_path_from_bytes;
use crate::utils::hg_path::HgPath; use crate::utils::hg_path::HgPath;
use crate::utils::SliceExt; use crate::utils::SliceExt;
use std::path::PathBuf; use std::path::PathBuf;
/// A specialized `Revlog` to work with file data logs. /// A specialized `Revlog` to work with file data logs.
pub struct Filelog { pub struct Filelog {
/// The generic `revlog` format. /// The generic `revlog` format.
revlog: Revlog, revlog: Revlog,
} }
impl Filelog { impl Filelog {
pub fn open(repo: &Repo, file_path: &HgPath) -> Result<Self, HgError> { pub fn open(repo: &Repo, file_path: &HgPath) -> Result<Self, HgError> {
let index_path = store_path(file_path, b".i"); let index_path = store_path(file_path, b".i");
let data_path = store_path(file_path, b".d"); let data_path = store_path(file_path, b".d");
let revlog = Revlog::open(repo, index_path, Some(&data_path))?; let revlog = Revlog::open(repo, index_path, Some(&data_path))?;
Ok(Self { revlog }) Ok(Self { revlog })
} }
/// The given node ID is that of the file as found in a manifest, not of a /// The given node ID is that of the file as found in a filelog, not of a
/// changeset. /// changeset.
pub fn data_for_node( pub fn data_for_node(
&self, &self,
file_node: impl Into<NodePrefix>, file_node: impl Into<NodePrefix>,
) -> Result<FilelogRevisionData, RevlogError> { ) -> Result<FilelogRevisionData, RevlogError> {
let file_rev = self.revlog.rev_from_node(file_node.into())?; let file_rev = self.revlog.rev_from_node(file_node.into())?;
self.data_for_rev(file_rev) self.data_for_rev(file_rev)
} }
/// The given revision is that of the file as found in a manifest, not of a /// The given revision is that of the file as found in a filelog, not of a
/// changeset. /// changeset.
pub fn data_for_rev( pub fn data_for_rev(
&self, &self,
file_rev: Revision, file_rev: Revision,
) -> Result<FilelogRevisionData, RevlogError> { ) -> Result<FilelogRevisionData, RevlogError> {
let data: Vec<u8> = self.revlog.get_rev_data(file_rev)?.into_owned(); let data: Vec<u8> = self.revlog.get_rev_data(file_rev)?.into_owned();
Ok(FilelogRevisionData(data.into())) Ok(FilelogRevisionData(data.into()))
} }
/// The given node ID is that of the file as found in a filelog, not of a
/// changeset.
pub fn entry_for_node(
&self,
file_node: impl Into<NodePrefix>,
) -> Result<FilelogEntry, RevlogError> {
let file_rev = self.revlog.rev_from_node(file_node.into())?;
self.entry_for_rev(file_rev)
}
/// The given revision is that of the file as found in a filelog, not of a
/// changeset.
pub fn entry_for_rev(
&self,
file_rev: Revision,
) -> Result<FilelogEntry, RevlogError> {
Ok(FilelogEntry(self.revlog.get_entry(file_rev)?))
}
} }
fn store_path(hg_path: &HgPath, suffix: &[u8]) -> PathBuf { fn store_path(hg_path: &HgPath, suffix: &[u8]) -> PathBuf {
let encoded_bytes = let encoded_bytes =
path_encode(&[b"data/", hg_path.as_bytes(), suffix].concat()); path_encode(&[b"data/", hg_path.as_bytes(), suffix].concat());
get_path_from_bytes(&encoded_bytes).into() get_path_from_bytes(&encoded_bytes).into()
} }
pub struct FilelogEntry<'a>(RevlogEntry<'a>);
impl FilelogEntry<'_> {
pub fn data(&self) -> Result<FilelogRevisionData, HgError> {
Ok(FilelogRevisionData(self.0.data()?.into_owned()))
}
}
/// The data for one revision in a filelog, uncompressed and delta-resolved. /// The data for one revision in a filelog, uncompressed and delta-resolved.
pub struct FilelogRevisionData(Vec<u8>); pub struct FilelogRevisionData(Vec<u8>);
impl FilelogRevisionData { impl FilelogRevisionData {
/// Split into metadata and data /// Split into metadata and data
pub fn split(&self) -> Result<(Option<&[u8]>, &[u8]), HgError> { pub fn split(&self) -> Result<(Option<&[u8]>, &[u8]), HgError> {
const DELIMITER: &[u8; 2] = &[b'\x01', b'\n']; const DELIMITER: &[u8; 2] = &[b'\x01', b'\n'];

rust/hg-core/src/revlog/revlog.rs

fn from(error: NodeMapError) -> Self { fn from(error: NodeMapError) -> Self {
match error { match error {
NodeMapError::MultipleResults => RevlogError::AmbiguousPrefix, NodeMapError::MultipleResults => RevlogError::AmbiguousPrefix,
NodeMapError::RevisionNotInIndex(_) => RevlogError::corrupted(), NodeMapError::RevisionNotInIndex(_) => RevlogError::corrupted(),
} }
} }
} }
fn corrupted() -> HgError {
HgError::corrupted("corrupted revlog")
}
impl RevlogError { impl RevlogError {
fn corrupted() -> Self { fn corrupted() -> Self {
RevlogError::Other(HgError::corrupted("corrupted revlog")) RevlogError::Other(corrupted())
} }
} }
/// Read only implementation of revlog. /// Read only implementation of revlog.
pub struct Revlog { pub struct Revlog {
/// When index and data are not interleaved: bytes of the revlog index. /// When index and data are not interleaved: bytes of the revlog index.
/// When index and data are interleaved: bytes of the revlog index and /// When index and data are interleaved: bytes of the revlog index and
/// data. /// data.
#[timed] #[timed]
pub fn get_rev_data( pub fn get_rev_data(
&self, &self,
rev: Revision, rev: Revision,
) -> Result<Cow<[u8]>, RevlogError> { ) -> Result<Cow<[u8]>, RevlogError> {
if rev == NULL_REVISION { if rev == NULL_REVISION {
return Ok(Cow::Borrowed(&[])); return Ok(Cow::Borrowed(&[]));
}; };
self.get_entry(rev)?.data() Ok(self.get_entry(rev)?.data()?)
} }
/// Check the hash of some given data against the recorded hash. /// Check the hash of some given data against the recorded hash.
pub fn check_hash( pub fn check_hash(
&self, &self,
p1: Revision, p1: Revision,
p2: Revision, p2: Revision,
expected: &[u8], expected: &[u8],
} }
/// Build the full data of a revision out its snapshot /// Build the full data of a revision out its snapshot
/// and its deltas. /// and its deltas.
#[timed] #[timed]
fn build_data_from_deltas( fn build_data_from_deltas(
snapshot: RevlogEntry, snapshot: RevlogEntry,
deltas: &[RevlogEntry], deltas: &[RevlogEntry],
) -> Result<Vec<u8>, RevlogError> { ) -> Result<Vec<u8>, HgError> {
let snapshot = snapshot.data_chunk()?; let snapshot = snapshot.data_chunk()?;
let deltas = deltas let deltas = deltas
.iter() .iter()
.rev() .rev()
.map(RevlogEntry::data_chunk) .map(RevlogEntry::data_chunk)
.collect::<Result<Vec<Cow<'_, [u8]>>, RevlogError>>()?; .collect::<Result<Vec<_>, _>>()?;
let patches: Vec<_> = let patches: Vec<_> =
deltas.iter().map(|d| patch::PatchList::new(d)).collect(); deltas.iter().map(|d| patch::PatchList::new(d)).collect();
let patch = patch::fold_patch_lists(&patches); let patch = patch::fold_patch_lists(&patches);
Ok(patch.apply(&snapshot)) Ok(patch.apply(&snapshot))
} }
/// Return the revlog data. /// Return the revlog data.
fn data(&self) -> &[u8] { fn data(&self) -> &[u8] {
match self.data_bytes { match self.data_bytes {
Some(ref data_bytes) => &data_bytes, Some(ref data_bytes) => &data_bytes,
None => panic!( None => panic!(
"forgot to load the data or trying to access inline data" "forgot to load the data or trying to access inline data"
), ),
} }
} }
/// Get an entry of the revlog. /// Get an entry of the revlog.
fn get_entry(&self, rev: Revision) -> Result<RevlogEntry, RevlogError> { pub fn get_entry(
&self,
rev: Revision,
) -> Result<RevlogEntry, RevlogError> {
let index_entry = self let index_entry = self
.index .index
.get_entry(rev) .get_entry(rev)
.ok_or(RevlogError::InvalidRevision)?; .ok_or(RevlogError::InvalidRevision)?;
let start = index_entry.offset(); let start = index_entry.offset();
let end = start + index_entry.compressed_len(); let end = start + index_entry.compressed_len();
let data = if self.index.is_inline() { let data = if self.index.is_inline() {
self.index.data(start, end) self.index.data(start, end)
Ok(entry) Ok(entry)
} }
/// when resolving internal references within revlog, any errors /// when resolving internal references within revlog, any errors
/// should be reported as corruption, instead of e.g. "invalid revision" /// should be reported as corruption, instead of e.g. "invalid revision"
fn get_entry_internal( fn get_entry_internal(
&self, &self,
rev: Revision, rev: Revision,
) -> Result<RevlogEntry, RevlogError> { ) -> Result<RevlogEntry, HgError> {
return self.get_entry(rev).map_err(|_| RevlogError::corrupted()); return self.get_entry(rev).map_err(|_| corrupted());
} }
} }
/// The revlog entry's bytes and the necessary informations to extract /// The revlog entry's bytes and the necessary informations to extract
/// the entry's data. /// the entry's data.
#[derive(Clone)] #[derive(Clone)]
pub struct RevlogEntry<'a> { pub struct RevlogEntry<'a> {
revlog: &'a Revlog, revlog: &'a Revlog,
rev: Revision, rev: Revision,
bytes: &'a [u8], bytes: &'a [u8],
compressed_len: usize, compressed_len: usize,
uncompressed_len: usize, uncompressed_len: usize,
base_rev_or_base_of_delta_chain: Option<Revision>, base_rev_or_base_of_delta_chain: Option<Revision>,
} }
impl<'a> RevlogEntry<'a> { impl<'a> RevlogEntry<'a> {
pub fn revision(&self) -> Revision { pub fn revision(&self) -> Revision {
self.rev self.rev
} }
/// The data for this entry, after resolving deltas if any. /// The data for this entry, after resolving deltas if any.
pub fn data(&self) -> Result<Cow<'a, [u8]>, RevlogError> { pub fn data(&self) -> Result<Cow<'a, [u8]>, HgError> {
let mut entry = self.clone(); let mut entry = self.clone();
let mut delta_chain = vec![]; let mut delta_chain = vec![];
// The meaning of `base_rev_or_base_of_delta_chain` depends on // The meaning of `base_rev_or_base_of_delta_chain` depends on
// generaldelta. See the doc on `ENTRY_DELTA_BASE` in // generaldelta. See the doc on `ENTRY_DELTA_BASE` in
// `mercurial/revlogutils/constants.py` and the code in // `mercurial/revlogutils/constants.py` and the code in
// [_chaininfo] and in [index_deltachain]. // [_chaininfo] and in [index_deltachain].
let uses_generaldelta = self.revlog.index.uses_generaldelta(); let uses_generaldelta = self.revlog.index.uses_generaldelta();
while let Some(base_rev) = entry.base_rev_or_base_of_delta_chain { while let Some(base_rev) = entry.base_rev_or_base_of_delta_chain {
let base_rev = if uses_generaldelta { let base_rev = if uses_generaldelta {
base_rev base_rev
} else { } else {
entry.rev - 1 entry.rev - 1
}; };
delta_chain.push(entry); delta_chain.push(entry);
entry = self.revlog.get_entry_internal(base_rev)?; entry = self.revlog.get_entry_internal(base_rev)?;
} }
// TODO do not look twice in the index // TODO do not look twice in the index
let index_entry = self let index_entry = self
.revlog .revlog
.index .index
.get_entry(self.rev) .get_entry(self.rev)
.ok_or(RevlogError::InvalidRevision)?; .ok_or_else(corrupted)?;
let data = if delta_chain.is_empty() { let data = if delta_chain.is_empty() {
entry.data_chunk()? entry.data_chunk()?
} else { } else {
Revlog::build_data_from_deltas(entry, &delta_chain)?.into() Revlog::build_data_from_deltas(entry, &delta_chain)?.into()
}; };
if self.revlog.check_hash( if self.revlog.check_hash(
index_entry.p1(), index_entry.p1(),
index_entry.p2(), index_entry.p2(),
index_entry.hash().as_bytes(), index_entry.hash().as_bytes(),
&data, &data,
) { ) {
Ok(data) Ok(data)
} else { } else {
Err(RevlogError::corrupted()) Err(corrupted())
} }
} }
/// Extract the data contained in the entry. /// Extract the data contained in the entry.
/// This may be a delta. (See `is_delta`.) /// This may be a delta. (See `is_delta`.)
fn data_chunk(&self) -> Result<Cow<'a, [u8]>, RevlogError> { fn data_chunk(&self) -> Result<Cow<'a, [u8]>, HgError> {
if self.bytes.is_empty() { if self.bytes.is_empty() {
return Ok(Cow::Borrowed(&[])); return Ok(Cow::Borrowed(&[]));
} }
match self.bytes[0] { match self.bytes[0] {
// Revision data is the entirety of the entry, including this // Revision data is the entirety of the entry, including this
// header. // header.
b'\0' => Ok(Cow::Borrowed(self.bytes)), b'\0' => Ok(Cow::Borrowed(self.bytes)),
// Raw revision data follows. // Raw revision data follows.
b'u' => Ok(Cow::Borrowed(&self.bytes[1..])), b'u' => Ok(Cow::Borrowed(&self.bytes[1..])),
// zlib (RFC 1950) data. // zlib (RFC 1950) data.
b'x' => Ok(Cow::Owned(self.uncompressed_zlib_data()?)), b'x' => Ok(Cow::Owned(self.uncompressed_zlib_data()?)),
// zstd data. // zstd data.
b'\x28' => Ok(Cow::Owned(self.uncompressed_zstd_data()?)), b'\x28' => Ok(Cow::Owned(self.uncompressed_zstd_data()?)),
// A proper new format should have had a repo/store requirement. // A proper new format should have had a repo/store requirement.
_format_type => Err(RevlogError::corrupted()), _format_type => Err(corrupted()),
} }
} }
fn uncompressed_zlib_data(&self) -> Result<Vec<u8>, RevlogError> { fn uncompressed_zlib_data(&self) -> Result<Vec<u8>, HgError> {
let mut decoder = ZlibDecoder::new(self.bytes); let mut decoder = ZlibDecoder::new(self.bytes);
if self.is_delta() { if self.is_delta() {
let mut buf = Vec::with_capacity(self.compressed_len); let mut buf = Vec::with_capacity(self.compressed_len);
decoder decoder.read_to_end(&mut buf).map_err(|_| corrupted())?;
.read_to_end(&mut buf)
.map_err(|_| RevlogError::corrupted())?;
Ok(buf) Ok(buf)
} else { } else {
let mut buf = vec![0; self.uncompressed_len]; let mut buf = vec![0; self.uncompressed_len];
decoder decoder.read_exact(&mut buf).map_err(|_| corrupted())?;
.read_exact(&mut buf)
.map_err(|_| RevlogError::corrupted())?;
Ok(buf) Ok(buf)
} }
} }
fn uncompressed_zstd_data(&self) -> Result<Vec<u8>, RevlogError> { fn uncompressed_zstd_data(&self) -> Result<Vec<u8>, HgError> {
if self.is_delta() { if self.is_delta() {
let mut buf = Vec::with_capacity(self.compressed_len); let mut buf = Vec::with_capacity(self.compressed_len);
zstd::stream::copy_decode(self.bytes, &mut buf) zstd::stream::copy_decode(self.bytes, &mut buf)
.map_err(|_| RevlogError::corrupted())?; .map_err(|_| corrupted())?;
Ok(buf) Ok(buf)
} else { } else {
let mut buf = vec![0; self.uncompressed_len]; let mut buf = vec![0; self.uncompressed_len];
let len = zstd::block::decompress_to_buffer(self.bytes, &mut buf) let len = zstd::block::decompress_to_buffer(self.bytes, &mut buf)
.map_err(|_| RevlogError::corrupted())?; .map_err(|_| corrupted())?;
if len != self.uncompressed_len { if len != self.uncompressed_len {
Err(RevlogError::corrupted()) Err(corrupted())
} else { } else {
Ok(buf) Ok(buf)
} }
} }
} }
/// Tell if the entry is a snapshot or a delta /// Tell if the entry is a snapshot or a delta
/// (influences on decompression). /// (influences on decompression).

rust/rhg/src/commands/status.rs

let entry = manifest let entry = manifest
.find_by_path(hg_path)? .find_by_path(hg_path)?
.expect("ambgious file not in p1"); .expect("ambgious file not in p1");
if entry.flags != fs_flags { if entry.flags != fs_flags {
return Ok(true); return Ok(true);
} }
let filelog = repo.filelog(hg_path)?; let filelog = repo.filelog(hg_path)?;
let fs_len = fs_metadata.len(); let fs_len = fs_metadata.len();
let filelog_entry =
filelog.entry_for_node(entry.node_id()?).map_err(|_| {
HgError::corrupted("filelog missing node from manifest")
})?;
// TODO: check `fs_len` here like below, but based on // TODO: check `fs_len` here like below, but based on
// `RevlogEntry::uncompressed_len` without decompressing the full filelog // `RevlogEntry::uncompressed_len` without decompressing the full filelog
// contents where possible. This is only valid if the revlog data does not // contents where possible. This is only valid if the revlog data does not
// contain metadata. See how Python’s `revlog.rawsize` calls // contain metadata. See how Python’s `revlog.rawsize` calls
// `storageutil.filerevisioncopied`. // `storageutil.filerevisioncopied`.
// (Maybe also check for content-modifying flags? See `revlog.size`.) // (Maybe also check for content-modifying flags? See `revlog.size`.)
let filelog_entry = let filelog_data = filelog_entry.data()?;
filelog.data_for_node(entry.node_id()?).map_err(|_| { let contents_in_p1 = filelog_data.file_data()?;
HgError::corrupted("filelog missing node from manifest")
})?;
let contents_in_p1 = filelog_entry.file_data()?;
if contents_in_p1.len() as u64 != fs_len { if contents_in_p1.len() as u64 != fs_len {
// No need to read the file contents: // No need to read the file contents:
// it cannot be equal if it has a different length. // it cannot be equal if it has a different length.
return Ok(true); return Ok(true);
} }
let fs_contents = if is_symlink { let fs_contents = if is_symlink {
get_bytes_from_os_string(vfs.read_link(fs_path)?.into_os_string()) get_bytes_from_os_string(vfs.read_link(fs_path)?.into_os_string())
} else { } else {
vfs.read(fs_path)? vfs.read(fs_path)?
}; };
Ok(contents_in_p1 != &*fs_contents) Ok(contents_in_p1 != &*fs_contents)
} }