This is an archive of the discontinued Mercurial Phabricator instance.

Differential D11881

rhg: centralize index header parsing
ClosedPublic

Authored by aalekseyev on Dec 7 2021, 1:58 PM.

Details

Reviewers
martinvonz
Alphare
Group Reviewers
hg-reviewers
Commits
rHG1fb3615dfce2: rhg: centralize index header parsing
Summary

Centralize index header parsing, parse the generaldelta flag,
and leave breadcrumbs to relate the code to python.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

aalekseyev created this revision.Dec 7 2021, 1:58 PM
aalekseyev updated this revision to Diff 31366.Dec 7 2021, 2:04 PM
martinvonz requested changes to this revision.Dec 7 2021, 3:12 PM
martinvonz added a subscriber: martinvonz.
martinvonz added inline comments.
rust/hg-core/src/revlog/index.rs
410–411
error[E0425]: cannot find function `is_inline` in this scope
   --> hg-core/src/revlog/index.rs:390:20
    |
390 |         assert_eq!(is_inline(&bytes), false)
    |                    ^^^^^^^^^ not found in this scope

error[E0425]: cannot find function `is_inline` in this scope
   --> hg-core/src/revlog/index.rs:401:20
    |
401 |         assert_eq!(is_inline(&bytes), true)
    |                    ^^^^^^^^^ not found in this scope

error[E0425]: cannot find function `is_inline` in this scope
   --> hg-core/src/revlog/index.rs:412:20
    |
412 |         assert_eq!(is_inline(&bytes), true)
    |                    ^^^^^^^^^ not found in this scope

error[E0425]: cannot find function `get_version` in this scope
   --> hg-core/src/revlog/revlog.rs:414:20
    |
414 |         assert_eq!(get_version(&bytes).map_err(|_err| ()), Ok(1))
    |                    ^^^^^^^^^^^ not found in this scope
This revision now requires changes to proceed.Dec 7 2021, 3:12 PM
aalekseyev updated this revision to Diff 31392.Dec 8 2021, 6:40 AM
aalekseyev marked an inline comment as done.Dec 8 2021, 6:41 AM
aalekseyev added inline comments.
rust/hg-core/src/revlog/index.rs
410–411

Oh, missed those. Thanks!
Pushed a fix now.

Alphare accepted this revision.Dec 10 2021, 5:56 AM
aalekseyev marked an inline comment as done.Dec 10 2021, 6:10 AM
aalekseyev updated this revision to Diff 31404.
Alphare accepted this revision.Dec 13 2021, 6:35 AM
aalekseyev added a commit: rHG1fb3615dfce2: rhg: centralize index header parsing.Dec 13 2021, 8:51 AM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG1fb3615dfce2: rhg: centralize index header parsing (authored by aalekseyev). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 31452

rust/hg-core/src/revlog/index.rs

use std::convert::TryInto; use std::convert::TryInto;
use std::ops::Deref; use std::ops::Deref;
use byteorder::{BigEndian, ByteOrder}; use byteorder::{BigEndian, ByteOrder};
use crate::errors::HgError; use crate::errors::HgError;
use crate::revlog::node::Node; use crate::revlog::node::Node;
use crate::revlog::{Revision, NULL_REVISION}; use crate::revlog::{Revision, NULL_REVISION};
pub const INDEX_ENTRY_SIZE: usize = 64; pub const INDEX_ENTRY_SIZE: usize = 64;
pub struct IndexHeader {
header_bytes: [u8; 4],
}
#[derive(Copy, Clone)]
pub struct IndexHeaderFlags {
flags: u16,
}
/// Corresponds to the high bits of `_format_flags` in python
impl IndexHeaderFlags {
/// Corresponds to FLAG_INLINE_DATA in python
pub fn is_inline(self) -> bool {
return self.flags & 1 != 0;
}
/// Corresponds to FLAG_GENERALDELTA in python
pub fn uses_generaldelta(self) -> bool {
return self.flags & 2 != 0;
}
}
/// Corresponds to the INDEX_HEADER structure,
/// which is parsed as a `header` variable in `_loadindex` in `revlog.py`
impl IndexHeader {
fn format_flags(&self) -> IndexHeaderFlags {
// No "unknown flags" check here, unlike in python. Maybe there should
// be.
return IndexHeaderFlags {
flags: BigEndian::read_u16(&self.header_bytes[0..2]),
};
}
/// The only revlog version currently supported by rhg.
const REVLOGV1: u16 = 1;
/// Corresponds to `_format_version` in Python.
fn format_version(&self) -> u16 {
return BigEndian::read_u16(&self.header_bytes[2..4]);
}
const EMPTY_INDEX_HEADER: IndexHeader = IndexHeader {
// We treat an empty file as a valid index with no entries.
// Here we make an arbitrary choice of what we assume the format of the
// index to be (V1, using generaldelta).
// This doesn't matter too much, since we're only doing read-only
// access. but the value corresponds to the `new_header` variable in
// `revlog.py`, `_loadindex`
header_bytes: [0, 3, 0, 1],
};
fn parse(index_bytes: &[u8]) -> Result<IndexHeader, HgError> {
if index_bytes.len() == 0 {
return Ok(IndexHeader::EMPTY_INDEX_HEADER);
}
if index_bytes.len() < 4 {
return Err(HgError::corrupted(
"corrupted revlog: can't read the index format header",
));
}
return Ok(IndexHeader {
header_bytes: {
let bytes: [u8; 4] =
index_bytes[0..4].try_into().expect("impossible");
bytes
},
});
}
}
/// A Revlog index /// A Revlog index
pub struct Index { pub struct Index {
bytes: Box<dyn Deref<Target = [u8]> + Send>, bytes: Box<dyn Deref<Target = [u8]> + Send>,
/// Offsets of starts of index blocks. /// Offsets of starts of index blocks.
/// Only needed when the index is interleaved with data. /// Only needed when the index is interleaved with data.
offsets: Option<Vec<usize>>, offsets: Option<Vec<usize>>,
} }
impl Index { impl Index {
/// Create an index from bytes. /// Create an index from bytes.
/// Calculate the start of each entry when is_inline is true. /// Calculate the start of each entry when is_inline is true.
pub fn new( pub fn new(
bytes: Box<dyn Deref<Target = [u8]> + Send>, bytes: Box<dyn Deref<Target = [u8]> + Send>,
) -> Result<Self, HgError> { ) -> Result<Self, HgError> {
if is_inline(&bytes) { let header = IndexHeader::parse(bytes.as_ref())?;
if header.format_version() != IndexHeader::REVLOGV1 {
// A proper new version should have had a repo/store
// requirement.
return Err(HgError::corrupted("unsupported revlog version"));
}
if header.format_flags().is_inline() {
let mut offset: usize = 0; let mut offset: usize = 0;
let mut offsets = Vec::new(); let mut offsets = Vec::new();
while offset + INDEX_ENTRY_SIZE <= bytes.len() { while offset + INDEX_ENTRY_SIZE <= bytes.len() {
offsets.push(offset); offsets.push(offset);
let end = offset + INDEX_ENTRY_SIZE; let end = offset + INDEX_ENTRY_SIZE;
let entry = IndexEntry { let entry = IndexEntry {
bytes: &bytes[offset..end], bytes: &bytes[offset..end],
/// ///
/// Currently, SHA-1 is used and only the first 20 bytes of this field /// Currently, SHA-1 is used and only the first 20 bytes of this field
/// are used. /// are used.
pub fn hash(&self) -> &'a Node { pub fn hash(&self) -> &'a Node {
(&self.bytes[32..52]).try_into().unwrap() (&self.bytes[32..52]).try_into().unwrap()
} }
} }
/// Value of the inline flag.
pub fn is_inline(index_bytes: &[u8]) -> bool {
if index_bytes.len() < 4 {
return true;
}
match &index_bytes[0..=1] {
[0, 0] | [0, 2] => false,
_ => true,
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
#[cfg(test)] #[cfg(test)]
#[derive(Debug, Copy, Clone)] #[derive(Debug, Copy, Clone)]
pub struct IndexEntryBuilder { pub struct IndexEntryBuilder {
is_first: bool, is_first: bool,
bytes.extend(&[0u8; 2]); // Revision flags. bytes.extend(&[0u8; 2]); // Revision flags.
bytes.extend(&(self.compressed_len as u32).to_be_bytes()); bytes.extend(&(self.compressed_len as u32).to_be_bytes());
bytes.extend(&(self.uncompressed_len as u32).to_be_bytes()); bytes.extend(&(self.uncompressed_len as u32).to_be_bytes());
bytes.extend(&self.base_revision.to_be_bytes()); bytes.extend(&self.base_revision.to_be_bytes());
bytes bytes
} }
} }
pub fn is_inline(index_bytes: &[u8]) -> bool {
IndexHeader::parse(index_bytes)
.expect("too short")
.format_flags()
.is_inline()
}
pub fn uses_generaldelta(index_bytes: &[u8]) -> bool {
IndexHeader::parse(index_bytes)
.expect("too short")
.format_flags()
.uses_generaldelta()
}
pub fn get_version(index_bytes: &[u8]) -> u16 {
IndexHeader::parse(index_bytes)
.expect("too short")
.format_version()
}
#[test] #[test]
fn is_not_inline_when_no_inline_flag_test() { fn flags_when_no_inline_flag_test() {
let bytes = IndexEntryBuilder::new() let bytes = IndexEntryBuilder::new()
.is_first(true) .is_first(true)
.with_general_delta(false) .with_general_delta(false)
.with_inline(false) .with_inline(false)
.build(); .build();
assert_eq!(is_inline(&bytes), false) assert_eq!(is_inline(&bytes), false);
assert_eq!(uses_generaldelta(&bytes), false);
martinvonzUnsubmitted
Done
error[E0425]: cannot find function `is_inline` in this scope
   --> hg-core/src/revlog/index.rs:390:20
    |
390 |         assert_eq!(is_inline(&bytes), false)
    |                    ^^^^^^^^^ not found in this scope

error[E0425]: cannot find function `is_inline` in this scope
   --> hg-core/src/revlog/index.rs:401:20
    |
401 |         assert_eq!(is_inline(&bytes), true)
    |                    ^^^^^^^^^ not found in this scope

error[E0425]: cannot find function `is_inline` in this scope
   --> hg-core/src/revlog/index.rs:412:20
    |
412 |         assert_eq!(is_inline(&bytes), true)
    |                    ^^^^^^^^^ not found in this scope

error[E0425]: cannot find function `get_version` in this scope
   --> hg-core/src/revlog/revlog.rs:414:20
    |
414 |         assert_eq!(get_version(&bytes).map_err(|_err| ()), Ok(1))
    |                    ^^^^^^^^^^^ not found in this scope
martinvonz: ``` error[E0425]: cannot find function `is_inline` in this scope --> hg…
aalekseyevAuthorUnsubmitted
Done

Oh, missed those. Thanks!
Pushed a fix now.

aalekseyev: Oh, missed those. Thanks! Pushed a fix now.
} }
#[test] #[test]
fn is_inline_when_inline_flag_test() { fn flags_when_inline_flag_test() {
let bytes = IndexEntryBuilder::new() let bytes = IndexEntryBuilder::new()
.is_first(true) .is_first(true)
.with_general_delta(false) .with_general_delta(false)
.with_inline(true) .with_inline(true)
.build(); .build();
assert_eq!(is_inline(&bytes), true) assert_eq!(is_inline(&bytes), true);
assert_eq!(uses_generaldelta(&bytes), false);
} }
#[test] #[test]
fn is_inline_when_inline_and_generaldelta_flags_test() { fn flags_when_inline_and_generaldelta_flags_test() {
let bytes = IndexEntryBuilder::new() let bytes = IndexEntryBuilder::new()
.is_first(true) .is_first(true)
.with_general_delta(true) .with_general_delta(true)
.with_inline(true) .with_inline(true)
.build(); .build();
assert_eq!(is_inline(&bytes), true) assert_eq!(is_inline(&bytes), true);
assert_eq!(uses_generaldelta(&bytes), true);
} }
#[test] #[test]
fn test_offset() { fn test_offset() {
let bytes = IndexEntryBuilder::new().with_offset(1).build(); let bytes = IndexEntryBuilder::new().with_offset(1).build();
let entry = IndexEntry { let entry = IndexEntry {
bytes: &bytes, bytes: &bytes,
offset_override: None, offset_override: None,
let bytes = IndexEntryBuilder::new().with_base_revision(1).build(); let bytes = IndexEntryBuilder::new().with_base_revision(1).build();
let entry = IndexEntry { let entry = IndexEntry {
bytes: &bytes, bytes: &bytes,
offset_override: None, offset_override: None,
}; };
assert_eq!(entry.base_revision(), 1) assert_eq!(entry.base_revision(), 1)
} }
#[test]
fn version_test() {
let bytes = IndexEntryBuilder::new()
.is_first(true)
.with_version(1)
.build();
assert_eq!(get_version(&bytes), 1)
}
} }
#[cfg(test)] #[cfg(test)]
pub use tests::IndexEntryBuilder; pub use tests::IndexEntryBuilder;

rust/hg-core/src/revlog/revlog.rs

use std::borrow::Cow; use std::borrow::Cow;
use std::io::Read; use std::io::Read;
use std::ops::Deref; use std::ops::Deref;
use std::path::Path; use std::path::Path;
use byteorder::{BigEndian, ByteOrder};
use flate2::read::ZlibDecoder; use flate2::read::ZlibDecoder;
use micro_timer::timed; use micro_timer::timed;
use sha1::{Digest, Sha1}; use sha1::{Digest, Sha1};
use zstd; use zstd;
use super::index::Index; use super::index::Index;
use super::node::{NodePrefix, NODE_BYTES_LENGTH, NULL_NODE}; use super::node::{NodePrefix, NODE_BYTES_LENGTH, NULL_NODE};
use super::nodemap; use super::nodemap;
index_path: impl AsRef<Path>, index_path: impl AsRef<Path>,
data_path: Option<&Path>, data_path: Option<&Path>,
) -> Result<Self, HgError> { ) -> Result<Self, HgError> {
let index_path = index_path.as_ref(); let index_path = index_path.as_ref();
let index = { let index = {
match repo.store_vfs().mmap_open_opt(&index_path)? { match repo.store_vfs().mmap_open_opt(&index_path)? {
None => Index::new(Box::new(vec![])), None => Index::new(Box::new(vec![])),
Some(index_mmap) => { Some(index_mmap) => {
let version = get_version(&index_mmap)?;
if version != 1 {
// A proper new version should have had a repo/store
// requirement.
return Err(HgError::corrupted("corrupted revlog"));
}
let index = Index::new(Box::new(index_mmap))?; let index = Index::new(Box::new(index_mmap))?;
Ok(index) Ok(index)
} }
} }
}?; }?;
let default_data_path = index_path.with_extension("d"); let default_data_path = index_path.with_extension("d");
/// Tell if the entry is a snapshot or a delta /// Tell if the entry is a snapshot or a delta
/// (influences on decompression). /// (influences on decompression).
fn is_delta(&self) -> bool { fn is_delta(&self) -> bool {
self.base_rev.is_some() self.base_rev.is_some()
} }
} }
/// Format version of the revlog.
pub fn get_version(index_bytes: &[u8]) -> Result<u16, HgError> {
if index_bytes.len() == 0 {
return Ok(1);
};
if index_bytes.len() < 4 {
return Err(HgError::corrupted(
"corrupted revlog: can't read the index format header",
));
};
Ok(BigEndian::read_u16(&index_bytes[2..=3]))
}
/// Calculate the hash of a revision given its data and its parents. /// Calculate the hash of a revision given its data and its parents.
fn hash( fn hash(
data: &[u8], data: &[u8],
p1_hash: &[u8], p1_hash: &[u8],
p2_hash: &[u8], p2_hash: &[u8],
) -> [u8; NODE_BYTES_LENGTH] { ) -> [u8; NODE_BYTES_LENGTH] {
let mut hasher = Sha1::new(); let mut hasher = Sha1::new();
let (a, b) = (p1_hash, p2_hash); let (a, b) = (p1_hash, p2_hash);
if a > b { if a > b {
hasher.update(b); hasher.update(b);
hasher.update(a); hasher.update(a);
} else { } else {
hasher.update(a); hasher.update(a);
hasher.update(b); hasher.update(b);
} }
hasher.update(data); hasher.update(data);
*hasher.finalize().as_ref() *hasher.finalize().as_ref()
} }
#[cfg(test)]
mod tests {
use super::*;
use super::super::index::IndexEntryBuilder;
#[test]
fn version_test() {
let bytes = IndexEntryBuilder::new()
.is_first(true)
.with_version(1)
.build();
assert_eq!(get_version(&bytes).map_err(|_err| ()), Ok(1))
}
}