This is an archive of the discontinued Mercurial Phabricator instance.

Differential D9788

rust: use the bytes-cast crate to parse persistent nodemaps
ClosedPublic

Authored by SimonSapin on Jan 15 2021, 10:12 AM.

Details

Reviewers
Alphare
pulkit
Group Reviewers
hg-reviewers
Commits
rHG0800aa42bb4c: rust: use the bytes-cast crate to parse persistent nodemaps
rHG1eb72345be29: rust: use the bytes-cast crate to parse persistent nodemaps
Summary

This crate casts pointers to custom structs, with compile-time safety checks,
for easy and efficient binary data parsing.

See https://crates.io/crates/bytes-cast and
https://docs.rs/bytes-cast/0.1.0/bytes_cast/

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

SimonSapin created this revision.Jan 15 2021, 10:12 AM
Alphare added a subscriber: Alphare.Jan 15 2021, 10:43 AM
Alphare added inline comments.
rust/hg-core/src/revlog/nodemap.rs
225

This comment is now obsolete.

225

Could that be kept as the following and reused later?

rust
pub const BLOCK_SIZE: usize = size_of::<Block>;
388

If you're not going to use a constant (as per my comment above), use size_of as mem is already imported.

614

I'd use expect to explain better why this cannot happen.

SimonSapin updated this revision to Diff 25109.Jan 18 2021, 9:22 AM
Alphare accepted this revision.Jan 22 2021, 8:35 AM
pulkit accepted this revision.Jan 26 2021, 3:04 PM
This revision is now accepted and ready to land.Jan 26 2021, 3:04 PM

Revision Contents
Changeset List

Diff 24900

rust/Cargo.lock

] ]
[[package]] [[package]]
name = "byteorder" name = "byteorder"
version = "1.3.4" version = "1.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "bytes-cast"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bytes-cast-derive 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "bytes-cast-derive"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "cc" name = "cc"
version = "1.0.66" version = "1.0.66"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"jobserver 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", "jobserver 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]] [[package]]
version = "0.4.2" version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "hg-core" name = "hg-core"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
"bytes-cast 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.33.3 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.33.3 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-channel 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-channel 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
"flate2 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)", "flate2 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)",
"format-bytes 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "format-bytes 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"hex 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"im-rc 15.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "im-rc 15.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)",
"checksum adler 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" "checksum adler 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
"checksum aho-corasick 0.7.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" "checksum aho-corasick 0.7.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5"
"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
"checksum atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" "checksum atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
"checksum autocfg 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" "checksum autocfg 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
"checksum bitmaps 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" "checksum bitmaps 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2"
"checksum byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" "checksum byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de"
"checksum bytes-cast 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3196ba300c7bc9282a4331e878496cb3e9603a898a8f1446601317163e16ca52"
"checksum bytes-cast-derive 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cb936af9de38476664d6b58e529aff30d482e4ce1c5e150293d00730b0d81fdb"
"checksum cc 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" "checksum cc 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
"checksum cfg-if 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" "checksum cfg-if 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
"checksum clap 2.33.3 (registry+https://github.com/rust-lang/crates.io-index)" = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" "checksum clap 2.33.3 (registry+https://github.com/rust-lang/crates.io-index)" = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002"
"checksum const_fn 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" "checksum const_fn 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826"
"checksum cpython 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bfaf3847ab963e40c4f6dd8d6be279bdf74007ae2413786a0dcbb28c52139a95" "checksum cpython 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bfaf3847ab963e40c4f6dd8d6be279bdf74007ae2413786a0dcbb28c52139a95"
"checksum crc32fast 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" "checksum crc32fast 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
"checksum crossbeam-channel 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87" "checksum crossbeam-channel 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87"

rust/hg-core/Cargo.toml

[package] [package]
name = "hg-core" name = "hg-core"
version = "0.1.0" version = "0.1.0"
authors = ["Georges Racinet <gracinet@anybox.fr>"] authors = ["Georges Racinet <gracinet@anybox.fr>"]
description = "Mercurial pure Rust core library, with no assumption on Python bindings (FFI)" description = "Mercurial pure Rust core library, with no assumption on Python bindings (FFI)"
edition = "2018" edition = "2018"
[lib] [lib]
name = "hg" name = "hg"
[dependencies] [dependencies]
bytes-cast = "0.1"
byteorder = "1.3.4" byteorder = "1.3.4"
hex = "0.4.2" hex = "0.4.2"
im-rc = "15.0.*" im-rc = "15.0.*"
lazy_static = "1.4.0" lazy_static = "1.4.0"
memchr = "2.3.3" memchr = "2.3.3"
rand = "0.7.3" rand = "0.7.3"
rand_pcg = "0.2.1" rand_pcg = "0.2.1"
rand_distr = "0.2.2" rand_distr = "0.2.2"

rust/hg-core/src/revlog/nodemap.rs

//! Following existing implicit conventions, the "nodemap" terminology //! Following existing implicit conventions, the "nodemap" terminology
//! is used in a more abstract context. //! is used in a more abstract context.
use super::{ use super::{
node::NULL_NODE, Node, NodeError, NodePrefix, NodePrefixRef, Revision, node::NULL_NODE, Node, NodeError, NodePrefix, NodePrefixRef, Revision,
RevlogIndex, NULL_REVISION, RevlogIndex, NULL_REVISION,
}; };
use bytes_cast::{unaligned, BytesCast};
use mem::size_of;
use std::cmp::max; use std::cmp::max;
use std::fmt; use std::fmt;
use std::mem; use std::mem;
use std::ops::Deref; use std::ops::Deref;
use std::ops::Index; use std::ops::Index;
use std::slice;
#[derive(Debug, PartialEq)] #[derive(Debug, PartialEq)]
pub enum NodeMapError { pub enum NodeMapError {
MultipleResults, MultipleResults,
InvalidNodePrefix(NodeError), InvalidNodePrefix(NodeError),
/// A `Revision` stored in the nodemap could not be found in the index /// A `Revision` stored in the nodemap could not be found in the index
RevisionNotInIndex(Revision), RevisionNotInIndex(Revision),
} }
node: &Node, node: &Node,
rev: Revision, rev: Revision,
) -> Result<(), NodeMapError>; ) -> Result<(), NodeMapError>;
} }
/// Low level NodeTree [`Blocks`] elements /// Low level NodeTree [`Blocks`] elements
/// ///
/// These are exactly as for instance on persistent storage. /// These are exactly as for instance on persistent storage.
type RawElement = i32; type RawElement = unaligned::I32Be;
/// High level representation of values in NodeTree /// High level representation of values in NodeTree
/// [`Blocks`](struct.Block.html) /// [`Blocks`](struct.Block.html)
/// ///
/// This is the high level representation that most algorithms should /// This is the high level representation that most algorithms should
/// use. /// use.
#[derive(Clone, Debug, Eq, PartialEq)] #[derive(Clone, Debug, Eq, PartialEq)]
enum Element { enum Element {
Rev(Revision), Rev(Revision),
Block(usize), Block(usize),
None, None,
} }
impl From<RawElement> for Element { impl From<RawElement> for Element {
/// Conversion from low level representation, after endianness conversion. /// Conversion from low level representation, after endianness conversion.
/// ///
/// See [`Block`](struct.Block.html) for explanation about the encoding. /// See [`Block`](struct.Block.html) for explanation about the encoding.
fn from(raw: RawElement) -> Element { fn from(raw: RawElement) -> Element {
if raw >= 0 { let int = raw.get();
Element::Block(raw as usize) if int >= 0 {
} else if raw == -1 { Element::Block(int as usize)
} else if int == -1 {
Element::None Element::None
} else { } else {
Element::Rev(-raw - 2) Element::Rev(-int - 2)
} }
} }
} }
impl From<Element> for RawElement { impl From<Element> for RawElement {
fn from(element: Element) -> RawElement { fn from(element: Element) -> RawElement {
match element { RawElement::from(match element {
Element::None => 0, Element::None => 0,
Element::Block(i) => i as RawElement, Element::Block(i) => i as i32,
Element::Rev(rev) => -rev - 2, Element::Rev(rev) => -rev - 2,
} })
} }
} }
/// A logical block of the `NodeTree`, packed with a fixed size. /// A logical block of the `NodeTree`, packed with a fixed size.
/// ///
/// These are always used in container types implementing `Index<Block>`, /// These are always used in container types implementing `Index<Block>`,
/// such as `&Block` /// such as `&Block`
/// ///
/// A key difference with the C `nodetree` is that we need to be /// A key difference with the C `nodetree` is that we need to be
/// able to represent the [`Block`] at index 0, hence -1 is the empty marker /// able to represent the [`Block`] at index 0, hence -1 is the empty marker
/// rather than 0 and the `Revision` range upper limit of -2 instead of -1. /// rather than 0 and the `Revision` range upper limit of -2 instead of -1.
/// ///
/// Another related difference is that `NULL_REVISION` (-1) is not /// Another related difference is that `NULL_REVISION` (-1) is not
/// represented at all, because we want an immutable empty nodetree /// represented at all, because we want an immutable empty nodetree
/// to be valid. /// to be valid.
#[derive(Copy, Clone)] const ELEMENTS_PER_BLOCK: usize = 16; // number of different values in a nybble
pub struct Block([u8; BLOCK_SIZE]);
/// Not derivable for arrays of length >32 until const generics are stable #[derive(Copy, Clone, BytesCast, PartialEq)]
impl PartialEq for Block { #[repr(transparent)]
fn eq(&self, other: &Self) -> bool { pub struct Block([RawElement; ELEMENTS_PER_BLOCK]);
self.0[..] == other.0[..]
}
}
pub const BLOCK_SIZE: usize = 64;
AlphareUnsubmitted
Not Done

Could that be kept as the following and reused later?

rust
pub const BLOCK_SIZE: usize = size_of::<Block>;
Alphare: Could that be kept as the following and reused later? ```rust pub const BLOCK_SIZE: usize =…
impl Block { impl Block {
fn new() -> Self { fn new() -> Self {
// -1 in 2's complement to create an absent node // -1 in 2's complement to create an absent node
AlphareUnsubmitted
Not Done

This comment is now obsolete.

Alphare: This comment is now obsolete.
let byte: u8 = 255; let absent_node = RawElement::from(-1);
Block([byte; BLOCK_SIZE]) Block([absent_node; ELEMENTS_PER_BLOCK])
} }
fn get(&self, nybble: u8) -> Element { fn get(&self, nybble: u8) -> Element {
let index = nybble as usize * mem::size_of::<RawElement>(); self.0[nybble as usize].into()
Element::from(RawElement::from_be_bytes([
self.0[index],
self.0[index + 1],
self.0[index + 2],
self.0[index + 3],
]))
} }
fn set(&mut self, nybble: u8, element: Element) { fn set(&mut self, nybble: u8, element: Element) {
let values = RawElement::to_be_bytes(element.into()); self.0[nybble as usize] = element.into()
let index = nybble as usize * mem::size_of::<RawElement>();
self.0[index] = values[0];
self.0[index + 1] = values[1];
self.0[index + 2] = values[2];
self.0[index + 3] = values[3];
} }
} }
impl fmt::Debug for Block { impl fmt::Debug for Block {
/// sparse representation for testing and debugging purposes /// sparse representation for testing and debugging purposes
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_map() f.debug_map()
.entries((0..16).filter_map(|i| match self.get(i) { .entries((0..16).filter_map(|i| match self.get(i) {
let (readonly, vec) = self.into_readonly_and_added(); let (readonly, vec) = self.into_readonly_and_added();
// Prevent running `v`'s destructor so we are in complete control // Prevent running `v`'s destructor so we are in complete control
// of the allocation. // of the allocation.
let vec = mem::ManuallyDrop::new(vec); let vec = mem::ManuallyDrop::new(vec);
// Transmute the `Vec<Block>` to a `Vec<u8>`. Blocks are contiguous // Transmute the `Vec<Block>` to a `Vec<u8>`. Blocks are contiguous
// bytes, so this is perfectly safe. // bytes, so this is perfectly safe.
let bytes = unsafe { let bytes = unsafe {
// Assert that `Block` hasn't been changed and has no padding // Check for compatible allocation layout.
let _: [u8; 4 * BLOCK_SIZE] = // (Optimized away by constant-folding + dead code elimination.)
std::mem::transmute([Block::new(); 4]); assert_eq!(mem::size_of::<Block>(), 64);
AlphareUnsubmitted
Not Done

If you're not going to use a constant (as per my comment above), use size_of as mem is already imported.

Alphare: If you're not going to use a constant (as per my comment above), use `size_of` as `mem` is…
assert_eq!(mem::align_of::<Block>(), 1);
// /!\ Any use of `vec` after this is use-after-free. // /!\ Any use of `vec` after this is use-after-free.
// TODO: use `into_raw_parts` once stabilized // TODO: use `into_raw_parts` once stabilized
Vec::from_raw_parts( Vec::from_raw_parts(
vec.as_ptr() as *mut u8, vec.as_ptr() as *mut u8,
vec.len() * BLOCK_SIZE, vec.len() * size_of::<Block>(),
vec.capacity() * BLOCK_SIZE, vec.capacity() * size_of::<Block>(),
) )
}; };
(readonly, bytes) (readonly, bytes)
} }
/// Total number of blocks /// Total number of blocks
fn len(&self) -> usize { fn len(&self) -> usize {
self.readonly.len() + self.growable.len() + 1 self.readonly.len() + self.growable.len() + 1
} }
impl NodeTreeBytes { impl NodeTreeBytes {
fn new( fn new(
buffer: Box<dyn Deref<Target = [u8]> + Send>, buffer: Box<dyn Deref<Target = [u8]> + Send>,
amount: usize, amount: usize,
) -> Self { ) -> Self {
assert!(buffer.len() >= amount); assert!(buffer.len() >= amount);
let len_in_blocks = amount / BLOCK_SIZE; let len_in_blocks = amount / size_of::<Block>();
NodeTreeBytes { NodeTreeBytes {
buffer, buffer,
len_in_blocks, len_in_blocks,
} }
} }
} }
impl Deref for NodeTreeBytes { impl Deref for NodeTreeBytes {
type Target = [Block]; type Target = [Block];
fn deref(&self) -> &[Block] { fn deref(&self) -> &[Block] {
unsafe { Block::slice_from_bytes(&self.buffer, self.len_in_blocks)
AlphareUnsubmitted
Not Done

I'd use expect to explain better why this cannot happen.

Alphare: I'd use `expect` to explain better why this cannot happen.
slice::from_raw_parts( .unwrap()
(&self.buffer).as_ptr() as *const Block, .0
self.len_in_blocks,
)
}
} }
} }
struct NodeTreeVisitor<'n, 'p> { struct NodeTreeVisitor<'n, 'p> {
nt: &'n NodeTree, nt: &'n NodeTree,
prefix: NodePrefixRef<'p>, prefix: NodePrefixRef<'p>,
visit: usize, visit: usize,
nybble_idx: usize, nybble_idx: usize,
assert_eq!(format!("{:?}", block), "{5: Block(2), 13: Rev(15)}"); assert_eq!(format!("{:?}", block), "{5: Block(2), 13: Rev(15)}");
} }
#[test] #[test]
fn test_raw_block() { fn test_raw_block() {
let mut raw = [255u8; 64]; let mut raw = [255u8; 64];
let mut counter = 0; let mut counter = 0;
for val in [0, 15, -2, -1, -3].iter() { for val in [0_i32, 15, -2, -1, -3].iter() {
for byte in RawElement::to_be_bytes(*val).iter() { for byte in val.to_be_bytes().iter() {
raw[counter] = *byte; raw[counter] = *byte;
counter += 1; counter += 1;
} }
} }
let block = Block(raw); let (block, _) = Block::from_bytes(&raw).unwrap();
assert_eq!(block.get(0), Element::Block(0)); assert_eq!(block.get(0), Element::Block(0));
assert_eq!(block.get(1), Element::Block(15)); assert_eq!(block.get(1), Element::Block(15));
assert_eq!(block.get(3), Element::None); assert_eq!(block.get(3), Element::None);
assert_eq!(block.get(2), Element::Rev(0)); assert_eq!(block.get(2), Element::Rev(0));
assert_eq!(block.get(4), Element::Rev(1)); assert_eq!(block.get(4), Element::Rev(1));
} }
type TestIndex = HashMap<Revision, Node>; type TestIndex = HashMap<Revision, Node>;
fn test_into_added_bytes() -> Result<(), NodeMapError> { fn test_into_added_bytes() -> Result<(), NodeMapError> {
let mut idx = TestNtIndex::new(); let mut idx = TestNtIndex::new();
idx.insert(0, "1234")?; idx.insert(0, "1234")?;
let mut idx = idx.commit(); let mut idx = idx.commit();
idx.insert(4, "cafe")?; idx.insert(4, "cafe")?;
let (_, bytes) = idx.nt.into_readonly_and_added_bytes(); let (_, bytes) = idx.nt.into_readonly_and_added_bytes();
// only the root block has been changed // only the root block has been changed
assert_eq!(bytes.len(), BLOCK_SIZE); assert_eq!(bytes.len(), size_of::<Block>());
// big endian for -2 // big endian for -2
assert_eq!(&bytes[4..2 * 4], [255, 255, 255, 254]); assert_eq!(&bytes[4..2 * 4], [255, 255, 255, 254]);
// big endian for -6 // big endian for -6
assert_eq!(&bytes[12 * 4..13 * 4], [255, 255, 255, 250]); assert_eq!(&bytes[12 * 4..13 * 4], [255, 255, 255, 250]);
Ok(()) Ok(())
} }
} }

rust/hg-core/src/revlog/nodemap_docket.rs

use bytes_cast::{unaligned, BytesCast};
use memmap::Mmap; use memmap::Mmap;
use std::convert::TryInto;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use super::revlog::RevlogError; use super::revlog::RevlogError;
use crate::repo::Repo; use crate::repo::Repo;
use crate::utils::strip_suffix; use crate::utils::strip_suffix;
const ONDISK_VERSION: u8 = 1; const ONDISK_VERSION: u8 = 1;
pub(super) struct NodeMapDocket { pub(super) struct NodeMapDocket {
pub data_length: usize, pub data_length: usize,
// TODO: keep here more of the data from `parse()` when we need it // TODO: keep here more of the data from `parse()` when we need it
} }
#[derive(BytesCast)]
#[repr(C)]
struct DocketHeader {
uid_size: u8,
_tip_rev: unaligned::U64Be,
data_length: unaligned::U64Be,
_data_unused: unaligned::U64Be,
tip_node_size: unaligned::U64Be,
}
impl NodeMapDocket { impl NodeMapDocket {
/// Return `Ok(None)` when the caller should proceed without a persistent /// Return `Ok(None)` when the caller should proceed without a persistent
/// nodemap: /// nodemap:
/// ///
/// * This revlog does not have a `.n` docket file (it is not generated for /// * This revlog does not have a `.n` docket file (it is not generated for
/// small revlogs), or /// small revlogs), or
/// * The docket has an unsupported version number (repositories created by /// * The docket has an unsupported version number (repositories created by
/// later hg, maybe that should be a requirement instead?), or /// later hg, maybe that should be a requirement instead?), or
/// * The docket file points to a missing (likely deleted) data file (this /// * The docket file points to a missing (likely deleted) data file (this
/// can happen in a rare race condition). /// can happen in a rare race condition).
pub fn read_from_file( pub fn read_from_file(
repo: &Repo, repo: &Repo,
index_path: &Path, index_path: &Path,
) -> Result<Option<(Self, Mmap)>, RevlogError> { ) -> Result<Option<(Self, Mmap)>, RevlogError> {
let docket_path = index_path.with_extension("n"); let docket_path = index_path.with_extension("n");
let docket_bytes = match repo.store_vfs().read(&docket_path) { let docket_bytes = match repo.store_vfs().read(&docket_path) {
Err(e) if e.kind() == std::io::ErrorKind::NotFound => { Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
return Ok(None) return Ok(None)
} }
Err(e) => return Err(RevlogError::IoError(e)), Err(e) => return Err(RevlogError::IoError(e)),
Ok(bytes) => bytes, Ok(bytes) => bytes,
}; };
let mut input = if let Some((&ONDISK_VERSION, rest)) = let input = if let Some((&ONDISK_VERSION, rest)) =
docket_bytes.split_first() docket_bytes.split_first()
{ {
rest rest
} else { } else {
return Ok(None); return Ok(None);
}; };
let input = &mut input;
let uid_size = read_u8(input)? as usize; let (header, rest) = DocketHeader::from_bytes(input)?;
let _tip_rev = read_be_u64(input)?; let uid_size = header.uid_size as usize;
// TODO: do we care about overflow for 4 GB+ nodemap files on 32-bit // TODO: do we care about overflow for 4 GB+ nodemap files on 32-bit
// systems? // systems?
let data_length = read_be_u64(input)? as usize; let tip_node_size = header.tip_node_size.get() as usize;
let _data_unused = read_be_u64(input)?; let data_length = header.data_length.get() as usize;
let tip_node_size = read_be_u64(input)? as usize; let (uid, rest) = u8::slice_from_bytes(rest, uid_size)?;
let uid = read_bytes(input, uid_size)?; let (_tip_node, _rest) = u8::slice_from_bytes(rest, tip_node_size)?;
let _tip_node = read_bytes(input, tip_node_size)?;
let uid = let uid =
std::str::from_utf8(uid).map_err(|_| RevlogError::Corrupted)?; std::str::from_utf8(uid).map_err(|_| RevlogError::Corrupted)?;
let docket = NodeMapDocket { data_length }; let docket = NodeMapDocket { data_length };
let data_path = rawdata_path(&docket_path, uid); let data_path = rawdata_path(&docket_path, uid);
// TODO: use `std::fs::read` here when the `persistent-nodemap.mmap` // TODO: use `std::fs::read` here when the `persistent-nodemap.mmap`
// config is false? // config is false?
match repo.store_vfs().mmap_open(&data_path) { match repo.store_vfs().mmap_open(&data_path) {
} else { } else {
Err(RevlogError::IoError(error)) Err(RevlogError::IoError(error))
} }
} }
} }
} }
} }
fn read_bytes<'a>(
input: &mut &'a [u8],
count: usize,
) -> Result<&'a [u8], RevlogError> {
if let Some(start) = input.get(..count) {
*input = &input[count..];
Ok(start)
} else {
Err(RevlogError::Corrupted)
}
}
fn read_u8<'a>(input: &mut &[u8]) -> Result<u8, RevlogError> {
Ok(read_bytes(input, 1)?[0])
}
fn read_be_u64<'a>(input: &mut &[u8]) -> Result<u64, RevlogError> {
let array = read_bytes(input, std::mem::size_of::<u64>())?
.try_into()
.unwrap();
Ok(u64::from_be_bytes(array))
}
fn rawdata_path(docket_path: &Path, uid: &str) -> PathBuf { fn rawdata_path(docket_path: &Path, uid: &str) -> PathBuf {
let docket_name = docket_path let docket_name = docket_path
.file_name() .file_name()
.expect("expected a base name") .expect("expected a base name")
.to_str() .to_str()
.expect("expected an ASCII file name in the store"); .expect("expected an ASCII file name in the store");
let prefix = strip_suffix(docket_name, ".n.a") let prefix = strip_suffix(docket_name, ".n.a")
.or_else(|| strip_suffix(docket_name, ".n")) .or_else(|| strip_suffix(docket_name, ".n"))
.expect("expected docket path in .n or .n.a"); .expect("expected docket path in .n or .n.a");
let name = format!("{}-{}.nd", prefix, uid); let name = format!("{}-{}.nd", prefix, uid);
docket_path docket_path
.parent() .parent()
.expect("expected a non-root path") .expect("expected a non-root path")
.join(name) .join(name)
} }

rust/hg-core/src/revlog/revlog.rs

UnsuportedVersion(u16), UnsuportedVersion(u16),
InvalidRevision, InvalidRevision,
/// Found more than one entry whose ID match the requested prefix /// Found more than one entry whose ID match the requested prefix
AmbiguousPrefix, AmbiguousPrefix,
Corrupted, Corrupted,
UnknowDataFormat(u8), UnknowDataFormat(u8),
} }
impl From<bytes_cast::FromBytesError> for RevlogError {
fn from(_: bytes_cast::FromBytesError) -> Self {
RevlogError::Corrupted
}
}
/// Read only implementation of revlog. /// Read only implementation of revlog.
pub struct Revlog { pub struct Revlog {
/// When index and data are not interleaved: bytes of the revlog index. /// When index and data are not interleaved: bytes of the revlog index.
/// When index and data are interleaved: bytes of the revlog index and /// When index and data are interleaved: bytes of the revlog index and
/// data. /// data.
index: Index, index: Index,
/// When index and data are not interleaved: bytes of the revlog data /// When index and data are not interleaved: bytes of the revlog data
data_bytes: Option<Box<dyn Deref<Target = [u8]> + Send>>, data_bytes: Option<Box<dyn Deref<Target = [u8]> + Send>>,