This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10920

dirstate-v2: Use 32-bit integers instead of 64-bit for offsets
ClosedPublic

Authored by SimonSapin on Jul 2 2021, 8:36 AM.

Details

Reviewers
Alphare
Group Reviewers
hg-reviewers
Commits
rHGf23eafb036af: dirstate-v2: Use 32-bit integers instead of 64-bit for offsets
Summary

This saves 12 bytes per node. (Nodes representing files or directories.)

These are offsets to other parts of the file. This would only be a limitation
for a .hg/dirstate file larger than 4 GiB, which would only happen for a
repository with dozens of millions of files and directories.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

Diff 28739

rust/hg-core/src/dirstate_tree/on_disk.rs

use crate::dirstate_tree::dirstate_map::{self, DirstateMap, NodeRef}; use crate::dirstate_tree::dirstate_map::{self, DirstateMap, NodeRef};
use crate::dirstate_tree::path_with_basename::WithBasename; use crate::dirstate_tree::path_with_basename::WithBasename;
use crate::errors::HgError; use crate::errors::HgError;
use crate::utils::hg_path::HgPath; use crate::utils::hg_path::HgPath;
use crate::DirstateEntry; use crate::DirstateEntry;
use crate::DirstateError; use crate::DirstateError;
use crate::DirstateParents; use crate::DirstateParents;
use crate::EntryState; use crate::EntryState;
use bytes_cast::unaligned::{I32Be, I64Be, U32Be, U64Be}; use bytes_cast::unaligned::{I32Be, I64Be, U32Be};
use bytes_cast::BytesCast; use bytes_cast::BytesCast;
use std::borrow::Cow; use std::borrow::Cow;
use std::convert::TryFrom; use std::convert::TryFrom;
use std::time::{Duration, SystemTime, UNIX_EPOCH}; use std::time::{Duration, SystemTime, UNIX_EPOCH};
/// Added at the start of `.hg/dirstate` when the "v2" format is used. /// Added at the start of `.hg/dirstate` when the "v2" format is used.
/// This a redundant sanity check more than an actual "magic number" since /// This a redundant sanity check more than an actual "magic number" since
/// `.hg/requires` already governs which format should be used. /// `.hg/requires` already governs which format should be used.
/// ///
/// This timestamp is later or earlier than `(seconds, 0)` by this many /// This timestamp is later or earlier than `(seconds, 0)` by this many
/// nanoseconds, if `seconds` is non-negative or negative, respectively. /// nanoseconds, if `seconds` is non-negative or negative, respectively.
nanoseconds: U32Be, nanoseconds: U32Be,
} }
/// Counted in bytes from the start of the file /// Counted in bytes from the start of the file
/// ///
/// NOTE: If we decide to never support `.hg/dirstate` files larger than 4 GiB /// NOTE: not supporting `.hg/dirstate` files larger than 4 GiB.
/// we could save space by using `U32Be` instead. type Offset = U32Be;
type Offset = U64Be;
/// Counted in number of items /// Counted in number of items
/// ///
/// NOTE: not supporting directories with more than 4 billion direct children, /// NOTE: not supporting directories with more than 4 billion direct children,
/// or filenames more than 4 GiB. /// or filenames more than 4 GiB.
type Size = U32Be; type Size = U32Be;
/// Location of consecutive, fixed-size items. /// Location of consecutive, fixed-size items.
/// A `HgPath` of `len` bytes /// A `HgPath` of `len` bytes
type PathSlice = Slice; type PathSlice = Slice;
/// Either nothing if `start == 0`, or a `HgPath` of `len` bytes /// Either nothing if `start == 0`, or a `HgPath` of `len` bytes
type OptPathSlice = Slice; type OptPathSlice = Slice;
/// Make sure that size-affecting changes are made knowingly /// Make sure that size-affecting changes are made knowingly
fn _static_assert_size_of() { fn _static_assert_size_of() {
let _ = std::mem::transmute::<Header, [u8; 92]>; let _ = std::mem::transmute::<Header, [u8; 88]>;
let _ = std::mem::transmute::<Node, [u8; 57]>; let _ = std::mem::transmute::<Node, [u8; 45]>;
} }
/// Unexpected file format found in `.hg/dirstate` with the "v2" format. /// Unexpected file format found in `.hg/dirstate` with the "v2" format.
/// ///
/// This should only happen if Mercurial is buggy or a repository is corrupted. /// This should only happen if Mercurial is buggy or a repository is corrupted.
#[derive(Debug)] #[derive(Debug)]
pub struct DirstateV2ParseError; pub struct DirstateV2ParseError;
// … so we can write them contiguously // … so we can write them contiguously
Ok(write_slice::<Node>(&on_disk_nodes, out)) Ok(write_slice::<Node>(&on_disk_nodes, out))
} }
fn write_slice<T>(slice: &[T], out: &mut Vec<u8>) -> Slice fn write_slice<T>(slice: &[T], out: &mut Vec<u8>) -> Slice
where where
T: BytesCast, T: BytesCast,
{ {
let start = u64::try_from(out.len()) let start = u32::try_from(out.len())
// Could only panic on a 128-bit CPU with a dirstate over 16 EiB // Could only panic for a dirstate file larger than 4 GiB
.expect("dirstate-v2 offset overflow") .expect("dirstate-v2 offset overflow")
.into(); .into();
let len = u32::try_from(slice.len()) let len = u32::try_from(slice.len())
// Could only panic for paths over 4 GiB or nodes with over 4 billions // Could only panic for paths over 4 GiB or nodes with over 4 billions
// child nodes // child nodes
.expect("dirstate-v2 offset overflow") .expect("dirstate-v2 offset overflow")
.into(); .into();
out.extend(slice.as_bytes()); out.extend(slice.as_bytes());
Slice { start, len } Slice { start, len }
} }

tests/test-hgignore.t

(ignore rule in $TESTTMP\ignorerepo\.hgignore, line 4: 'dir1/file*') (ignore rule in $TESTTMP\ignorerepo\.hgignore, line 4: 'dir1/file*')
$ hg up -qC . $ hg up -qC .
#endif #endif
#if dirstate-v2 #if dirstate-v2
Check the hash of ignore patterns written in the dirstate at offset Check the hash of ignore patterns written in the dirstate at offset
12 + 20 + 20 + 8 + 4 + 4 + 4 = 72 12 + 20 + 20 + 4 + 4 + 4 + 4 = 68
$ hg status > /dev/null $ hg status > /dev/null
$ cat .hg/testhgignore .hg/testhgignorerel .hgignore dir2/.hgignore dir1/.hgignore dir1/.hgignoretwo | $TESTDIR/f --sha1 $ cat .hg/testhgignore .hg/testhgignorerel .hgignore dir2/.hgignore dir1/.hgignore dir1/.hgignoretwo | $TESTDIR/f --sha1
sha1=6e315b60f15fb5dfa02be00f3e2c8f923051f5ff sha1=6e315b60f15fb5dfa02be00f3e2c8f923051f5ff
>>> import binascii; print(binascii.hexlify(open(".hg/dirstate", "rb").read()[72:][:20]).decode()) >>> import binascii; print(binascii.hexlify(open(".hg/dirstate", "rb").read()[68:][:20]).decode())
6e315b60f15fb5dfa02be00f3e2c8f923051f5ff 6e315b60f15fb5dfa02be00f3e2c8f923051f5ff
$ echo rel > .hg/testhgignorerel $ echo rel > .hg/testhgignorerel
$ hg status > /dev/null $ hg status > /dev/null
$ cat .hg/testhgignore .hg/testhgignorerel .hgignore dir2/.hgignore dir1/.hgignore dir1/.hgignoretwo | $TESTDIR/f --sha1 $ cat .hg/testhgignore .hg/testhgignorerel .hgignore dir2/.hgignore dir1/.hgignore dir1/.hgignoretwo | $TESTDIR/f --sha1
sha1=dea19cc7119213f24b6b582a4bae7b0cb063e34e sha1=dea19cc7119213f24b6b582a4bae7b0cb063e34e
>>> import binascii; print(binascii.hexlify(open(".hg/dirstate", "rb").read()[72:][:20]).decode()) >>> import binascii; print(binascii.hexlify(open(".hg/dirstate", "rb").read()[68:][:20]).decode())
dea19cc7119213f24b6b582a4bae7b0cb063e34e dea19cc7119213f24b6b582a4bae7b0cb063e34e
#endif #endif