( )⚙ D10646 revlog: add a "data compression mode" entry in the index tuple

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10646

revlog: add a "data compression mode" entry in the index tuple
ClosedPublic

Authored by marmoute on May 4 2021, 10:14 AM.

Details

Reviewers
indygreg
Alphare
Group Reviewers
hg-reviewers
Commits
rHG130c9f7ed914: revlog: add a "data compression mode" entry in the index tuple
Summary

That will make it possible to keep track of compression information in the
revlog index, opening the way to more efficient revision restoration (in native
code, but the python usage is already defeating performance work).

We start with adding a new entry to the index tuple, using a value matching the
current behavior. We will introduce storage and other value in later changesets.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

marmoute created this revision.May 4 2021, 10:14 AM
joerg.sonnenberger added a subscriber: joerg.sonnenberger.May 4 2021, 1:03 PM

I don't understand the motivation for this change.

marmoute added a comment.May 5 2021, 5:46 PM
In D10646#161852, @joerg.sonnenberger wrote:

I don't understand the motivation for this change.

This is an implementation of the idea written down by Greg here (in the section "Chunk format / compression stored in chunk)":

https://www.mercurial-scm.org/pipermail/mercurial-devel/2017-February/093657.html

baymax updated this revision to Diff 27583.May 5 2021, 5:48 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

baymax updated this revision to Diff 27885.May 11 2021, 1:45 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

Alphare accepted this revision.May 17 2021, 5:49 PM
This revision is now accepted and ready to land.May 17 2021, 5:49 PM

Revision Contents
Changeset List

Diff 28039

mercurial/bundlerepo.py

-1, -1,
baserev, baserev,
linkrev, linkrev,
self.rev(p1), self.rev(p1),
self.rev(p2), self.rev(p2),
node, node,
0, 0,
0, 0,
revlog_constants.COMP_MODE_INLINE,
) )
self.index.append(e) self.index.append(e)
self.bundlerevs.add(n) self.bundlerevs.add(n)
n += 1 n += 1
def _chunk(self, rev, df=None): def _chunk(self, rev, df=None):
# Warning: in case of bundle, the diff is against what we stored as # Warning: in case of bundle, the diff is against what we stored as
# delta base, not against rev - 1 # delta base, not against rev - 1

mercurial/cext/parsers.c

{"fm1readmarkers", fm1readmarkers, METH_VARARGS, {"fm1readmarkers", fm1readmarkers, METH_VARARGS,
"parse v1 obsolete markers\n"}, "parse v1 obsolete markers\n"},
{NULL, NULL}}; {NULL, NULL}};
void dirs_module_init(PyObject *mod); void dirs_module_init(PyObject *mod);
void manifest_module_init(PyObject *mod); void manifest_module_init(PyObject *mod);
void revlog_module_init(PyObject *mod); void revlog_module_init(PyObject *mod);
static const int version = 18; static const int version = 19;
static void module_init(PyObject *mod) static void module_init(PyObject *mod)
{ {
PyObject *capsule = NULL; PyObject *capsule = NULL;
PyModule_AddIntConstant(mod, "version", version); PyModule_AddIntConstant(mod, "version", version);
/* This module constant has two purposes. First, it lets us unit test /* This module constant has two purposes. First, it lets us unit test
* the ImportError raised without hard-coding any error text. This * the ImportError raised without hard-coding any error text. This

mercurial/cext/revlog.c

static const char nullid[32] = {0}; static const char nullid[32] = {0};
static const Py_ssize_t nullrev = -1; static const Py_ssize_t nullrev = -1;
static Py_ssize_t inline_scan(indexObject *self, const char **offsets); static Py_ssize_t inline_scan(indexObject *self, const char **offsets);
static int index_find_node(indexObject *self, const char *node); static int index_find_node(indexObject *self, const char *node);
#if LONG_MAX == 0x7fffffffL #if LONG_MAX == 0x7fffffffL
static const char *const tuple_format = PY23("Kiiiiiis#Ki", "Kiiiiiiy#Ki"); static const char *const tuple_format = PY23("Kiiiiiis#KiB", "Kiiiiiiy#KiB");
#else #else
static const char *const tuple_format = PY23("kiiiiiis#ki", "kiiiiiiy#ki"); static const char *const tuple_format = PY23("kiiiiiis#kiB", "kiiiiiiy#kiB");
#endif #endif
/* A RevlogNG v1 index entry is 64 bytes long. */ /* A RevlogNG v1 index entry is 64 bytes long. */
static const long v1_entry_size = 64; static const long v1_entry_size = 64;
/* A Revlogv2 index entry is 96 bytes long. */ /* A Revlogv2 index entry is 96 bytes long. */
static const long v2_entry_size = 96; static const long v2_entry_size = 96;
static const long format_v1 = 1; /* Internal only, could be any number */ static const long format_v1 = 1; /* Internal only, could be any number */
static const long format_v2 = 2; /* Internal only, could be any number */ static const long format_v2 = 2; /* Internal only, could be any number */
static const char comp_mode_inline = 2;
static void raise_revlog_error(void) static void raise_revlog_error(void)
{ {
PyObject *mod = NULL, *dict = NULL, *errclass = NULL; PyObject *mod = NULL, *dict = NULL, *errclass = NULL;
mod = PyImport_ImportModule("mercurial.error"); mod = PyImport_ImportModule("mercurial.error");
if (mod == NULL) { if (mod == NULL) {
goto cleanup; goto cleanup;
} }
* 4 bytes: parent 2 revision * 4 bytes: parent 2 revision
* 32 bytes: nodeid (only 20 bytes used with SHA-1) * 32 bytes: nodeid (only 20 bytes used with SHA-1)
*/ */
static PyObject *index_get(indexObject *self, Py_ssize_t pos) static PyObject *index_get(indexObject *self, Py_ssize_t pos)
{ {
uint64_t offset_flags, sidedata_offset; uint64_t offset_flags, sidedata_offset;
int comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2, int comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2,
sidedata_comp_len; sidedata_comp_len;
char data_comp_mode;
const char *c_node_id; const char *c_node_id;
const char *data; const char *data;
Py_ssize_t length = index_length(self); Py_ssize_t length = index_length(self);
if (pos == nullrev) { if (pos == nullrev) {
Py_INCREF(self->nullentry); Py_INCREF(self->nullentry);
return self->nullentry; return self->nullentry;
} }
if (self->entry_size == v1_entry_size) { if (self->entry_size == v1_entry_size) {
sidedata_offset = 0; sidedata_offset = 0;
sidedata_comp_len = 0; sidedata_comp_len = 0;
} else { } else {
sidedata_offset = getbe64(data + 64); sidedata_offset = getbe64(data + 64);
sidedata_comp_len = getbe32(data + 72); sidedata_comp_len = getbe32(data + 72);
} }
data_comp_mode = comp_mode_inline;
return Py_BuildValue(tuple_format, offset_flags, comp_len, uncomp_len, return Py_BuildValue(tuple_format, offset_flags, comp_len, uncomp_len,
base_rev, link_rev, parent_1, parent_2, c_node_id, base_rev, link_rev, parent_1, parent_2, c_node_id,
self->nodelen, sidedata_offset, sidedata_comp_len); self->nodelen, sidedata_offset, sidedata_comp_len,
data_comp_mode);
} }
/* /*
* Pack header information in binary * Pack header information in binary
*/ */
static PyObject *index_pack_header(indexObject *self, PyObject *args) static PyObject *index_pack_header(indexObject *self, PyObject *args)
{ {
int header; int header;
char out[4]; char out[4];
thisnodelen, nodelen); thisnodelen, nodelen);
return -1; return -1;
} }
static PyObject *index_append(indexObject *self, PyObject *obj) static PyObject *index_append(indexObject *self, PyObject *obj)
{ {
uint64_t offset_flags, sidedata_offset; uint64_t offset_flags, sidedata_offset;
int rev, comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2; int rev, comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2;
char data_comp_mode;
Py_ssize_t c_node_id_len, sidedata_comp_len; Py_ssize_t c_node_id_len, sidedata_comp_len;
const char *c_node_id; const char *c_node_id;
char *data; char *data;
if (!PyArg_ParseTuple(obj, tuple_format, &offset_flags, &comp_len, if (!PyArg_ParseTuple(obj, tuple_format, &offset_flags, &comp_len,
&uncomp_len, &base_rev, &link_rev, &parent_1, &uncomp_len, &base_rev, &link_rev, &parent_1,
&parent_2, &c_node_id, &c_node_id_len, &parent_2, &c_node_id, &c_node_id_len,
&sidedata_offset, &sidedata_comp_len)) { &sidedata_offset, &sidedata_comp_len,
PyErr_SetString(PyExc_TypeError, "10-tuple required"); &data_comp_mode)) {
PyErr_SetString(PyExc_TypeError, "11-tuple required");
return NULL; return NULL;
} }
if (c_node_id_len != self->nodelen) { if (c_node_id_len != self->nodelen) {
PyErr_SetString(PyExc_TypeError, "invalid node"); PyErr_SetString(PyExc_TypeError, "invalid node");
return NULL; return NULL;
} }
if (data_comp_mode != comp_mode_inline) {
PyErr_Format(PyExc_ValueError,
"invalid data compression mode: %i",
data_comp_mode);
return NULL;
}
if (self->new_length == self->added_length) { if (self->new_length == self->added_length) {
size_t new_added_length = size_t new_added_length =
self->added_length ? self->added_length * 2 : 4096; self->added_length ? self->added_length * 2 : 4096;
void *new_added = PyMem_Realloc( void *new_added = PyMem_Realloc(
self->added, new_added_length * self->entry_size); self->added, new_added_length * self->entry_size);
if (!new_added) if (!new_added)
return PyErr_NoMemory(); return PyErr_NoMemory();
if (revlogv2 && PyObject_IsTrue(revlogv2)) { if (revlogv2 && PyObject_IsTrue(revlogv2)) {
self->format_version = format_v2; self->format_version = format_v2;
self->entry_size = v2_entry_size; self->entry_size = v2_entry_size;
} else { } else {
self->format_version = format_v1; self->format_version = format_v1;
self->entry_size = v1_entry_size; self->entry_size = v1_entry_size;
} }
self->nullentry = self->nullentry = Py_BuildValue(PY23("iiiiiiis#iiB", "iiiiiiiy#iiB"), 0,
Py_BuildValue(PY23("iiiiiiis#ii", "iiiiiiiy#ii"), 0, 0, 0, -1, -1, 0, 0, -1, -1, -1, -1, nullid,
-1, -1, nullid, self->nodelen, 0, 0); self->nodelen, 0, 0, comp_mode_inline);
if (!self->nullentry) if (!self->nullentry)
return -1; return -1;
PyObject_GC_UnTrack(self->nullentry); PyObject_GC_UnTrack(self->nullentry);
if (PyObject_GetBuffer(data_obj, &self->buf, PyBUF_SIMPLE) == -1) if (PyObject_GetBuffer(data_obj, &self->buf, PyBUF_SIMPLE) == -1)
return -1; return -1;
size = self->buf.len; size = self->buf.len;

mercurial/policy.py

# keep in sync with "version" in C modules # keep in sync with "version" in C modules
_cextversions = { _cextversions = {
('cext', 'base85'): 1, ('cext', 'base85'): 1,
('cext', 'bdiff'): 3, ('cext', 'bdiff'): 3,
('cext', 'mpatch'): 1, ('cext', 'mpatch'): 1,
('cext', 'osutil'): 4, ('cext', 'osutil'): 4,
('cext', 'parsers'): 18, ('cext', 'parsers'): 19,
} }
# map import request to other package or module # map import request to other package or module
_modredirects = { _modredirects = {
('cext', 'charencode'): ('cext', 'parsers'), ('cext', 'charencode'): ('cext', 'parsers'),
('cffi', 'base85'): ('pure', 'base85'), ('cffi', 'base85'): ('pure', 'base85'),
('cffi', 'charencode'): ('pure', 'charencode'), ('cffi', 'charencode'): ('pure', 'charencode'),
('cffi', 'parsers'): ('pure', 'parsers'), ('cffi', 'parsers'): ('pure', 'parsers'),

mercurial/pure/parsers.py

class BaseIndexObject(object): class BaseIndexObject(object):
# Format of an index entry according to Python's `struct` language # Format of an index entry according to Python's `struct` language
index_format = revlog_constants.INDEX_ENTRY_V1 index_format = revlog_constants.INDEX_ENTRY_V1
# Size of a C unsigned long long int, platform independent # Size of a C unsigned long long int, platform independent
big_int_size = struct.calcsize(b'>Q') big_int_size = struct.calcsize(b'>Q')
# Size of a C long int, platform independent # Size of a C long int, platform independent
int_size = struct.calcsize(b'>i') int_size = struct.calcsize(b'>i')
# An empty index entry, used as a default value to be overridden, or nullrev # An empty index entry, used as a default value to be overridden, or nullrev
null_item = (0, 0, 0, -1, -1, -1, -1, sha1nodeconstants.nullid, 0, 0) null_item = (
0,
0,
0,
-1,
-1,
-1,
-1,
sha1nodeconstants.nullid,
0,
0,
revlog_constants.COMP_MODE_INLINE,
)
@util.propertycache @util.propertycache
def entry_size(self): def entry_size(self):
return self.index_format.size return self.index_format.size
@property @property
def nodemap(self): def nodemap(self):
msg = b"index.nodemap is deprecated, use index.[has_node|rev|get_rev]" msg = b"index.nodemap is deprecated, use index.[has_node|rev|get_rev]"
data = self._data[index : index + self.entry_size] data = self._data[index : index + self.entry_size]
r = self._unpack_entry(data) r = self._unpack_entry(data)
if self._lgt and i == 0: if self._lgt and i == 0:
r = (offset_type(0, gettype(r[0])),) + r[1:] r = (offset_type(0, gettype(r[0])),) + r[1:]
return r return r
def _unpack_entry(self, data): def _unpack_entry(self, data):
r = self.index_format.unpack(data) r = self.index_format.unpack(data)
r = r + (0, 0) r = r + (0, 0, revlog_constants.COMP_MODE_INLINE)
return r return r
def pack_header(self, header): def pack_header(self, header):
"""pack header information as binary""" """pack header information as binary"""
v_fmt = revlog_constants.INDEX_HEADER v_fmt = revlog_constants.INDEX_HEADER
return v_fmt.pack(header) return v_fmt.pack(header)
def entry_binary(self, rev): def entry_binary(self, rev):
entry[0] = offset_flags entry[0] = offset_flags
entry[8] = sidedata_offset entry[8] = sidedata_offset
entry[9] = sidedata_length entry[9] = sidedata_length
entry = tuple(entry) entry = tuple(entry)
new = self._pack_entry(entry) new = self._pack_entry(entry)
self._extra[rev - self._lgt] = new self._extra[rev - self._lgt] = new
def _unpack_entry(self, data): def _unpack_entry(self, data):
return self.index_format.unpack(data) return self.index_format.unpack(data) + (
revlog_constants.COMP_MODE_INLINE,
)
def _pack_entry(self, entry): def _pack_entry(self, entry):
return self.index_format.pack(*entry) return self.index_format.pack(*entry[:10])
def entry_binary(self, rev): def entry_binary(self, rev):
"""return the raw binary string representing a revision""" """return the raw binary string representing a revision"""
entry = self[rev] entry = self[rev]
p = revlog_constants.INDEX_ENTRY_V2.pack(*entry) return self._pack_entry(entry)
return p
def pack_header(self, header): def pack_header(self, header):
"""pack header information as binary""" """pack header information as binary"""
msg = 'version header should go in the docket, not the index: %d' msg = 'version header should go in the docket, not the index: %d'
msg %= header msg %= header
raise error.ProgrammingError(msg) raise error.ProgrammingError(msg)

mercurial/revlog.py

sha1nodeconstants, sha1nodeconstants,
short, short,
wdirrev, wdirrev,
) )
from .i18n import _ from .i18n import _
from .pycompat import getattr from .pycompat import getattr
from .revlogutils.constants import ( from .revlogutils.constants import (
ALL_KINDS, ALL_KINDS,
COMP_MODE_INLINE,
FEATURES_BY_VERSION, FEATURES_BY_VERSION,
FLAG_GENERALDELTA, FLAG_GENERALDELTA,
FLAG_INLINE_DATA, FLAG_INLINE_DATA,
INDEX_HEADER, INDEX_HEADER,
REVLOGV0, REVLOGV0,
REVLOGV1, REVLOGV1,
REVLOGV1_FLAGS, REVLOGV1_FLAGS,
REVLOGV2, REVLOGV2,
[7] node id: [7] node id:
The node id of the current revision The node id of the current revision
[8] sidedata offset: [8] sidedata offset:
The byte index of the start of the revision's side-data chunk. The byte index of the start of the revision's side-data chunk.
[9] sidedata chunk length: [9] sidedata chunk length:
The size, in bytes, of the revision's side-data chunk. The size, in bytes, of the revision's side-data chunk.
[10] data compression mode:
two bits that detail the way the data chunk is compressed on disk.
(see "COMP_MODE_*" constants for details). For revlog version 0 and
1 this will always be COMP_MODE_INLINE.
""" """
_flagserrorclass = error.RevlogError _flagserrorclass = error.RevlogError
def __init__( def __init__(
self, self,
opener, opener,
target, target,
textlen, textlen,
deltainfo.base, deltainfo.base,
link, link,
p1r, p1r,
p2r, p2r,
node, node,
sidedata_offset, sidedata_offset,
len(serialized_sidedata), len(serialized_sidedata),
COMP_MODE_INLINE,
) )
self.index.append(e) self.index.append(e)
entry = self.index.entry_binary(curr) entry = self.index.entry_binary(curr)
if curr == 0 and self._docket is None: if curr == 0 and self._docket is None:
header = self._format_flags | self._format_version header = self._format_flags | self._format_version
header = self.index.pack_header(header) header = self.index.pack_header(header)
entry = header + entry entry = header + entry

mercurial/revlogutils/constants.py

# revlogdeltas.py - constant used for revlog logic # revlogdeltas.py - constant used for revlog logic.
# #
# Copyright 2005-2007 Olivia Mackall <olivia@selenic.com> # Copyright 2005-2007 Olivia Mackall <olivia@selenic.com>
# Copyright 2018 Octobus <contact@octobus.net> # Copyright 2018 Octobus <contact@octobus.net>
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
"""Helper class to compute deltas stored inside revlogs""" """Helper class to compute deltas stored inside revlogs"""
REVIDX_ELLIPSIS, REVIDX_ELLIPSIS,
REVIDX_EXTSTORED, REVIDX_EXTSTORED,
REVIDX_HASCOPIESINFO, REVIDX_HASCOPIESINFO,
] ]
# bitmark for flags that could cause rawdata content change # bitmark for flags that could cause rawdata content change
REVIDX_RAWTEXT_CHANGING_FLAGS = REVIDX_ISCENSORED | REVIDX_EXTSTORED REVIDX_RAWTEXT_CHANGING_FLAGS = REVIDX_ISCENSORED | REVIDX_EXTSTORED
## chunk compression mode constants:
# These constants are used in revlog version >=2 to denote the compression used
# for a chunk.
# Chunk use a compression mode stored "inline" at the start of the chunk
# itself. This is the mode always used for revlog version "0" and "1"
COMP_MODE_INLINE = 2
SUPPORTED_FLAGS = { SUPPORTED_FLAGS = {
REVLOGV0: REVLOGV0_FLAGS, REVLOGV0: REVLOGV0_FLAGS,
REVLOGV1: REVLOGV1_FLAGS, REVLOGV1: REVLOGV1_FLAGS,
REVLOGV2: REVLOGV2_FLAGS, REVLOGV2: REVLOGV2_FLAGS,
} }
_no = lambda flags: False _no = lambda flags: False
_yes = lambda flags: True _yes = lambda flags: True
# means to reduce the number of files for revlogv2. # means to reduce the number of files for revlogv2.
b'inline': _no, b'inline': _no,
b'generaldelta': _yes, b'generaldelta': _yes,
b'sidedata': True, b'sidedata': True,
b'docket': True, b'docket': True,
}, },
} }
SPARSE_REVLOG_MAX_CHAIN_LENGTH = 1000 SPARSE_REVLOG_MAX_CHAIN_LENGTH = 1000

mercurial/revlogutils/revlogv0.py

# revlogv0 - code related to revlog format "V0" # revlogv0 - code related to revlog format "V0"
# #
# Copyright 2005-2007 Olivia Mackall <olivia@selenic.com> # Copyright 2005-2007 Olivia Mackall <olivia@selenic.com>
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
from ..node import sha1nodeconstants from ..node import sha1nodeconstants
from .constants import ( from .constants import (
COMP_MODE_INLINE,
INDEX_ENTRY_V0, INDEX_ENTRY_V0,
) )
from ..i18n import _ from ..i18n import _
from .. import ( from .. import (
error, error,
node, node,
pycompat, pycompat,
def offset_type(offset, type): def offset_type(offset, type):
if (type & ~flagutil.REVIDX_KNOWN_FLAGS) != 0: if (type & ~flagutil.REVIDX_KNOWN_FLAGS) != 0:
raise ValueError(b'unknown revlog index flags') raise ValueError(b'unknown revlog index flags')
return int(int(offset) << 16 | type) return int(int(offset) << 16 | type)
class revlogoldindex(list): class revlogoldindex(list):
entry_size = INDEX_ENTRY_V0.size entry_size = INDEX_ENTRY_V0.size
null_item = (0, 0, 0, -1, -1, -1, -1, sha1nodeconstants.nullid, 0, 0) null_item = (
0,
0,
0,
-1,
-1,
-1,
-1,
sha1nodeconstants.nullid,
0,
0,
COMP_MODE_INLINE,
)
@property @property
def nodemap(self): def nodemap(self):
msg = b"index.nodemap is deprecated, use index.[has_node|rev|get_rev]" msg = b"index.nodemap is deprecated, use index.[has_node|rev|get_rev]"
util.nouideprecwarn(msg, b'5.3', stacklevel=2) util.nouideprecwarn(msg, b'5.3', stacklevel=2)
return self._nodemap return self._nodemap
@util.propertycache @util.propertycache
-1, -1,
e[2], e[2],
e[3], e[3],
nodemap.get(e[4], node.nullrev), nodemap.get(e[4], node.nullrev),
nodemap.get(e[5], node.nullrev), nodemap.get(e[5], node.nullrev),
e[6], e[6],
0, # no side data support 0, # no side data support
0, # no side data support 0, # no side data support
COMP_MODE_INLINE,
) )
index.append(e2) index.append(e2)
nodemap[e[6]] = n nodemap[e[6]] = n
n += 1 n += 1
index = revlogoldindex(index) index = revlogoldindex(index)
return index, None return index, None

mercurial/unionrepo.py

manifest, manifest,
mdiff, mdiff,
pathutil, pathutil,
revlog, revlog,
util, util,
vfs as vfsmod, vfs as vfsmod,
) )
from .revlogutils import (
constants as revlog_constants,
)
class unionrevlog(revlog.revlog): class unionrevlog(revlog.revlog):
def __init__(self, opener, radix, revlog2, linkmapper): def __init__(self, opener, radix, revlog2, linkmapper):
# How it works: # How it works:
# To retrieve a revision, we just need to know the node id so we can # To retrieve a revision, we just need to know the node id so we can
# look it up in revlog2. # look it up in revlog2.
# #
# To differentiate a rev in the second revlog from a rev in the revlog, # To differentiate a rev in the second revlog from a rev in the revlog,
rsize, rsize,
base, base,
linkrev, linkrev,
p1rev, p1rev,
p2rev, p2rev,
node, node,
_sdo, _sdo,
_sds, _sds,
_dcm,
) = rev ) = rev
flags = _start & 0xFFFF flags = _start & 0xFFFF
if linkmapper is None: # link is to same revlog if linkmapper is None: # link is to same revlog
assert linkrev == rev2 # we never link back assert linkrev == rev2 # we never link back
link = n link = n
else: # rev must be mapped from repo2 cl to unified cl by linkmapper else: # rev must be mapped from repo2 cl to unified cl by linkmapper
link = linkmapper(linkrev) link = linkmapper(linkrev)
rsize, rsize,
base, base,
link, link,
self.rev(p1node), self.rev(p1node),
self.rev(p2node), self.rev(p2node),
node, node,
0, # sidedata offset 0, # sidedata offset
0, # sidedata size 0, # sidedata size
revlog_constants.COMP_MODE_INLINE,
) )
self.index.append(e) self.index.append(e)
self.bundlerevs.add(n) self.bundlerevs.add(n)
n += 1 n += 1
def _chunk(self, rev): def _chunk(self, rev):
if rev <= self.repotiprev: if rev <= self.repotiprev:
return revlog.revlog._chunk(self, rev) return revlog.revlog._chunk(self, rev)

tests/test-parseindex2.py

hex, hex,
nullrev, nullrev,
sha1nodeconstants, sha1nodeconstants,
) )
from mercurial import ( from mercurial import (
policy, policy,
pycompat, pycompat,
) )
from mercurial.revlogutils import (
constants,
)
parsers = policy.importmod('parsers') parsers = policy.importmod('parsers')
# original python implementation # original python implementation
def gettype(q): def gettype(q):
return int(q & 0xFFFF) return int(q & 0xFFFF)
n = off = 0 n = off = 0
l = len(data) - s l = len(data) - s
append = index.append append = index.append
if inline: if inline:
cache = (0, data) cache = (0, data)
while off <= l: while off <= l:
e = struct.unpack(indexformatng, data[off : off + s]) e = struct.unpack(indexformatng, data[off : off + s])
e = e + (0, 0) e = e + (0, 0, constants.COMP_MODE_INLINE)
nodemap[e[7]] = n nodemap[e[7]] = n
append(e) append(e)
n += 1 n += 1
if e[1] < 0: if e[1] < 0:
break break
off += e[1] + s off += e[1] + s
else: else:
while off <= l: while off <= l:
e = struct.unpack(indexformatng, data[off : off + s]) e = struct.unpack(indexformatng, data[off : off + s])
e = e + (0, 0) e = e + (0, 0, constants.COMP_MODE_INLINE)
nodemap[e[7]] = n nodemap[e[7]] = n
append(e) append(e)
n += 1 n += 1
off += s off += s
e = list(index[0]) e = list(index[0])
type = gettype(e[0]) type = gettype(e[0])
e[0] = offset_type(0, type) e[0] = offset_type(0, type)
i, i,
'Reverse lookup inconsistent for %r' % hex(r[7]), 'Reverse lookup inconsistent for %r' % hex(r[7]),
) )
except TypeError: except TypeError:
# pure version doesn't support this # pure version doesn't support this
break break
def testminusone(self): def testminusone(self):
want = (0, 0, 0, -1, -1, -1, -1, sha1nodeconstants.nullid, 0, 0) want = (
0,
0,
0,
-1,
-1,
-1,
-1,
sha1nodeconstants.nullid,
0,
0,
constants.COMP_MODE_INLINE,
)
index, junk = parsers.parse_index2(data_inlined, True) index, junk = parsers.parse_index2(data_inlined, True)
got = index[-1] got = index[-1]
self.assertEqual(want, got) # inline data self.assertEqual(want, got) # inline data
index, junk = parsers.parse_index2(data_non_inlined, False) index, junk = parsers.parse_index2(data_non_inlined, False)
got = index[-1] got = index[-1]
self.assertEqual(want, got) # no inline data self.assertEqual(want, got) # no inline data
def testdelitemwithoutnodetree(self): def testdelitemwithoutnodetree(self):
index, _junk = parsers.parse_index2(data_non_inlined, False) index, _junk = parsers.parse_index2(data_non_inlined, False)
def hexrev(rev): def hexrev(rev):
if rev == nullrev: if rev == nullrev:
return b'\xff\xff\xff\xff' return b'\xff\xff\xff\xff'
else: else:
return bin('%08x' % rev) return bin('%08x' % rev)
def appendrev(p1, p2=nullrev): def appendrev(p1, p2=nullrev):
# node won't matter for this test, let's just make sure # node won't matter for this test, let's just make sure
# they don't collide. Other data don't matter either. # they don't collide. Other data don't matter either.
node = hexrev(p1) + hexrev(p2) + b'.' * 12 node = hexrev(p1) + hexrev(p2) + b'.' * 12
index.append((0, 0, 12, 1, 34, p1, p2, node, 0, 0)) e = (
0,
0,
12,
1,
34,
p1,
p2,
node,
0,
0,
constants.COMP_MODE_INLINE,
)
index.append(e)
appendrev(4) appendrev(4)
appendrev(5) appendrev(5)
appendrev(6) appendrev(6)
self.assertEqual(len(index), 7) self.assertEqual(len(index), 7)
del index[1:-1] del index[1:-1]