( )⚙ D10653 revlog: introduce a compression mode for sidedata in the revlog index

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10653

revlog: introduce a compression mode for sidedata in the revlog index
ClosedPublic

Authored by marmoute on May 4 2021, 10:15 AM.

Details

Reviewers
indygreg
Alphare
Group Reviewers
hg-reviewers
Commits
rHG2b69555e4875: revlog: introduce a compression mode for sidedata in the revlog index
Summary

We will use this for compression for the sidedata payload.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

marmoute created this revision.May 4 2021, 10:15 AM
baymax updated this revision to Diff 27590.May 5 2021, 5:48 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

baymax updated this revision to Diff 27892.May 11 2021, 1:46 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

Alphare accepted this revision.May 17 2021, 5:49 PM
This revision is now accepted and ready to land.May 17 2021, 5:49 PM

Revision Contents
Changeset List

Diff 28046

mercurial/bundlerepo.py

baserev, baserev,
linkrev, linkrev,
self.rev(p1), self.rev(p1),
self.rev(p2), self.rev(p2),
node, node,
0, 0,
0, 0,
revlog_constants.COMP_MODE_INLINE, revlog_constants.COMP_MODE_INLINE,
revlog_constants.COMP_MODE_INLINE,
) )
self.index.append(e) self.index.append(e)
self.bundlerevs.add(n) self.bundlerevs.add(n)
n += 1 n += 1
def _chunk(self, rev, df=None): def _chunk(self, rev, df=None):
# Warning: in case of bundle, the diff is against what we stored as # Warning: in case of bundle, the diff is against what we stored as
# delta base, not against rev - 1 # delta base, not against rev - 1

mercurial/cext/revlog.c

static const char nullid[32] = {0}; static const char nullid[32] = {0};
static const Py_ssize_t nullrev = -1; static const Py_ssize_t nullrev = -1;
static Py_ssize_t inline_scan(indexObject *self, const char **offsets); static Py_ssize_t inline_scan(indexObject *self, const char **offsets);
static int index_find_node(indexObject *self, const char *node); static int index_find_node(indexObject *self, const char *node);
#if LONG_MAX == 0x7fffffffL #if LONG_MAX == 0x7fffffffL
static const char *const tuple_format = PY23("Kiiiiiis#KiB", "Kiiiiiiy#KiB"); static const char *const tuple_format = PY23("Kiiiiiis#KiBB", "Kiiiiiiy#KiBB");
#else #else
static const char *const tuple_format = PY23("kiiiiiis#kiB", "kiiiiiiy#kiB"); static const char *const tuple_format = PY23("kiiiiiis#kiBB", "kiiiiiiy#kiBB");
#endif #endif
/* A RevlogNG v1 index entry is 64 bytes long. */ /* A RevlogNG v1 index entry is 64 bytes long. */
static const long v1_entry_size = 64; static const long v1_entry_size = 64;
/* A Revlogv2 index entry is 96 bytes long. */ /* A Revlogv2 index entry is 96 bytes long. */
static const long v2_entry_size = 96; static const long v2_entry_size = 96;
* 4 bytes: parent 2 revision * 4 bytes: parent 2 revision
* 32 bytes: nodeid (only 20 bytes used with SHA-1) * 32 bytes: nodeid (only 20 bytes used with SHA-1)
*/ */
static PyObject *index_get(indexObject *self, Py_ssize_t pos) static PyObject *index_get(indexObject *self, Py_ssize_t pos)
{ {
uint64_t offset_flags, sidedata_offset; uint64_t offset_flags, sidedata_offset;
int comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2, int comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2,
sidedata_comp_len; sidedata_comp_len;
char data_comp_mode; char data_comp_mode, sidedata_comp_mode;
const char *c_node_id; const char *c_node_id;
const char *data; const char *data;
Py_ssize_t length = index_length(self); Py_ssize_t length = index_length(self);
if (pos == nullrev) { if (pos == nullrev) {
Py_INCREF(self->nullentry); Py_INCREF(self->nullentry);
return self->nullentry; return self->nullentry;
} }
parent_1 = getbe32(data + 24); parent_1 = getbe32(data + 24);
parent_2 = getbe32(data + 28); parent_2 = getbe32(data + 28);
c_node_id = data + 32; c_node_id = data + 32;
if (self->format_version == format_v1) { if (self->format_version == format_v1) {
sidedata_offset = 0; sidedata_offset = 0;
sidedata_comp_len = 0; sidedata_comp_len = 0;
data_comp_mode = comp_mode_inline; data_comp_mode = comp_mode_inline;
sidedata_comp_mode = comp_mode_inline;
} else { } else {
sidedata_offset = getbe64(data + 64); sidedata_offset = getbe64(data + 64);
sidedata_comp_len = getbe32(data + 72); sidedata_comp_len = getbe32(data + 72);
data_comp_mode = data[76]; data_comp_mode = data[76] & 3;
sidedata_comp_mode = ((data[76] >> 2) & 3);
} }
return Py_BuildValue(tuple_format, offset_flags, comp_len, uncomp_len, return Py_BuildValue(tuple_format, offset_flags, comp_len, uncomp_len,
base_rev, link_rev, parent_1, parent_2, c_node_id, base_rev, link_rev, parent_1, parent_2, c_node_id,
self->nodelen, sidedata_offset, sidedata_comp_len, self->nodelen, sidedata_offset, sidedata_comp_len,
data_comp_mode); data_comp_mode, sidedata_comp_mode);
} }
/* /*
* Pack header information in binary * Pack header information in binary
*/ */
static PyObject *index_pack_header(indexObject *self, PyObject *args) static PyObject *index_pack_header(indexObject *self, PyObject *args)
{ {
int header; int header;
char out[4]; char out[4];
thisnodelen, nodelen); thisnodelen, nodelen);
return -1; return -1;
} }
static PyObject *index_append(indexObject *self, PyObject *obj) static PyObject *index_append(indexObject *self, PyObject *obj)
{ {
uint64_t offset_flags, sidedata_offset; uint64_t offset_flags, sidedata_offset;
int rev, comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2; int rev, comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2;
char data_comp_mode; char data_comp_mode, sidedata_comp_mode;
Py_ssize_t c_node_id_len, sidedata_comp_len; Py_ssize_t c_node_id_len, sidedata_comp_len;
const char *c_node_id; const char *c_node_id;
char comp_field;
char *data; char *data;
if (!PyArg_ParseTuple(obj, tuple_format, &offset_flags, &comp_len, if (!PyArg_ParseTuple(obj, tuple_format, &offset_flags, &comp_len,
&uncomp_len, &base_rev, &link_rev, &parent_1, &uncomp_len, &base_rev, &link_rev, &parent_1,
&parent_2, &c_node_id, &c_node_id_len, &parent_2, &c_node_id, &c_node_id_len,
&sidedata_offset, &sidedata_comp_len, &sidedata_offset, &sidedata_comp_len,
&data_comp_mode)) { &data_comp_mode, &sidedata_comp_mode)) {
PyErr_SetString(PyExc_TypeError, "11-tuple required"); PyErr_SetString(PyExc_TypeError, "11-tuple required");
return NULL; return NULL;
} }
if (c_node_id_len != self->nodelen) { if (c_node_id_len != self->nodelen) {
PyErr_SetString(PyExc_TypeError, "invalid node"); PyErr_SetString(PyExc_TypeError, "invalid node");
return NULL; return NULL;
} }
if (self->format_version == format_v1 && if (self->format_version == format_v1) {
data_comp_mode != comp_mode_inline) {
if (data_comp_mode != comp_mode_inline) {
PyErr_Format(PyExc_ValueError, PyErr_Format(PyExc_ValueError,
"invalid data compression mode: %i", "invalid data compression mode: %i",
data_comp_mode); data_comp_mode);
return NULL; return NULL;
} }
if (sidedata_comp_mode != comp_mode_inline) {
PyErr_Format(PyExc_ValueError,
"invalid sidedata compression mode: %i",
sidedata_comp_mode);
return NULL;
}
}
if (self->new_length == self->added_length) { if (self->new_length == self->added_length) {
size_t new_added_length = size_t new_added_length =
self->added_length ? self->added_length * 2 : 4096; self->added_length ? self->added_length * 2 : 4096;
void *new_added = PyMem_Realloc( void *new_added = PyMem_Realloc(
self->added, new_added_length * self->entry_size); self->added, new_added_length * self->entry_size);
if (!new_added) if (!new_added)
return PyErr_NoMemory(); return PyErr_NoMemory();
putbe32(parent_1, data + 24); putbe32(parent_1, data + 24);
putbe32(parent_2, data + 28); putbe32(parent_2, data + 28);
memcpy(data + 32, c_node_id, c_node_id_len); memcpy(data + 32, c_node_id, c_node_id_len);
/* Padding since SHA-1 is only 20 bytes for now */ /* Padding since SHA-1 is only 20 bytes for now */
memset(data + 32 + c_node_id_len, 0, 32 - c_node_id_len); memset(data + 32 + c_node_id_len, 0, 32 - c_node_id_len);
if (self->format_version == format_v2) { if (self->format_version == format_v2) {
putbe64(sidedata_offset, data + 64); putbe64(sidedata_offset, data + 64);
putbe32(sidedata_comp_len, data + 72); putbe32(sidedata_comp_len, data + 72);
data[76] = (char)data_comp_mode; comp_field = data_comp_mode & 3;
comp_field = comp_field | (sidedata_comp_mode & 3) << 2;
data[76] = comp_field;
/* Padding for 96 bytes alignment */ /* Padding for 96 bytes alignment */
memset(data + 77, 0, self->entry_size - 77); memset(data + 77, 0, self->entry_size - 77);
} }
if (self->ntinitialized) if (self->ntinitialized)
nt_insert(&self->nt, c_node_id, rev); nt_insert(&self->nt, c_node_id, rev);
Py_CLEAR(self->headrevs); Py_CLEAR(self->headrevs);
if (revlogv2 && PyObject_IsTrue(revlogv2)) { if (revlogv2 && PyObject_IsTrue(revlogv2)) {
self->format_version = format_v2; self->format_version = format_v2;
self->entry_size = v2_entry_size; self->entry_size = v2_entry_size;
} else { } else {
self->format_version = format_v1; self->format_version = format_v1;
self->entry_size = v1_entry_size; self->entry_size = v1_entry_size;
} }
self->nullentry = Py_BuildValue(PY23("iiiiiiis#iiB", "iiiiiiiy#iiB"), 0, self->nullentry = Py_BuildValue(
0, 0, -1, -1, -1, -1, nullid, PY23("iiiiiiis#iiBB", "iiiiiiiy#iiBB"), 0, 0, 0, -1, -1, -1, -1,
self->nodelen, 0, 0, comp_mode_inline); nullid, self->nodelen, 0, 0, comp_mode_inline, comp_mode_inline);
if (!self->nullentry) if (!self->nullentry)
return -1; return -1;
PyObject_GC_UnTrack(self->nullentry); PyObject_GC_UnTrack(self->nullentry);
if (PyObject_GetBuffer(data_obj, &self->buf, PyBUF_SIMPLE) == -1) if (PyObject_GetBuffer(data_obj, &self->buf, PyBUF_SIMPLE) == -1)
return -1; return -1;
size = self->buf.len; size = self->buf.len;

mercurial/pure/parsers.py

-1, -1,
-1, -1,
-1, -1,
-1, -1,
sha1nodeconstants.nullid, sha1nodeconstants.nullid,
0, 0,
0, 0,
revlog_constants.COMP_MODE_INLINE, revlog_constants.COMP_MODE_INLINE,
revlog_constants.COMP_MODE_INLINE,
) )
@util.propertycache @util.propertycache
def entry_size(self): def entry_size(self):
return self.index_format.size return self.index_format.size
@property @property
def nodemap(self): def nodemap(self):
data = self._data[index : index + self.entry_size] data = self._data[index : index + self.entry_size]
r = self._unpack_entry(data) r = self._unpack_entry(data)
if self._lgt and i == 0: if self._lgt and i == 0:
r = (offset_type(0, gettype(r[0])),) + r[1:] r = (offset_type(0, gettype(r[0])),) + r[1:]
return r return r
def _unpack_entry(self, data): def _unpack_entry(self, data):
r = self.index_format.unpack(data) r = self.index_format.unpack(data)
r = r + (0, 0, revlog_constants.COMP_MODE_INLINE) r = r + (
0,
0,
revlog_constants.COMP_MODE_INLINE,
revlog_constants.COMP_MODE_INLINE,
)
return r return r
def pack_header(self, header): def pack_header(self, header):
"""pack header information as binary""" """pack header information as binary"""
v_fmt = revlog_constants.INDEX_HEADER v_fmt = revlog_constants.INDEX_HEADER
return v_fmt.pack(header) return v_fmt.pack(header)
def entry_binary(self, rev): def entry_binary(self, rev):
entry[0] = offset_flags entry[0] = offset_flags
entry[8] = sidedata_offset entry[8] = sidedata_offset
entry[9] = sidedata_length entry[9] = sidedata_length
entry = tuple(entry) entry = tuple(entry)
new = self._pack_entry(entry) new = self._pack_entry(entry)
self._extra[rev - self._lgt] = new self._extra[rev - self._lgt] = new
def _unpack_entry(self, data): def _unpack_entry(self, data):
return self.index_format.unpack(data) data = self.index_format.unpack(data)
entry = data[:10]
data_comp = data[10] & 3
sidedata_comp = (data[10] & (3 << 2)) >> 2
return entry + (data_comp, sidedata_comp)
def _pack_entry(self, entry): def _pack_entry(self, entry):
return self.index_format.pack(*entry[:11]) data = entry[:10]
data_comp = entry[10] & 3
sidedata_comp = (entry[11] & 3) << 2
data += (data_comp | sidedata_comp,)
return self.index_format.pack(*data)
def entry_binary(self, rev): def entry_binary(self, rev):
"""return the raw binary string representing a revision""" """return the raw binary string representing a revision"""
entry = self[rev] entry = self[rev]
return self._pack_entry(entry) return self._pack_entry(entry)
def pack_header(self, header): def pack_header(self, header):
"""pack header information as binary""" """pack header information as binary"""

mercurial/revlog.py

[9] sidedata chunk length: [9] sidedata chunk length:
The size, in bytes, of the revision's side-data chunk. The size, in bytes, of the revision's side-data chunk.
[10] data compression mode: [10] data compression mode:
two bits that detail the way the data chunk is compressed on disk. two bits that detail the way the data chunk is compressed on disk.
(see "COMP_MODE_*" constants for details). For revlog version 0 and (see "COMP_MODE_*" constants for details). For revlog version 0 and
1 this will always be COMP_MODE_INLINE. 1 this will always be COMP_MODE_INLINE.
[11] side-data compression mode:
two bits that detail the way the sidedata chunk is compressed on disk.
(see "COMP_MODE_*" constants for details)
""" """
_flagserrorclass = error.RevlogError _flagserrorclass = error.RevlogError
def __init__( def __init__(
self, self,
opener, opener,
target, target,
elif t == self._docket.default_compression_header: elif t == self._docket.default_compression_header:
compression_mode = COMP_MODE_DEFAULT compression_mode = COMP_MODE_DEFAULT
elif h == b'u': elif h == b'u':
# we have a more efficient way to declare uncompressed # we have a more efficient way to declare uncompressed
h = b'' h = b''
compression_mode = COMP_MODE_PLAIN compression_mode = COMP_MODE_PLAIN
deltainfo = deltautil.drop_u_compression(deltainfo) deltainfo = deltautil.drop_u_compression(deltainfo)
sidedata_compression_mode = COMP_MODE_INLINE
if sidedata and self.hassidedata: if sidedata and self.hassidedata:
sidedata_compression_mode = COMP_MODE_PLAIN
serialized_sidedata = sidedatautil.serialize_sidedata(sidedata) serialized_sidedata = sidedatautil.serialize_sidedata(sidedata)
sidedata_offset = offset + deltainfo.deltalen sidedata_offset = offset + deltainfo.deltalen
else: else:
serialized_sidedata = b"" serialized_sidedata = b""
# Don't store the offset if the sidedata is empty, that way # Don't store the offset if the sidedata is empty, that way
# we can easily detect empty sidedata and they will be no different # we can easily detect empty sidedata and they will be no different
# than ones we manually add. # than ones we manually add.
sidedata_offset = 0 sidedata_offset = 0
e = ( e = (
offset_type(offset, flags), offset_type(offset, flags),
deltainfo.deltalen, deltainfo.deltalen,
textlen, textlen,
deltainfo.base, deltainfo.base,
link, link,
p1r, p1r,
p2r, p2r,
node, node,
sidedata_offset, sidedata_offset,
len(serialized_sidedata), len(serialized_sidedata),
compression_mode, compression_mode,
sidedata_compression_mode,
) )
self.index.append(e) self.index.append(e)
entry = self.index.entry_binary(curr) entry = self.index.entry_binary(curr)
if curr == 0 and self._docket is None: if curr == 0 and self._docket is None:
header = self._format_flags | self._format_version header = self._format_flags | self._format_version
header = self.index.pack_header(header) header = self.index.pack_header(header)
entry = header + entry entry = header + entry

mercurial/revlogutils/revlogv0.py

-1, -1,
-1, -1,
-1, -1,
-1, -1,
sha1nodeconstants.nullid, sha1nodeconstants.nullid,
0, 0,
0, 0,
COMP_MODE_INLINE, COMP_MODE_INLINE,
COMP_MODE_INLINE,
) )
@property @property
def nodemap(self): def nodemap(self):
msg = b"index.nodemap is deprecated, use index.[has_node|rev|get_rev]" msg = b"index.nodemap is deprecated, use index.[has_node|rev|get_rev]"
util.nouideprecwarn(msg, b'5.3', stacklevel=2) util.nouideprecwarn(msg, b'5.3', stacklevel=2)
return self._nodemap return self._nodemap

mercurial/unionrepo.py

base, base,
linkrev, linkrev,
p1rev, p1rev,
p2rev, p2rev,
node, node,
_sdo, _sdo,
_sds, _sds,
_dcm, _dcm,
_sdcm,
) = rev ) = rev
flags = _start & 0xFFFF flags = _start & 0xFFFF
if linkmapper is None: # link is to same revlog if linkmapper is None: # link is to same revlog
assert linkrev == rev2 # we never link back assert linkrev == rev2 # we never link back
link = n link = n
else: # rev must be mapped from repo2 cl to unified cl by linkmapper else: # rev must be mapped from repo2 cl to unified cl by linkmapper
link = linkmapper(linkrev) link = linkmapper(linkrev)
base, base,
link, link,
self.rev(p1node), self.rev(p1node),
self.rev(p2node), self.rev(p2node),
node, node,
0, # sidedata offset 0, # sidedata offset
0, # sidedata size 0, # sidedata size
revlog_constants.COMP_MODE_INLINE, revlog_constants.COMP_MODE_INLINE,
revlog_constants.COMP_MODE_INLINE,
) )
self.index.append(e) self.index.append(e)
self.bundlerevs.add(n) self.bundlerevs.add(n)
n += 1 n += 1
def _chunk(self, rev): def _chunk(self, rev):
if rev <= self.repotiprev: if rev <= self.repotiprev:
return revlog.revlog._chunk(self, rev) return revlog.revlog._chunk(self, rev)

tests/test-parseindex2.py

n = off = 0 n = off = 0
l = len(data) - s l = len(data) - s
append = index.append append = index.append
if inline: if inline:
cache = (0, data) cache = (0, data)
while off <= l: while off <= l:
e = struct.unpack(indexformatng, data[off : off + s]) e = struct.unpack(indexformatng, data[off : off + s])
e = e + (0, 0, constants.COMP_MODE_INLINE) e = e + (
0,
0,
constants.COMP_MODE_INLINE,
constants.COMP_MODE_INLINE,
)
nodemap[e[7]] = n nodemap[e[7]] = n
append(e) append(e)
n += 1 n += 1
if e[1] < 0: if e[1] < 0:
break break
off += e[1] + s off += e[1] + s
else: else:
while off <= l: while off <= l:
e = struct.unpack(indexformatng, data[off : off + s]) e = struct.unpack(indexformatng, data[off : off + s])
e = e + (0, 0, constants.COMP_MODE_INLINE) e = e + (
0,
0,
constants.COMP_MODE_INLINE,
constants.COMP_MODE_INLINE,
)
nodemap[e[7]] = n nodemap[e[7]] = n
append(e) append(e)
n += 1 n += 1
off += s off += s
e = list(index[0]) e = list(index[0])
type = gettype(e[0]) type = gettype(e[0])
e[0] = offset_type(0, type) e[0] = offset_type(0, type)
-1, -1,
-1, -1,
-1, -1,
-1, -1,
sha1nodeconstants.nullid, sha1nodeconstants.nullid,
0, 0,
0, 0,
constants.COMP_MODE_INLINE, constants.COMP_MODE_INLINE,
constants.COMP_MODE_INLINE,
) )
index, junk = parsers.parse_index2(data_inlined, True) index, junk = parsers.parse_index2(data_inlined, True)
got = index[-1] got = index[-1]
self.assertEqual(want, got) # inline data self.assertEqual(want, got) # inline data
index, junk = parsers.parse_index2(data_non_inlined, False) index, junk = parsers.parse_index2(data_non_inlined, False)
got = index[-1] got = index[-1]
self.assertEqual(want, got) # no inline data self.assertEqual(want, got) # no inline data
1, 1,
34, 34,
p1, p1,
p2, p2,
node, node,
0, 0,
0, 0,
constants.COMP_MODE_INLINE, constants.COMP_MODE_INLINE,
constants.COMP_MODE_INLINE,
) )
index.append(e) index.append(e)
appendrev(4) appendrev(4)
appendrev(5) appendrev(5)
appendrev(6) appendrev(6)
self.assertEqual(len(index), 7) self.assertEqual(len(index), 7)