( )⚙ D10508 revlog: add a `entry_binary` method on index

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10508

revlog: add a `entry_binary` method on index
ClosedPublic

Authored by marmoute on Apr 23 2021, 10:29 AM.

Details

Reviewers
indygreg
Alphare
pulkit
Group Reviewers
hg-reviewers
Commits
rHG0d8ff1f4ab0c: revlog: add a `entry_binary` method on index
rHG07dd7ea78113: revlog: add a `entry_binary` method on index
Summary

The revlog index is already responsible for unpacking the binary entry, it would be
simpler to make it responsible for packing them. In practice the C version of
the index is already doing this internally.

We introduce a "entry_binary" method that return the binary version of an
existing revision. The method currently need to also take the revlog header to
deal with the "first revision" special case. We will introduce further refactor
in a later changeset to split that logic out.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

Diff 27273

mercurial/cext/revlog.c

sidedata_comp_len = getbe32(data + 72); sidedata_comp_len = getbe32(data + 72);
return Py_BuildValue(v2_tuple_format, offset_flags, comp_len, return Py_BuildValue(v2_tuple_format, offset_flags, comp_len,
uncomp_len, base_rev, link_rev, parent_1, uncomp_len, base_rev, link_rev, parent_1,
parent_2, c_node_id, self->nodelen, parent_2, c_node_id, self->nodelen,
sidedata_offset, sidedata_comp_len); sidedata_offset, sidedata_comp_len);
} }
} }
/*
* Return the raw binary string representing a revision
*/
static PyObject *index_entry_binary(indexObject *self, PyObject *args)
{
long rev;
int header;
const char *data;
char entry[v2_hdrsize];
Py_ssize_t length = index_length(self);
if (!PyArg_ParseTuple(args, "lI", &rev, &header)) {
return NULL;
}
if (rev < 0 || rev >= length) {
PyErr_Format(PyExc_ValueError, "revlog index out of range: %ld",
rev);
return NULL;
};
data = index_deref(self, rev);
if (data == NULL)
return NULL;
if (rev == 0) {
// put the header at the start of the first entry
memcpy(entry, data, self->hdrsize);
putbe32(header, entry);
return PyBytes_FromStringAndSize(entry, self->hdrsize);
}
return PyBytes_FromStringAndSize(data, self->hdrsize);
}
/* /*
* Return the hash of node corresponding to the given rev. * Return the hash of node corresponding to the given rev.
*/ */
static const char *index_node(indexObject *self, Py_ssize_t pos) static const char *index_node(indexObject *self, Py_ssize_t pos)
{ {
Py_ssize_t length = index_length(self); Py_ssize_t length = index_length(self);
const char *data; const char *data;
{"slicechunktodensity", (PyCFunction)index_slicechunktodensity, {"slicechunktodensity", (PyCFunction)index_slicechunktodensity,
METH_VARARGS, "determine revisions with deltas to reconstruct fulltext"}, METH_VARARGS, "determine revisions with deltas to reconstruct fulltext"},
{"append", (PyCFunction)index_append, METH_O, "append an index entry"}, {"append", (PyCFunction)index_append, METH_O, "append an index entry"},
{"partialmatch", (PyCFunction)index_partialmatch, METH_VARARGS, {"partialmatch", (PyCFunction)index_partialmatch, METH_VARARGS,
"match a potentially ambiguous node ID"}, "match a potentially ambiguous node ID"},
{"shortest", (PyCFunction)index_shortest, METH_VARARGS, {"shortest", (PyCFunction)index_shortest, METH_VARARGS,
"find length of shortest hex nodeid of a binary ID"}, "find length of shortest hex nodeid of a binary ID"},
{"stats", (PyCFunction)index_stats, METH_NOARGS, "stats for the index"}, {"stats", (PyCFunction)index_stats, METH_NOARGS, "stats for the index"},
{"entry_binary", (PyCFunction)index_entry_binary, METH_VARARGS,
"return an entry in binary form"},
{NULL} /* Sentinel */ {NULL} /* Sentinel */
}; };
static PyGetSetDef index_getset[] = { static PyGetSetDef index_getset[] = {
{"nodemap", (getter)index_nodemap, NULL, "nodemap", NULL}, {"nodemap", (getter)index_nodemap, NULL, "nodemap", NULL},
{NULL} /* Sentinel */ {NULL} /* Sentinel */
}; };

mercurial/pure/parsers.py

else: else:
index = self._calculate_index(i) index = self._calculate_index(i)
data = self._data[index : index + self.entry_size] data = self._data[index : index + self.entry_size]
r = self.index_format.unpack(data) r = self.index_format.unpack(data)
if self._lgt and i == 0: if self._lgt and i == 0:
r = (offset_type(0, gettype(r[0])),) + r[1:] r = (offset_type(0, gettype(r[0])),) + r[1:]
return r return r
def entry_binary(self, rev, header):
"""return the raw binary string representing a revision"""
entry = self[rev]
p = revlog_constants.INDEX_ENTRY_V1.pack(*entry)
if rev == 0:
v_fmt = revlog_constants.INDEX_HEADER
v_bin = v_fmt.pack(header)
p = v_bin + p[v_fmt.size :]
return p
class IndexObject(BaseIndexObject): class IndexObject(BaseIndexObject):
def __init__(self, data): def __init__(self, data):
assert len(data) % self.entry_size == 0 assert len(data) % self.entry_size == 0, (
len(data),
self.entry_size,
len(data) % self.entry_size,
)
self._data = data self._data = data
self._lgt = len(data) // self.entry_size self._lgt = len(data) // self.entry_size
self._extra = [] self._extra = []
def _calculate_index(self, i): def _calculate_index(self, i):
return i * self.entry_size return i * self.entry_size
def __delitem__(self, i): def __delitem__(self, i):
packed = _pack(sidedata_format, sidedata_offset, sidedata_length) packed = _pack(sidedata_format, sidedata_offset, sidedata_length)
old = self._extra[i - self._lgt] old = self._extra[i - self._lgt]
new = old[:64] + packed + old[64 + packed_size :] new = old[:64] + packed + old[64 + packed_size :]
self._extra[i - self._lgt] = new self._extra[i - self._lgt] = new
else: else:
msg = b"cannot rewrite entries outside of this transaction" msg = b"cannot rewrite entries outside of this transaction"
raise KeyError(msg) raise KeyError(msg)
def entry_binary(self, rev, header):
"""return the raw binary string representing a revision"""
entry = self[rev]
p = revlog_constants.INDEX_ENTRY_V2.pack(*entry)
if rev == 0:
v_fmt = revlog_constants.INDEX_HEADER
v_bin = v_fmt.pack(header)
p = v_bin + p[v_fmt.size :]
return p
class IndexObject2(Index2Mixin, IndexObject): class IndexObject2(Index2Mixin, IndexObject):
pass pass
class InlinedIndexObject2(Index2Mixin, InlinedIndexObject): class InlinedIndexObject2(Index2Mixin, InlinedIndexObject):
def _inline_scan(self, lgt): def _inline_scan(self, lgt):
sidedata_length_pos = 72 sidedata_length_pos = 72

mercurial/revlog.py

def clearcaches(self): def clearcaches(self):
self.__dict__.pop('_nodemap', None) self.__dict__.pop('_nodemap', None)
def __getitem__(self, i): def __getitem__(self, i):
if i == -1: if i == -1:
return (0, 0, 0, -1, -1, -1, -1, sha1nodeconstants.nullid) return (0, 0, 0, -1, -1, -1, -1, sha1nodeconstants.nullid)
return list.__getitem__(self, i) return list.__getitem__(self, i)
def entry_binary(self, rev, header):
"""return the raw binary string representing a revision"""
entry = self[rev]
if gettype(entry[0]):
raise error.RevlogError(
_(b'index entry flags need revlog version 1')
)
e2 = (
getoffset(entry[0]),
entry[1],
entry[3],
entry[4],
self[entry[5]][7],
self[entry[6]][7],
entry[7],
)
return INDEX_ENTRY_V0.pack(*e2)
class revlogoldio(object): class revlogoldio(object):
def parseindex(self, data, inline): def parseindex(self, data, inline):
s = INDEX_ENTRY_V0.size s = INDEX_ENTRY_V0.size
index = [] index = []
nodemap = nodemaputil.NodeMap({sha1nodeconstants.nullid: nullrev}) nodemap = nodemaputil.NodeMap({sha1nodeconstants.nullid: nullrev})
n = off = 0 n = off = 0
l = len(data) l = len(data)
) )
index.append(e2) index.append(e2)
nodemap[e[6]] = n nodemap[e[6]] = n
n += 1 n += 1
index = revlogoldindex(index) index = revlogoldindex(index)
return index, None return index, None
def packentry(self, entry, node, version, rev):
"""return the binary representation of an entry
entry: a tuple containing all the values (see index.__getitem__)
node: a callback to convert a revision to nodeid
version: the changelog version
rev: the revision number
"""
if gettype(entry[0]):
raise error.RevlogError(
_(b'index entry flags need revlog version 1')
)
e2 = (
getoffset(entry[0]),
entry[1],
entry[3],
entry[4],
node(entry[5]),
node(entry[6]),
entry[7],
)
return INDEX_ENTRY_V0.pack(*e2)
# corresponds to uncompressed length of indexformatng (2 gigs, 4-byte # corresponds to uncompressed length of indexformatng (2 gigs, 4-byte
# signed integer) # signed integer)
_maxentrysize = 0x7FFFFFFF _maxentrysize = 0x7FFFFFFF
class revlogio(object): class revlogio(object):
def parseindex(self, data, inline): def parseindex(self, data, inline):
# call the C implementation to parse the index data # call the C implementation to parse the index data
index, cache = parsers.parse_index2(data, inline) index, cache = parsers.parse_index2(data, inline)
return index, cache return index, cache
def packentry(self, entry, node, version, rev):
p = INDEX_ENTRY_V1.pack(*entry)
if rev == 0:
p = INDEX_HEADER.pack(version) + p[4:]
return p
class revlogv2io(object): class revlogv2io(object):
def parseindex(self, data, inline): def parseindex(self, data, inline):
index, cache = parsers.parse_index2(data, inline, revlogv2=True) index, cache = parsers.parse_index2(data, inline, revlogv2=True)
return index, cache return index, cache
def packentry(self, entry, node, version, rev):
p = INDEX_ENTRY_V2.pack(*entry)
if rev == 0:
p = INDEX_HEADER.pack(version) + p[4:]
return p
NodemapRevlogIO = None NodemapRevlogIO = None
if util.safehasattr(parsers, 'parse_index_devel_nodemap'): if util.safehasattr(parsers, 'parse_index_devel_nodemap'):
class NodemapRevlogIO(revlogio): class NodemapRevlogIO(revlogio):
"""A debug oriented IO class that return a PersistentNodeMapIndexObject """A debug oriented IO class that return a PersistentNodeMapIndexObject
if troffset <= self.start(r): if troffset <= self.start(r):
trindex = r trindex = r
with self._indexfp(b'w') as fp: with self._indexfp(b'w') as fp:
self.version &= ~FLAG_INLINE_DATA self.version &= ~FLAG_INLINE_DATA
self._inline = False self._inline = False
io = self._io io = self._io
for i in self: for i in self:
e = io.packentry(self.index[i], self.node, self.version, i) e = self.index.entry_binary(i, self.version)
fp.write(e) fp.write(e)
# the temp file replace the real index when we exit the context # the temp file replace the real index when we exit the context
# manager # manager
tr.replace(self.indexfile, trindex * self.index.entry_size) tr.replace(self.indexfile, trindex * self.index.entry_size)
nodemaputil.setup_persistent_nodemap(tr, self) nodemaputil.setup_persistent_nodemap(tr, self)
self._chunkclear() self._chunkclear()
sidedata_offset, sidedata_offset,
len(serialized_sidedata), len(serialized_sidedata),
) )
if self.version & 0xFFFF != REVLOGV2: if self.version & 0xFFFF != REVLOGV2:
e = e[:8] e = e[:8]
self.index.append(e) self.index.append(e)
entry = self._io.packentry(e, self.node, self.version, curr) entry = self.index.entry_binary(curr, self.version)
self._writeentry( self._writeentry(
transaction, transaction,
ifh, ifh,
dfh, dfh,
entry, entry,
deltainfo.data, deltainfo.data,
link, link,
offset, offset,
current_offset += len(serialized_sidedata) current_offset += len(serialized_sidedata)
# rewrite the new index entries # rewrite the new index entries
with self._indexfp(b'w+') as fp: with self._indexfp(b'w+') as fp:
fp.seek(startrev * self.index.entry_size) fp.seek(startrev * self.index.entry_size)
for i, entry in enumerate(new_entries): for i, entry in enumerate(new_entries):
rev = startrev + i rev = startrev + i
self.index.replace_sidedata_info(rev, entry[8], entry[9]) self.index.replace_sidedata_info(rev, entry[8], entry[9])
packed = self._io.packentry(entry, self.node, self.version, rev) packed = self.index.entry_binary(rev, self.version)
fp.write(packed) fp.write(packed)

rust/hg-cpython/src/revlog.rs

/// It is Python's responsibility to call `update_nodemap_data` again. /// It is Python's responsibility to call `update_nodemap_data` again.
def clearcaches(&self, *args, **kw) -> PyResult<PyObject> { def clearcaches(&self, *args, **kw) -> PyResult<PyObject> {
self.nt(py).borrow_mut().take(); self.nt(py).borrow_mut().take();
self.docket(py).borrow_mut().take(); self.docket(py).borrow_mut().take();
self.mmap(py).borrow_mut().take(); self.mmap(py).borrow_mut().take();
self.call_cindex(py, "clearcaches", args, kw) self.call_cindex(py, "clearcaches", args, kw)
} }
/// return the raw binary string representing a revision
def entry_binary(&self, *args, **kw) -> PyResult<PyObject> {
self.call_cindex(py, "entry_binary", args, kw)
}
/// get an index entry /// get an index entry
def get(&self, *args, **kw) -> PyResult<PyObject> { def get(&self, *args, **kw) -> PyResult<PyObject> {
self.call_cindex(py, "get", args, kw) self.call_cindex(py, "get", args, kw)
} }
/// compute phases /// compute phases
def computephasesmapsets(&self, *args, **kw) -> PyResult<PyObject> { def computephasesmapsets(&self, *args, **kw) -> PyResult<PyObject> {
self.call_cindex(py, "computephasesmapsets", args, kw) self.call_cindex(py, "computephasesmapsets", args, kw)