This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10510

revlog: have an explicit "pack_header" method
ClosedPublic

Authored by marmoute on Apr 23 2021, 10:30 AM.

Details

Reviewers
indygreg
Alphare
Group Reviewers
hg-reviewers
Commits
rHGd57386e5c80e: revlog: have an explicit "pack_header" method
rHGa955a27c62bb: revlog: have an explicit "pack_header" method
Summary

Having to pass the version header when retrieving the binary version of every single entry is a bit silly. So we extract that special logic in its own method. This also prepare the move to newer revlog format, not storing the header within an actual entry…

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

Diff 27276

mercurial/cext/revlog.c

return Py_BuildValue(v2_tuple_format, offset_flags, comp_len, return Py_BuildValue(v2_tuple_format, offset_flags, comp_len,
uncomp_len, base_rev, link_rev, parent_1, uncomp_len, base_rev, link_rev, parent_1,
parent_2, c_node_id, self->nodelen, parent_2, c_node_id, self->nodelen,
sidedata_offset, sidedata_comp_len); sidedata_offset, sidedata_comp_len);
} }
} }
/* /*
* Pack header information in binary
*/
static PyObject *index_pack_header(indexObject *self, PyObject *args)
{
int header;
char out[4];
if (!PyArg_ParseTuple(args, "I", &header)) {
return NULL;
}
putbe32(header, out);
return PyBytes_FromStringAndSize(out, 4);
}
/*
* Return the raw binary string representing a revision * Return the raw binary string representing a revision
*/ */
static PyObject *index_entry_binary(indexObject *self, PyObject *args) static PyObject *index_entry_binary(indexObject *self, PyObject *value)
{ {
long rev; long rev;
int header;
const char *data; const char *data;
char entry[v2_hdrsize];
Py_ssize_t length = index_length(self); Py_ssize_t length = index_length(self);
if (!PyArg_ParseTuple(args, "lI", &rev, &header)) { if (!pylong_to_long(value, &rev)) {
return NULL; return NULL;
} }
if (rev < 0 || rev >= length) { if (rev < 0 || rev >= length) {
PyErr_Format(PyExc_ValueError, "revlog index out of range: %ld", PyErr_Format(PyExc_ValueError, "revlog index out of range: %ld",
rev); rev);
return NULL; return NULL;
}; };
data = index_deref(self, rev); data = index_deref(self, rev);
if (data == NULL) if (data == NULL)
return NULL; return NULL;
if (rev == 0) { if (rev == 0) {
/* put the header at the start of the first entry */ /* the header is eating the start of the first entry */
memcpy(entry, data, self->hdrsize); return PyBytes_FromStringAndSize(data + 4, self->hdrsize - 4);
putbe32(header, entry);
return PyBytes_FromStringAndSize(entry, self->hdrsize);
} }
return PyBytes_FromStringAndSize(data, self->hdrsize); return PyBytes_FromStringAndSize(data, self->hdrsize);
} }
/* /*
* Return the hash of node corresponding to the given rev. * Return the hash of node corresponding to the given rev.
*/ */
static const char *index_node(indexObject *self, Py_ssize_t pos) static const char *index_node(indexObject *self, Py_ssize_t pos)
{"slicechunktodensity", (PyCFunction)index_slicechunktodensity, {"slicechunktodensity", (PyCFunction)index_slicechunktodensity,
METH_VARARGS, "determine revisions with deltas to reconstruct fulltext"}, METH_VARARGS, "determine revisions with deltas to reconstruct fulltext"},
{"append", (PyCFunction)index_append, METH_O, "append an index entry"}, {"append", (PyCFunction)index_append, METH_O, "append an index entry"},
{"partialmatch", (PyCFunction)index_partialmatch, METH_VARARGS, {"partialmatch", (PyCFunction)index_partialmatch, METH_VARARGS,
"match a potentially ambiguous node ID"}, "match a potentially ambiguous node ID"},
{"shortest", (PyCFunction)index_shortest, METH_VARARGS, {"shortest", (PyCFunction)index_shortest, METH_VARARGS,
"find length of shortest hex nodeid of a binary ID"}, "find length of shortest hex nodeid of a binary ID"},
{"stats", (PyCFunction)index_stats, METH_NOARGS, "stats for the index"}, {"stats", (PyCFunction)index_stats, METH_NOARGS, "stats for the index"},
{"entry_binary", (PyCFunction)index_entry_binary, METH_VARARGS, {"entry_binary", (PyCFunction)index_entry_binary, METH_O,
"return an entry in binary form"}, "return an entry in binary form"},
{"pack_header", (PyCFunction)index_pack_header, METH_VARARGS,
"pack the revlog header information into binary"},
{NULL} /* Sentinel */ {NULL} /* Sentinel */
}; };
static PyGetSetDef index_getset[] = { static PyGetSetDef index_getset[] = {
{"nodemap", (getter)index_nodemap, NULL, "nodemap", NULL}, {"nodemap", (getter)index_nodemap, NULL, "nodemap", NULL},
{NULL} /* Sentinel */ {NULL} /* Sentinel */
}; };

mercurial/pure/parsers.py

else: else:
index = self._calculate_index(i) index = self._calculate_index(i)
data = self._data[index : index + self.entry_size] data = self._data[index : index + self.entry_size]
r = self.index_format.unpack(data) r = self.index_format.unpack(data)
if self._lgt and i == 0: if self._lgt and i == 0:
r = (offset_type(0, gettype(r[0])),) + r[1:] r = (offset_type(0, gettype(r[0])),) + r[1:]
return r return r
def entry_binary(self, rev, header): def pack_header(self, header):
"""pack header information as binary"""
v_fmt = revlog_constants.INDEX_HEADER
return v_fmt.pack(header)
def entry_binary(self, rev):
"""return the raw binary string representing a revision""" """return the raw binary string representing a revision"""
entry = self[rev] entry = self[rev]
p = revlog_constants.INDEX_ENTRY_V1.pack(*entry) p = revlog_constants.INDEX_ENTRY_V1.pack(*entry)
if rev == 0: if rev == 0:
v_fmt = revlog_constants.INDEX_HEADER p = p[revlog_constants.INDEX_HEADER.size :]
v_bin = v_fmt.pack(header)
p = v_bin + p[v_fmt.size :]
return p return p
class IndexObject(BaseIndexObject): class IndexObject(BaseIndexObject):
def __init__(self, data): def __init__(self, data):
assert len(data) % self.entry_size == 0, ( assert len(data) % self.entry_size == 0, (
len(data), len(data),
self.entry_size, self.entry_size,
packed = _pack(sidedata_format, sidedata_offset, sidedata_length) packed = _pack(sidedata_format, sidedata_offset, sidedata_length)
old = self._extra[i - self._lgt] old = self._extra[i - self._lgt]
new = old[:64] + packed + old[64 + packed_size :] new = old[:64] + packed + old[64 + packed_size :]
self._extra[i - self._lgt] = new self._extra[i - self._lgt] = new
else: else:
msg = b"cannot rewrite entries outside of this transaction" msg = b"cannot rewrite entries outside of this transaction"
raise KeyError(msg) raise KeyError(msg)
def entry_binary(self, rev, header): def entry_binary(self, rev):
"""return the raw binary string representing a revision""" """return the raw binary string representing a revision"""
entry = self[rev] entry = self[rev]
p = revlog_constants.INDEX_ENTRY_V2.pack(*entry) p = revlog_constants.INDEX_ENTRY_V2.pack(*entry)
if rev == 0: if rev == 0:
v_fmt = revlog_constants.INDEX_HEADER p = p[revlog_constants.INDEX_HEADER.size :]
v_bin = v_fmt.pack(header)
p = v_bin + p[v_fmt.size :]
return p return p
class IndexObject2(Index2Mixin, IndexObject): class IndexObject2(Index2Mixin, IndexObject):
pass pass
class InlinedIndexObject2(Index2Mixin, InlinedIndexObject): class InlinedIndexObject2(Index2Mixin, InlinedIndexObject):

mercurial/revlog.py

def clearcaches(self): def clearcaches(self):
self.__dict__.pop('_nodemap', None) self.__dict__.pop('_nodemap', None)
def __getitem__(self, i): def __getitem__(self, i):
if i == -1: if i == -1:
return (0, 0, 0, -1, -1, -1, -1, sha1nodeconstants.nullid) return (0, 0, 0, -1, -1, -1, -1, sha1nodeconstants.nullid)
return list.__getitem__(self, i) return list.__getitem__(self, i)
def entry_binary(self, rev, header): def entry_binary(self, rev):
"""return the raw binary string representing a revision""" """return the raw binary string representing a revision"""
entry = self[rev] entry = self[rev]
if gettype(entry[0]): if gettype(entry[0]):
raise error.RevlogError( raise error.RevlogError(
_(b'index entry flags need revlog version 1') _(b'index entry flags need revlog version 1')
) )
e2 = ( e2 = (
getoffset(entry[0]), getoffset(entry[0]),
entry[1], entry[1],
entry[3], entry[3],
entry[4], entry[4],
self[entry[5]][7], self[entry[5]][7],
self[entry[6]][7], self[entry[6]][7],
entry[7], entry[7],
) )
return INDEX_ENTRY_V0.pack(*e2) return INDEX_ENTRY_V0.pack(*e2)
def pack_header(self, header):
"""Pack header information in binary"""
return b''
def parse_index_v0(data, inline): def parse_index_v0(data, inline):
s = INDEX_ENTRY_V0.size s = INDEX_ENTRY_V0.size
index = [] index = []
nodemap = nodemaputil.NodeMap({sha1nodeconstants.nullid: nullrev}) nodemap = nodemaputil.NodeMap({sha1nodeconstants.nullid: nullrev})
n = off = 0 n = off = 0
l = len(data) l = len(data)
while off + s <= l: while off + s <= l:
dfh.write(self._getsegmentforrevs(r, r, df=ifh)[1]) dfh.write(self._getsegmentforrevs(r, r, df=ifh)[1])
if troffset <= self.start(r): if troffset <= self.start(r):
trindex = r trindex = r
with self._indexfp(b'w') as fp: with self._indexfp(b'w') as fp:
self.version &= ~FLAG_INLINE_DATA self.version &= ~FLAG_INLINE_DATA
self._inline = False self._inline = False
for i in self: for i in self:
e = self.index.entry_binary(i, self.version) e = self.index.entry_binary(i)
if i == 0:
header = self.index.pack_header(self.version)
e = header + e
fp.write(e) fp.write(e)
# the temp file replace the real index when we exit the context # the temp file replace the real index when we exit the context
# manager # manager
tr.replace(self.indexfile, trindex * self.index.entry_size) tr.replace(self.indexfile, trindex * self.index.entry_size)
nodemaputil.setup_persistent_nodemap(tr, self) nodemaputil.setup_persistent_nodemap(tr, self)
self._chunkclear() self._chunkclear()
sidedata_offset, sidedata_offset,
len(serialized_sidedata), len(serialized_sidedata),
) )
if self.version & 0xFFFF != REVLOGV2: if self.version & 0xFFFF != REVLOGV2:
e = e[:8] e = e[:8]
self.index.append(e) self.index.append(e)
entry = self.index.entry_binary(curr, self.version) entry = self.index.entry_binary(curr)
if curr == 0:
header = self.index.pack_header(self.version)
entry = header + entry
self._writeentry( self._writeentry(
transaction, transaction,
ifh, ifh,
dfh, dfh,
entry, entry,
deltainfo.data, deltainfo.data,
link, link,
offset, offset,
current_offset += len(serialized_sidedata) current_offset += len(serialized_sidedata)
# rewrite the new index entries # rewrite the new index entries
with self._indexfp(b'w+') as fp: with self._indexfp(b'w+') as fp:
fp.seek(startrev * self.index.entry_size) fp.seek(startrev * self.index.entry_size)
for i, entry in enumerate(new_entries): for i, entry in enumerate(new_entries):
rev = startrev + i rev = startrev + i
self.index.replace_sidedata_info(rev, entry[8], entry[9]) self.index.replace_sidedata_info(rev, entry[8], entry[9])
packed = self.index.entry_binary(rev, self.version) packed = self.index.entry_binary(rev)
if rev == 0:
header = self.index.pack_header(self.version)
packed = header + packed
fp.write(packed) fp.write(packed)

rust/hg-cpython/src/revlog.rs

self.call_cindex(py, "clearcaches", args, kw) self.call_cindex(py, "clearcaches", args, kw)
} }
/// return the raw binary string representing a revision /// return the raw binary string representing a revision
def entry_binary(&self, *args, **kw) -> PyResult<PyObject> { def entry_binary(&self, *args, **kw) -> PyResult<PyObject> {
self.call_cindex(py, "entry_binary", args, kw) self.call_cindex(py, "entry_binary", args, kw)
} }
/// return a binary packed version of the header
def pack_header(&self, *args, **kw) -> PyResult<PyObject> {
self.call_cindex(py, "pack_header", args, kw)
}
/// get an index entry /// get an index entry
def get(&self, *args, **kw) -> PyResult<PyObject> { def get(&self, *args, **kw) -> PyResult<PyObject> {
self.call_cindex(py, "get", args, kw) self.call_cindex(py, "get", args, kw)
} }
/// compute phases /// compute phases
def computephasesmapsets(&self, *args, **kw) -> PyResult<PyObject> { def computephasesmapsets(&self, *args, **kw) -> PyResult<PyObject> {
self.call_cindex(py, "computephasesmapsets", args, kw) self.call_cindex(py, "computephasesmapsets", args, kw)