( )⚙ D10569 revlog: always "append" full size tuple

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10569

revlog: always "append" full size tuple
ClosedPublic

Authored by marmoute on May 3 2021, 7:47 AM.

Details

Reviewers
indygreg
Alphare
Group Reviewers
hg-reviewers
Commits
rHG47ffc754989a: revlog: always "append" full size tuple
Summary

Same reasoning as the previous patch.

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

marmoute created this revision.May 3 2021, 7:47 AM
baymax updated this revision to Diff 27484.May 3 2021, 9:43 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

Alphare accepted this revision.May 7 2021, 8:59 AM
This revision is now accepted and ready to land.May 7 2021, 8:59 AM
marmoute added a commit: rHG47ffc754989a: revlog: always "append" full size tuple.May 11 2021, 12:43 PM
Closed by commit rHG47ffc754989a: revlog: always "append" full size tuple (authored by marmoute). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 27346

mercurial/bundlerepo.py

) )
if not self.index.has_node(deltabase): if not self.index.has_node(deltabase):
raise LookupError( raise LookupError(
deltabase, self.indexfile, _(b'unknown delta base') deltabase, self.indexfile, _(b'unknown delta base')
) )
baserev = self.rev(deltabase) baserev = self.rev(deltabase)
# start, size, full unc. size, base (unused), link, p1, p2, node # start, size, full unc. size, base (unused), link, p1, p2, node, sidedata_offset (unused), sidedata_size (unused)
e = ( e = (
revlog.offset_type(start, flags), revlog.offset_type(start, flags),
size, size,
-1, -1,
baserev, baserev,
linkrev, linkrev,
self.rev(p1), self.rev(p1),
self.rev(p2), self.rev(p2),
node, node,
0,
0,
) )
self.index.append(e) self.index.append(e)
self.bundlerevs.add(n) self.bundlerevs.add(n)
n += 1 n += 1
def _chunk(self, rev, df=None): def _chunk(self, rev, df=None):
# Warning: in case of bundle, the diff is against what we stored as # Warning: in case of bundle, the diff is against what we stored as
# delta base, not against rev - 1 # delta base, not against rev - 1

mercurial/cext/revlog.c

static const char nullid[32] = {0}; static const char nullid[32] = {0};
static const Py_ssize_t nullrev = -1; static const Py_ssize_t nullrev = -1;
static Py_ssize_t inline_scan(indexObject *self, const char **offsets); static Py_ssize_t inline_scan(indexObject *self, const char **offsets);
static int index_find_node(indexObject *self, const char *node); static int index_find_node(indexObject *self, const char *node);
#if LONG_MAX == 0x7fffffffL #if LONG_MAX == 0x7fffffffL
static const char *const v1_tuple_format = PY23("Kiiiiiis#", "Kiiiiiiy#"); static const char *const tuple_format = PY23("Kiiiiiis#Ki", "Kiiiiiiy#Ki");
static const char *const v2_tuple_format = PY23("Kiiiiiis#Ki", "Kiiiiiiy#Ki");
#else #else
static const char *const v1_tuple_format = PY23("kiiiiiis#", "kiiiiiiy#"); static const char *const tuple_format = PY23("kiiiiiis#ki", "kiiiiiiy#ki");
static const char *const v2_tuple_format = PY23("kiiiiiis#ki", "kiiiiiiy#ki");
#endif #endif
/* A RevlogNG v1 index entry is 64 bytes long. */ /* A RevlogNG v1 index entry is 64 bytes long. */
static const long v1_entry_size = 64; static const long v1_entry_size = 64;
/* A Revlogv2 index entry is 96 bytes long. */ /* A Revlogv2 index entry is 96 bytes long. */
static const long v2_entry_size = 96; static const long v2_entry_size = 96;
if (self->entry_size == v1_entry_size) { if (self->entry_size == v1_entry_size) {
sidedata_offset = 0; sidedata_offset = 0;
sidedata_comp_len = 0; sidedata_comp_len = 0;
} else { } else {
sidedata_offset = getbe64(data + 64); sidedata_offset = getbe64(data + 64);
sidedata_comp_len = getbe32(data + 72); sidedata_comp_len = getbe32(data + 72);
} }
return Py_BuildValue(v2_tuple_format, offset_flags, comp_len, return Py_BuildValue(tuple_format, offset_flags, comp_len, uncomp_len,
uncomp_len, base_rev, link_rev, parent_1, parent_2, base_rev, link_rev, parent_1, parent_2, c_node_id,
c_node_id, self->nodelen, sidedata_offset, self->nodelen, sidedata_offset, sidedata_comp_len);
sidedata_comp_len);
} }
/* /*
* Pack header information in binary * Pack header information in binary
*/ */
static PyObject *index_pack_header(indexObject *self, PyObject *args) static PyObject *index_pack_header(indexObject *self, PyObject *args)
{ {
int header; int header;
char out[4]; char out[4];
static PyObject *index_append(indexObject *self, PyObject *obj) static PyObject *index_append(indexObject *self, PyObject *obj)
{ {
uint64_t offset_flags, sidedata_offset; uint64_t offset_flags, sidedata_offset;
int rev, comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2; int rev, comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2;
Py_ssize_t c_node_id_len, sidedata_comp_len; Py_ssize_t c_node_id_len, sidedata_comp_len;
const char *c_node_id; const char *c_node_id;
char *data; char *data;
if (self->entry_size == v1_entry_size) { if (!PyArg_ParseTuple(obj, tuple_format, &offset_flags, &comp_len,
if (!PyArg_ParseTuple(obj, v1_tuple_format, &offset_flags, &uncomp_len, &base_rev, &link_rev, &parent_1,
&comp_len, &uncomp_len, &base_rev, &parent_2, &c_node_id, &c_node_id_len,
&link_rev, &parent_1, &parent_2,
&c_node_id, &c_node_id_len)) {
PyErr_SetString(PyExc_TypeError, "8-tuple required");
return NULL;
}
} else {
if (!PyArg_ParseTuple(obj, v2_tuple_format, &offset_flags,
&comp_len, &uncomp_len, &base_rev,
&link_rev, &parent_1, &parent_2,
&c_node_id, &c_node_id_len,
&sidedata_offset, &sidedata_comp_len)) { &sidedata_offset, &sidedata_comp_len)) {
PyErr_SetString(PyExc_TypeError, "10-tuple required"); PyErr_SetString(PyExc_TypeError, "10-tuple required");
return NULL; return NULL;
} }
}
if (c_node_id_len != self->nodelen) { if (c_node_id_len != self->nodelen) {
PyErr_SetString(PyExc_TypeError, "invalid node"); PyErr_SetString(PyExc_TypeError, "invalid node");
return NULL; return NULL;
} }
if (self->new_length == self->added_length) { if (self->new_length == self->added_length) {
size_t new_added_length = size_t new_added_length =
putbe32(uncomp_len, data + 12); putbe32(uncomp_len, data + 12);
putbe32(base_rev, data + 16); putbe32(base_rev, data + 16);
putbe32(link_rev, data + 20); putbe32(link_rev, data + 20);
putbe32(parent_1, data + 24); putbe32(parent_1, data + 24);
putbe32(parent_2, data + 28); putbe32(parent_2, data + 28);
memcpy(data + 32, c_node_id, c_node_id_len); memcpy(data + 32, c_node_id, c_node_id_len);
/* Padding since SHA-1 is only 20 bytes for now */ /* Padding since SHA-1 is only 20 bytes for now */
memset(data + 32 + c_node_id_len, 0, 32 - c_node_id_len); memset(data + 32 + c_node_id_len, 0, 32 - c_node_id_len);
if (self->entry_size != v1_entry_size) { if (self->format_version == format_v2) {
putbe64(sidedata_offset, data + 64); putbe64(sidedata_offset, data + 64);
putbe32(sidedata_comp_len, data + 72); putbe32(sidedata_comp_len, data + 72);
/* Padding for 96 bytes alignment */ /* Padding for 96 bytes alignment */
memset(data + 76, 0, self->entry_size - 76); memset(data + 76, 0, self->entry_size - 76);
} }
if (self->ntinitialized) if (self->ntinitialized)
nt_insert(&self->nt, c_node_id, rev); nt_insert(&self->nt, c_node_id, rev);

mercurial/pure/parsers.py

self.__dict__.pop('_nodemap', None) self.__dict__.pop('_nodemap', None)
def __len__(self): def __len__(self):
return self._lgt + len(self._extra) return self._lgt + len(self._extra)
def append(self, tup): def append(self, tup):
if '_nodemap' in vars(self): if '_nodemap' in vars(self):
self._nodemap[tup[7]] = len(self) self._nodemap[tup[7]] = len(self)
data = self.index_format.pack(*tup) data = self._pack_entry(tup)
self._extra.append(data) self._extra.append(data)
def _pack_entry(self, entry):
assert entry[8] == 0
assert entry[9] == 0
return self.index_format.pack(*entry[:8])
def _check_index(self, i): def _check_index(self, i):
if not isinstance(i, int): if not isinstance(i, int):
raise TypeError(b"expecting int indexes") raise TypeError(b"expecting int indexes")
if i < 0 or i >= len(self): if i < 0 or i >= len(self):
raise IndexError raise IndexError
def __getitem__(self, i): def __getitem__(self, i):
if i == -1: if i == -1:
self._extra[i - self._lgt] = new self._extra[i - self._lgt] = new
else: else:
msg = b"cannot rewrite entries outside of this transaction" msg = b"cannot rewrite entries outside of this transaction"
raise KeyError(msg) raise KeyError(msg)
def _unpack_entry(self, data): def _unpack_entry(self, data):
return self.index_format.unpack(data) return self.index_format.unpack(data)
def _pack_entry(self, entry):
return self.index_format.pack(*entry)
def entry_binary(self, rev): def entry_binary(self, rev):
"""return the raw binary string representing a revision""" """return the raw binary string representing a revision"""
entry = self[rev] entry = self[rev]
p = revlog_constants.INDEX_ENTRY_V2.pack(*entry) p = revlog_constants.INDEX_ENTRY_V2.pack(*entry)
if rev == 0: if rev == 0:
p = p[revlog_constants.INDEX_HEADER.size :] p = p[revlog_constants.INDEX_HEADER.size :]
return p return p

mercurial/revlog.py

link, link,
p1r, p1r,
p2r, p2r,
node, node,
sidedata_offset, sidedata_offset,
len(serialized_sidedata), len(serialized_sidedata),
) )
if self._format_version != REVLOGV2:
e = e[:8]
self.index.append(e) self.index.append(e)
entry = self.index.entry_binary(curr) entry = self.index.entry_binary(curr)
if curr == 0: if curr == 0:
header = self._format_flags | self._format_version header = self._format_flags | self._format_version
header = self.index.pack_header(header) header = self.index.pack_header(header)
entry = header + entry entry = header + entry
self._writeentry( self._writeentry(
transaction, transaction,

mercurial/unionrepo.py

flags, flags,
-1, -1,
rsize, rsize,
base, base,
link, link,
self.rev(p1node), self.rev(p1node),
self.rev(p2node), self.rev(p2node),
node, node,
0, # sidedata offset
0, # sidedata size
) )
self.index.append(e) self.index.append(e)
self.bundlerevs.add(n) self.bundlerevs.add(n)
n += 1 n += 1
def _chunk(self, rev): def _chunk(self, rev):
if rev <= self.repotiprev: if rev <= self.repotiprev:
return revlog.revlog._chunk(self, rev) return revlog.revlog._chunk(self, rev)

tests/test-parseindex2.py

return b'\xff\xff\xff\xff' return b'\xff\xff\xff\xff'
else: else:
return bin('%08x' % rev) return bin('%08x' % rev)
def appendrev(p1, p2=nullrev): def appendrev(p1, p2=nullrev):
# node won't matter for this test, let's just make sure # node won't matter for this test, let's just make sure
# they don't collide. Other data don't matter either. # they don't collide. Other data don't matter either.
node = hexrev(p1) + hexrev(p2) + b'.' * 12 node = hexrev(p1) + hexrev(p2) + b'.' * 12
index.append((0, 0, 12, 1, 34, p1, p2, node)) index.append((0, 0, 12, 1, 34, p1, p2, node, 0, 0))
appendrev(4) appendrev(4)
appendrev(5) appendrev(5)
appendrev(6) appendrev(6)
self.assertEqual(len(index), 7) self.assertEqual(len(index), 7)
del index[1:-1] del index[1:-1]