This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10568

revlog: make the index always return the same tuple
ClosedPublic

Authored by marmoute on May 3 2021, 7:47 AM.

Details

Reviewers
Alphare
Group Reviewers
hg-reviewers
Commits
rHG4292bed8da7c: revlog: make the index always return the same tuple
Summary

It is simpler to manage the diferrence in on disk format in the internal index
code itself and lets the rest of the code always handle the same object.

This will become even more important when the data we store will be entirely
different (for example the changelog does not need the "linkrev" field.

We start with item reading, we will deal with item writing in the next
changesets.

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

marmoute created this revision.May 3 2021, 7:47 AM
baymax updated this revision to Diff 27483.May 3 2021, 9:43 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

Alphare accepted this revision.May 7 2021, 8:58 AM
This revision is now accepted and ready to land.May 7 2021, 8:58 AM
marmoute added a commit: rHG4292bed8da7c: revlog: make the index always return the same tuple.May 11 2021, 12:43 PM
Closed by commit rHG4292bed8da7c: revlog: make the index always return the same tuple (authored by marmoute). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 27483

mercurial/cext/revlog.c

uncomp_len = getbe32(data + 12); uncomp_len = getbe32(data + 12);
base_rev = getbe32(data + 16); base_rev = getbe32(data + 16);
link_rev = getbe32(data + 20); link_rev = getbe32(data + 20);
parent_1 = getbe32(data + 24); parent_1 = getbe32(data + 24);
parent_2 = getbe32(data + 28); parent_2 = getbe32(data + 28);
c_node_id = data + 32; c_node_id = data + 32;
if (self->entry_size == v1_entry_size) { if (self->entry_size == v1_entry_size) {
return Py_BuildValue(v1_tuple_format, offset_flags, comp_len, sidedata_offset = 0;
uncomp_len, base_rev, link_rev, parent_1, sidedata_comp_len = 0;
parent_2, c_node_id, self->nodelen);
} else { } else {
sidedata_offset = getbe64(data + 64); sidedata_offset = getbe64(data + 64);
sidedata_comp_len = getbe32(data + 72); sidedata_comp_len = getbe32(data + 72);
}
return Py_BuildValue(v2_tuple_format, offset_flags, comp_len, return Py_BuildValue(v2_tuple_format, offset_flags, comp_len,
uncomp_len, base_rev, link_rev, parent_1, uncomp_len, base_rev, link_rev, parent_1, parent_2,
parent_2, c_node_id, self->nodelen, c_node_id, self->nodelen, sidedata_offset,
sidedata_offset, sidedata_comp_len); sidedata_comp_len);
}
} }
/* /*
* Pack header information in binary * Pack header information in binary
*/ */
static PyObject *index_pack_header(indexObject *self, PyObject *args) static PyObject *index_pack_header(indexObject *self, PyObject *args)
{ {
int header; int header;
char out[4]; char out[4];
if (revlogv2 && PyObject_IsTrue(revlogv2)) { if (revlogv2 && PyObject_IsTrue(revlogv2)) {
self->format_version = format_v2; self->format_version = format_v2;
self->entry_size = v2_entry_size; self->entry_size = v2_entry_size;
} else { } else {
self->format_version = format_v1; self->format_version = format_v1;
self->entry_size = v1_entry_size; self->entry_size = v1_entry_size;
} }
if (self->format_version == format_v1) {
self->nullentry = self->nullentry =
Py_BuildValue(PY23("iiiiiiis#", "iiiiiiiy#"), 0, 0, 0, -1, Py_BuildValue(PY23("iiiiiiis#ii", "iiiiiiiy#ii"), 0, 0, 0, -1, -1,
-1, -1, -1, nullid, self->nodelen); -1, -1, nullid, self->nodelen, 0, 0);
} else {
self->nullentry =
Py_BuildValue(PY23("iiiiiiis#ii", "iiiiiiiy#ii"), 0, 0, 0,
-1, -1, -1, -1, nullid, self->nodelen, 0, 0);
}
if (!self->nullentry) if (!self->nullentry)
return -1; return -1;
PyObject_GC_UnTrack(self->nullentry); PyObject_GC_UnTrack(self->nullentry);
if (PyObject_GetBuffer(data_obj, &self->buf, PyBUF_SIMPLE) == -1) if (PyObject_GetBuffer(data_obj, &self->buf, PyBUF_SIMPLE) == -1)
return -1; return -1;
size = self->buf.len; size = self->buf.len;

mercurial/pure/parsers.py

class BaseIndexObject(object): class BaseIndexObject(object):
# Format of an index entry according to Python's `struct` language # Format of an index entry according to Python's `struct` language
index_format = revlog_constants.INDEX_ENTRY_V1 index_format = revlog_constants.INDEX_ENTRY_V1
# Size of a C unsigned long long int, platform independent # Size of a C unsigned long long int, platform independent
big_int_size = struct.calcsize(b'>Q') big_int_size = struct.calcsize(b'>Q')
# Size of a C long int, platform independent # Size of a C long int, platform independent
int_size = struct.calcsize(b'>i') int_size = struct.calcsize(b'>i')
# An empty index entry, used as a default value to be overridden, or nullrev # An empty index entry, used as a default value to be overridden, or nullrev
null_item = (0, 0, 0, -1, -1, -1, -1, sha1nodeconstants.nullid) null_item = (0, 0, 0, -1, -1, -1, -1, sha1nodeconstants.nullid, 0, 0)
@util.propertycache @util.propertycache
def entry_size(self): def entry_size(self):
return self.index_format.size return self.index_format.size
@property @property
def nodemap(self): def nodemap(self):
msg = b"index.nodemap is deprecated, use index.[has_node|rev|get_rev]" msg = b"index.nodemap is deprecated, use index.[has_node|rev|get_rev]"
if i == -1: if i == -1:
return self.null_item return self.null_item
self._check_index(i) self._check_index(i)
if i >= self._lgt: if i >= self._lgt:
data = self._extra[i - self._lgt] data = self._extra[i - self._lgt]
else: else:
index = self._calculate_index(i) index = self._calculate_index(i)
data = self._data[index : index + self.entry_size] data = self._data[index : index + self.entry_size]
r = self.index_format.unpack(data) r = self._unpack_entry(data)
if self._lgt and i == 0: if self._lgt and i == 0:
r = (offset_type(0, gettype(r[0])),) + r[1:] r = (offset_type(0, gettype(r[0])),) + r[1:]
return r return r
def _unpack_entry(self, data):
r = self.index_format.unpack(data)
r = r + (0, 0)
return r
def pack_header(self, header): def pack_header(self, header):
"""pack header information as binary""" """pack header information as binary"""
v_fmt = revlog_constants.INDEX_HEADER v_fmt = revlog_constants.INDEX_HEADER
return v_fmt.pack(header) return v_fmt.pack(header)
def entry_binary(self, rev): def entry_binary(self, rev):
"""return the raw binary string representing a revision""" """return the raw binary string representing a revision"""
entry = self[rev] entry = self[rev]
p = revlog_constants.INDEX_ENTRY_V1.pack(*entry) p = revlog_constants.INDEX_ENTRY_V1.pack(*entry[:8])
if rev == 0: if rev == 0:
p = p[revlog_constants.INDEX_HEADER.size :] p = p[revlog_constants.INDEX_HEADER.size :]
return p return p
class IndexObject(BaseIndexObject): class IndexObject(BaseIndexObject):
def __init__(self, data): def __init__(self, data):
assert len(data) % self.entry_size == 0, ( assert len(data) % self.entry_size == 0, (
cls = IndexObject2 if revlogv2 else IndexObject cls = IndexObject2 if revlogv2 else IndexObject
return cls(data), None return cls(data), None
cls = InlinedIndexObject2 if revlogv2 else InlinedIndexObject cls = InlinedIndexObject2 if revlogv2 else InlinedIndexObject
return cls(data, inline), (0, data) return cls(data, inline), (0, data)
class Index2Mixin(object): class Index2Mixin(object):
index_format = revlog_constants.INDEX_ENTRY_V2 index_format = revlog_constants.INDEX_ENTRY_V2
null_item = (0, 0, 0, -1, -1, -1, -1, sha1nodeconstants.nullid, 0, 0)
def replace_sidedata_info( def replace_sidedata_info(
self, i, sidedata_offset, sidedata_length, offset_flags self, i, sidedata_offset, sidedata_length, offset_flags
): ):
""" """
Replace an existing index entry's sidedata offset and length with new Replace an existing index entry's sidedata offset and length with new
ones. ones.
This cannot be used outside of the context of sidedata rewriting, This cannot be used outside of the context of sidedata rewriting,
old = self._extra[i - self._lgt] old = self._extra[i - self._lgt]
offset_flags = struct.pack(b">Q", offset_flags) offset_flags = struct.pack(b">Q", offset_flags)
new = offset_flags + old[8:64] + packed + old[64 + packed_size :] new = offset_flags + old[8:64] + packed + old[64 + packed_size :]
self._extra[i - self._lgt] = new self._extra[i - self._lgt] = new
else: else:
msg = b"cannot rewrite entries outside of this transaction" msg = b"cannot rewrite entries outside of this transaction"
raise KeyError(msg) raise KeyError(msg)
def _unpack_entry(self, data):
return self.index_format.unpack(data)
def entry_binary(self, rev): def entry_binary(self, rev):
"""return the raw binary string representing a revision""" """return the raw binary string representing a revision"""
entry = self[rev] entry = self[rev]
p = revlog_constants.INDEX_ENTRY_V2.pack(*entry) p = revlog_constants.INDEX_ENTRY_V2.pack(*entry)
if rev == 0: if rev == 0:
p = p[revlog_constants.INDEX_HEADER.size :] p = p[revlog_constants.INDEX_HEADER.size :]
return p return p

mercurial/revlogutils/revlogv0.py

offset_type(e[0], 0), offset_type(e[0], 0),
e[1], e[1],
-1, -1,
e[2], e[2],
e[3], e[3],
nodemap.get(e[4], node.nullrev), nodemap.get(e[4], node.nullrev),
nodemap.get(e[5], node.nullrev), nodemap.get(e[5], node.nullrev),
e[6], e[6],
0, # no side data support
0, # no side data support
) )
index.append(e2) index.append(e2)
nodemap[e[6]] = n nodemap[e[6]] = n
n += 1 n += 1
index = revlogoldindex(index) index = revlogoldindex(index)
return index, None return index, None

mercurial/unionrepo.py

self.revlog2 = revlog2 self.revlog2 = revlog2
n = len(self) n = len(self)
self.repotiprev = n - 1 self.repotiprev = n - 1
self.bundlerevs = set() # used by 'bundle()' revset expression self.bundlerevs = set() # used by 'bundle()' revset expression
for rev2 in self.revlog2: for rev2 in self.revlog2:
rev = self.revlog2.index[rev2] rev = self.revlog2.index[rev2]
# rev numbers - in revlog2, very different from self.rev # rev numbers - in revlog2, very different from self.rev
_start, _csize, rsize, base, linkrev, p1rev, p2rev, node = rev (
_start,
_csize,
rsize,
base,
linkrev,
p1rev,
p2rev,
node,
_sdo,
_sds,
) = rev
flags = _start & 0xFFFF flags = _start & 0xFFFF
if linkmapper is None: # link is to same revlog if linkmapper is None: # link is to same revlog
assert linkrev == rev2 # we never link back assert linkrev == rev2 # we never link back
link = n link = n
else: # rev must be mapped from repo2 cl to unified cl by linkmapper else: # rev must be mapped from repo2 cl to unified cl by linkmapper
link = linkmapper(linkrev) link = linkmapper(linkrev)

tests/test-parseindex2.py

n = off = 0 n = off = 0
l = len(data) - s l = len(data) - s
append = index.append append = index.append
if inline: if inline:
cache = (0, data) cache = (0, data)
while off <= l: while off <= l:
e = struct.unpack(indexformatng, data[off : off + s]) e = struct.unpack(indexformatng, data[off : off + s])
e = e + (0, 0)
nodemap[e[7]] = n nodemap[e[7]] = n
append(e) append(e)
n += 1 n += 1
if e[1] < 0: if e[1] < 0:
break break
off += e[1] + s off += e[1] + s
else: else:
while off <= l: while off <= l:
e = struct.unpack(indexformatng, data[off : off + s]) e = struct.unpack(indexformatng, data[off : off + s])
e = e + (0, 0)
nodemap[e[7]] = n nodemap[e[7]] = n
append(e) append(e)
n += 1 n += 1
off += s off += s
e = list(index[0]) e = list(index[0])
type = gettype(e[0]) type = gettype(e[0])
e[0] = offset_type(0, type) e[0] = offset_type(0, type)
i, i,
'Reverse lookup inconsistent for %r' % hex(r[7]), 'Reverse lookup inconsistent for %r' % hex(r[7]),
) )
except TypeError: except TypeError:
# pure version doesn't support this # pure version doesn't support this
break break
def testminusone(self): def testminusone(self):
want = (0, 0, 0, -1, -1, -1, -1, sha1nodeconstants.nullid) want = (0, 0, 0, -1, -1, -1, -1, sha1nodeconstants.nullid, 0, 0)
index, junk = parsers.parse_index2(data_inlined, True) index, junk = parsers.parse_index2(data_inlined, True)
got = index[-1] got = index[-1]
self.assertEqual(want, got) # inline data self.assertEqual(want, got) # inline data
index, junk = parsers.parse_index2(data_non_inlined, False) index, junk = parsers.parse_index2(data_non_inlined, False)
got = index[-1] got = index[-1]
self.assertEqual(want, got) # no inline data self.assertEqual(want, got) # no inline data