( )⚙ D10656 revlog: compress sidedata when doing "post-pull" sidedata update

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10656

revlog: compress sidedata when doing "post-pull" sidedata update
ClosedPublic

Authored by marmoute on May 4 2021, 10:15 AM.

Details

Reviewers
indygreg
Alphare
Group Reviewers
hg-reviewers
Commits
rHG07641bafa646: revlog: compress sidedata when doing "post-pull" sidedata update
Summary

All path writing sidedata are now using compression (when appropriate).

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

marmoute created this revision.May 4 2021, 10:15 AM
baymax updated this revision to Diff 27593.May 5 2021, 5:48 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

baymax updated this revision to Diff 27895.May 11 2021, 1:46 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

Alphare accepted this revision.May 17 2021, 5:49 PM
This revision is now accepted and ready to land.May 17 2021, 5:49 PM

Revision Contents
Changeset List

PathPackages
Mmercurial/cext/revlog.c (8 lines)
Mmercurial/pure/parsers.py (8 lines)
Mmercurial/revlog.py (21 lines)

Diff 27895

mercurial/cext/revlog.c

/* Replace an existing index entry's sidedata offset and length with new ones. /* Replace an existing index entry's sidedata offset and length with new ones.
This cannot be used outside of the context of sidedata rewriting, This cannot be used outside of the context of sidedata rewriting,
inside the transaction that creates the given revision. */ inside the transaction that creates the given revision. */
static PyObject *index_replace_sidedata_info(indexObject *self, PyObject *args) static PyObject *index_replace_sidedata_info(indexObject *self, PyObject *args)
{ {
uint64_t offset_flags, sidedata_offset; uint64_t offset_flags, sidedata_offset;
int rev; int rev;
char comp_mode;
Py_ssize_t sidedata_comp_len; Py_ssize_t sidedata_comp_len;
char *data; char *data;
#if LONG_MAX == 0x7fffffffL #if LONG_MAX == 0x7fffffffL
const char *const sidedata_format = PY23("nKiK", "nKiK"); const char *const sidedata_format = PY23("nKiKB", "nKiKB");
#else #else
const char *const sidedata_format = PY23("nkik", "nkik"); const char *const sidedata_format = PY23("nkikB", "nkikB");
#endif #endif
if (self->entry_size == v1_entry_size || self->inlined) { if (self->entry_size == v1_entry_size || self->inlined) {
/* /*
There is a bug in the transaction handling when going from an There is a bug in the transaction handling when going from an
inline revlog to a separate index and data file. Turn it off until inline revlog to a separate index and data file. Turn it off until
it's fixed, since v2 revlogs sometimes get rewritten on exchange. it's fixed, since v2 revlogs sometimes get rewritten on exchange.
See issue6485. See issue6485.
*/ */
raise_revlog_error(); raise_revlog_error();
return NULL; return NULL;
} }
if (!PyArg_ParseTuple(args, sidedata_format, &rev, &sidedata_offset, if (!PyArg_ParseTuple(args, sidedata_format, &rev, &sidedata_offset,
&sidedata_comp_len, &offset_flags)) &sidedata_comp_len, &offset_flags, &comp_mode))
return NULL; return NULL;
if (rev < 0 || rev >= index_length(self)) { if (rev < 0 || rev >= index_length(self)) {
PyErr_SetString(PyExc_IndexError, "revision outside index"); PyErr_SetString(PyExc_IndexError, "revision outside index");
return NULL; return NULL;
} }
if (rev < self->length) { if (rev < self->length) {
PyErr_SetString( PyErr_SetString(
PyExc_IndexError, PyExc_IndexError,
"cannot rewrite entries outside of this transaction"); "cannot rewrite entries outside of this transaction");
return NULL; return NULL;
} }
/* Find the newly added node, offset from the "already on-disk" length /* Find the newly added node, offset from the "already on-disk" length
*/ */
data = self->added + self->entry_size * (rev - self->length); data = self->added + self->entry_size * (rev - self->length);
putbe64(offset_flags, data); putbe64(offset_flags, data);
putbe64(sidedata_offset, data + 64); putbe64(sidedata_offset, data + 64);
putbe32(sidedata_comp_len, data + 72); putbe32(sidedata_comp_len, data + 72);
data[76] = (data[76] & ~(3 << 2)) | ((comp_mode & 3) << 2);
Py_RETURN_NONE; Py_RETURN_NONE;
} }
static PyObject *index_stats(indexObject *self) static PyObject *index_stats(indexObject *self)
{ {
PyObject *obj = PyDict_New(); PyObject *obj = PyDict_New();
PyObject *s = NULL; PyObject *s = NULL;

mercurial/pure/parsers.py

cls = InlinedIndexObject2 if revlogv2 else InlinedIndexObject cls = InlinedIndexObject2 if revlogv2 else InlinedIndexObject
return cls(data, inline), (0, data) return cls(data, inline), (0, data)
class Index2Mixin(object): class Index2Mixin(object):
index_format = revlog_constants.INDEX_ENTRY_V2 index_format = revlog_constants.INDEX_ENTRY_V2
def replace_sidedata_info( def replace_sidedata_info(
self, rev, sidedata_offset, sidedata_length, offset_flags self,
rev,
sidedata_offset,
sidedata_length,
offset_flags,
compression_mode,
): ):
""" """
Replace an existing index entry's sidedata offset and length with new Replace an existing index entry's sidedata offset and length with new
ones. ones.
This cannot be used outside of the context of sidedata rewriting, This cannot be used outside of the context of sidedata rewriting,
inside the transaction that creates the revision `rev`. inside the transaction that creates the revision `rev`.
""" """
if rev < 0: if rev < 0:
raise KeyError raise KeyError
self._check_index(rev) self._check_index(rev)
if rev < self._lgt: if rev < self._lgt:
msg = b"cannot rewrite entries outside of this transaction" msg = b"cannot rewrite entries outside of this transaction"
raise KeyError(msg) raise KeyError(msg)
else: else:
entry = list(self[rev]) entry = list(self[rev])
entry[0] = offset_flags entry[0] = offset_flags
entry[8] = sidedata_offset entry[8] = sidedata_offset
entry[9] = sidedata_length entry[9] = sidedata_length
entry[11] = compression_mode
entry = tuple(entry) entry = tuple(entry)
new = self._pack_entry(entry) new = self._pack_entry(entry)
self._extra[rev - self._lgt] = new self._extra[rev - self._lgt] = new
def _unpack_entry(self, data): def _unpack_entry(self, data):
data = self.index_format.unpack(data) data = self.index_format.unpack(data)
entry = data[:10] entry = data[:10]
data_comp = data[10] & 3 data_comp = data[10] & 3

mercurial/revlog.py

sidedata_helpers=helpers, sidedata_helpers=helpers,
sidedata={}, sidedata={},
rev=rev, rev=rev,
) )
serialized_sidedata = sidedatautil.serialize_sidedata( serialized_sidedata = sidedatautil.serialize_sidedata(
new_sidedata new_sidedata
) )
sidedata_compression_mode = COMP_MODE_INLINE
if serialized_sidedata and self.hassidedata:
sidedata_compression_mode = COMP_MODE_PLAIN
h, comp_sidedata = self.compress(serialized_sidedata)
if (
h != b'u'
and comp_sidedata[0] != b'\0'
and len(comp_sidedata) < len(serialized_sidedata)
):
assert not h
if (
comp_sidedata[0]
== self._docket.default_compression_header
):
sidedata_compression_mode = COMP_MODE_DEFAULT
serialized_sidedata = comp_sidedata
else:
sidedata_compression_mode = COMP_MODE_INLINE
serialized_sidedata = comp_sidedata
if entry[8] != 0 or entry[9] != 0: if entry[8] != 0 or entry[9] != 0:
# rewriting entries that already have sidedata is not # rewriting entries that already have sidedata is not
# supported yet, because it introduces garbage data in the # supported yet, because it introduces garbage data in the
# revlog. # revlog.
msg = b"rewriting existing sidedata is not supported yet" msg = b"rewriting existing sidedata is not supported yet"
raise error.Abort(msg) raise error.Abort(msg)
# Apply (potential) flags to add and to remove after running # Apply (potential) flags to add and to remove after running
# the sidedata helpers # the sidedata helpers
new_offset_flags = entry[0] | flags[0] & ~flags[1] new_offset_flags = entry[0] | flags[0] & ~flags[1]
entry_update = ( entry_update = (
current_offset, current_offset,
len(serialized_sidedata), len(serialized_sidedata),
new_offset_flags, new_offset_flags,
sidedata_compression_mode,
) )
# the sidedata computation might have move the file cursors around # the sidedata computation might have move the file cursors around
dfh.seek(current_offset, os.SEEK_SET) dfh.seek(current_offset, os.SEEK_SET)
dfh.write(serialized_sidedata) dfh.write(serialized_sidedata)
new_entries.append(entry_update) new_entries.append(entry_update)
current_offset += len(serialized_sidedata) current_offset += len(serialized_sidedata)
if self._docket is not None: if self._docket is not None: