This is an archive of the discontinued Mercurial Phabricator instance.

Differential D12178

revlog: implement changelogv2 packing and unpacking in C
ClosedPublic

Authored by pacien on Feb 14 2022, 1:06 PM.

Details

Reviewers
Alphare
Group Reviewers
hg-reviewers
Commits
rHG7dd5a2c0116a: revlog: implement changelogv2 packing and unpacking in C
rHG9c7a7b063b20: revlog: implement changelogv2 packing and unpacking in C
Summary

This introduces a C implementation of changelogv2 records packing and unpacking
operations matching the pure Python counterpart, similarly to what we already
have for revlogv1 and revlogv2.

This is also necessary to access changelogv2 record fields from future Rust
code without going through the Python part, which would annihilate any
performance benefit.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

pacien created this revision.Feb 14 2022, 1:06 PM
Alphare added a subscriber: Alphare.Feb 15 2022, 8:46 AM
Alphare added inline comments.
mercurial/cext/revlog.c
466

Maybe a small comment here saying that we need the base_rev and link_rev because they're declared but they're useless in clv2?

Alphare accepted this revision.Feb 15 2022, 9:38 AM

Amended manually to keep churn down

This revision is now accepted and ready to land.Feb 15 2022, 9:38 AM
pacien added a commit: rHG9c7a7b063b20: revlog: implement changelogv2 packing and unpacking in C.Feb 15 2022, 9:49 AM
Closed by commit rHG9c7a7b063b20: revlog: implement changelogv2 packing and unpacking in C (authored by pacien). · Explain Why
This revision was automatically updated to reflect the committed changes.
Alphare added inline comments.Feb 15 2022, 11:35 AM
mercurial/cext/revlog.c
705

For posterity: this was missing putbe32(rank, data + entry_cl2_offset_rank); here

Revision Contents
Changeset List

PathPackages
Mmercurial/cext/revlog.c (104 lines)

Diff 32207

mercurial/cext/revlog.c

#endif #endif
/* A RevlogNG v1 index entry is 64 bytes long. */ /* A RevlogNG v1 index entry is 64 bytes long. */
static const long v1_entry_size = 64; static const long v1_entry_size = 64;
/* A Revlogv2 index entry is 96 bytes long. */ /* A Revlogv2 index entry is 96 bytes long. */
static const long v2_entry_size = 96; static const long v2_entry_size = 96;
static const long format_v1 = 1; /* Internal only, could be any number */ static const long format_v1 = 1; /* Internal only, could be any number */
static const long format_v2 = 2; /* Internal only, could be any number */ static const long format_v2 = 2; /* Internal only, could be any number */
static const long format_cl2 = 3; /* Internal only, could be any number */
static const long entry_v1_offset_high = 0; static const long entry_v1_offset_high = 0;
static const long entry_v1_offset_offset_flags = 4; static const long entry_v1_offset_offset_flags = 4;
static const long entry_v1_offset_comp_len = 8; static const long entry_v1_offset_comp_len = 8;
static const long entry_v1_offset_uncomp_len = 12; static const long entry_v1_offset_uncomp_len = 12;
static const long entry_v1_offset_base_rev = 16; static const long entry_v1_offset_base_rev = 16;
static const long entry_v1_offset_link_rev = 20; static const long entry_v1_offset_link_rev = 20;
static const long entry_v1_offset_parent_1 = 24; static const long entry_v1_offset_parent_1 = 24;
static const long entry_v2_offset_parent_1 = 24; static const long entry_v2_offset_parent_1 = 24;
static const long entry_v2_offset_parent_2 = 28; static const long entry_v2_offset_parent_2 = 28;
static const long entry_v2_offset_node_id = 32; static const long entry_v2_offset_node_id = 32;
static const long entry_v2_offset_sidedata_offset = 64; static const long entry_v2_offset_sidedata_offset = 64;
static const long entry_v2_offset_sidedata_comp_len = 72; static const long entry_v2_offset_sidedata_comp_len = 72;
static const long entry_v2_offset_all_comp_mode = 76; static const long entry_v2_offset_all_comp_mode = 76;
/* next free offset: 77 */ /* next free offset: 77 */
static const long entry_cl2_offset_high = 0;
static const long entry_cl2_offset_offset_flags = 4;
static const long entry_cl2_offset_comp_len = 8;
static const long entry_cl2_offset_uncomp_len = 12;
static const long entry_cl2_offset_parent_1 = 16;
static const long entry_cl2_offset_parent_2 = 20;
static const long entry_cl2_offset_node_id = 24;
static const long entry_cl2_offset_sidedata_offset = 56;
static const long entry_cl2_offset_sidedata_comp_len = 64;
static const long entry_cl2_offset_all_comp_mode = 68;
static const long entry_cl2_offset_rank = 69;
/* next free offset: 73 */
static const char comp_mode_inline = 2; static const char comp_mode_inline = 2;
static const char rank_unknown = -1; static const char rank_unknown = -1;
static void raise_revlog_error(void) static void raise_revlog_error(void)
{ {
PyObject *mod = NULL, *dict = NULL, *errclass = NULL; PyObject *mod = NULL, *dict = NULL, *errclass = NULL;
mod = PyImport_ImportModule("mercurial.error"); mod = PyImport_ImportModule("mercurial.error");
const char *data = index_deref(self, rev); const char *data = index_deref(self, rev);
if (self->format_version == format_v1) { if (self->format_version == format_v1) {
ps[0] = getbe32(data + entry_v1_offset_parent_1); ps[0] = getbe32(data + entry_v1_offset_parent_1);
ps[1] = getbe32(data + entry_v1_offset_parent_2); ps[1] = getbe32(data + entry_v1_offset_parent_2);
} else if (self->format_version == format_v2) { } else if (self->format_version == format_v2) {
ps[0] = getbe32(data + entry_v2_offset_parent_1); ps[0] = getbe32(data + entry_v2_offset_parent_1);
ps[1] = getbe32(data + entry_v2_offset_parent_2); ps[1] = getbe32(data + entry_v2_offset_parent_2);
} else if (self->format_version == format_cl2) {
ps[0] = getbe32(data + entry_cl2_offset_parent_1);
ps[1] = getbe32(data + entry_cl2_offset_parent_2);
} else { } else {
raise_revlog_error(); raise_revlog_error();
return -1; return -1;
} }
/* If index file is corrupted, ps[] may point to invalid revisions. So /* If index file is corrupted, ps[] may point to invalid revisions. So
* there is a risk of buffer overflow to trust them unconditionally. */ * there is a risk of buffer overflow to trust them unconditionally. */
if (ps[0] < -1 || ps[0] > maxrev || ps[1] < -1 || ps[1] > maxrev) { if (ps[0] < -1 || ps[0] > maxrev || ps[1] < -1 || ps[1] > maxrev) {
if (rev == 0) { if (rev == 0) {
/* mask out version number for the first entry */ /* mask out version number for the first entry */
offset &= 0xFFFF; offset &= 0xFFFF;
} else { } else {
uint32_t offset_high = uint32_t offset_high =
getbe32(data + entry_v2_offset_high); getbe32(data + entry_v2_offset_high);
offset |= ((uint64_t)offset_high) << 32; offset |= ((uint64_t)offset_high) << 32;
} }
} else if (self->format_version == format_cl2) {
uint32_t offset_high = getbe32(data + entry_cl2_offset_high);
offset = getbe32(data + entry_cl2_offset_offset_flags);
offset |= ((uint64_t)offset_high) << 32;
} else { } else {
raise_revlog_error(); raise_revlog_error();
return -1; return -1;
} }
return (int64_t)(offset >> 16); return (int64_t)(offset >> 16);
} }
static inline int index_get_length(indexObject *self, Py_ssize_t rev) static inline int index_get_length(indexObject *self, Py_ssize_t rev)
{ {
const char *data; const char *data;
int tmp; int tmp;
if (rev == nullrev) if (rev == nullrev)
return 0; return 0;
data = index_deref(self, rev); data = index_deref(self, rev);
if (self->format_version == format_v1) { if (self->format_version == format_v1) {
tmp = (int)getbe32(data + entry_v1_offset_comp_len); tmp = (int)getbe32(data + entry_v1_offset_comp_len);
} else if (self->format_version == format_v2) { } else if (self->format_version == format_v2) {
tmp = (int)getbe32(data + entry_v2_offset_comp_len); tmp = (int)getbe32(data + entry_v2_offset_comp_len);
} else if (self->format_version == format_cl2) {
tmp = (int)getbe32(data + entry_cl2_offset_comp_len);
} else { } else {
raise_revlog_error(); raise_revlog_error();
return -1; return -1;
} }
if (tmp < 0) { if (tmp < 0) {
PyErr_Format(PyExc_OverflowError, PyErr_Format(PyExc_OverflowError,
"revlog entry size out of bound (%d)", tmp); "revlog entry size out of bound (%d)", tmp);
return -1; return -1;
* 4 bytes: parent 1 revision * 4 bytes: parent 1 revision
* 4 bytes: parent 2 revision * 4 bytes: parent 2 revision
* 32 bytes: nodeid (only 20 bytes used with SHA-1) * 32 bytes: nodeid (only 20 bytes used with SHA-1)
*/ */
static PyObject *index_get(indexObject *self, Py_ssize_t pos) static PyObject *index_get(indexObject *self, Py_ssize_t pos)
{ {
uint64_t offset_flags, sidedata_offset; uint64_t offset_flags, sidedata_offset;
int comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2, int comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2,
sidedata_comp_len; sidedata_comp_len, rank = rank_unknown;
char data_comp_mode, sidedata_comp_mode; char data_comp_mode, sidedata_comp_mode;
const char *c_node_id; const char *c_node_id;
const char *data; const char *data;
Py_ssize_t length = index_length(self); Py_ssize_t length = index_length(self);
if (pos == nullrev) { if (pos == nullrev) {
Py_INCREF(self->nullentry); Py_INCREF(self->nullentry);
return self->nullentry; return self->nullentry;
sidedata_offset = sidedata_offset =
getbe64(data + entry_v2_offset_sidedata_offset); getbe64(data + entry_v2_offset_sidedata_offset);
sidedata_comp_len = sidedata_comp_len =
getbe32(data + entry_v2_offset_sidedata_comp_len); getbe32(data + entry_v2_offset_sidedata_comp_len);
data_comp_mode = data[entry_v2_offset_all_comp_mode] & 3; data_comp_mode = data[entry_v2_offset_all_comp_mode] & 3;
sidedata_comp_mode = sidedata_comp_mode =
((data[entry_v2_offset_all_comp_mode] >> 2) & 3); ((data[entry_v2_offset_all_comp_mode] >> 2) & 3);
} else if (self->format_version == format_cl2) {
uint32_t offset_high = getbe32(data + entry_cl2_offset_high);
offset_flags = getbe32(data + entry_cl2_offset_offset_flags);
offset_flags |= ((uint64_t)offset_high) << 32;
comp_len = getbe32(data + entry_cl2_offset_comp_len);
uncomp_len = getbe32(data + entry_cl2_offset_uncomp_len);
/* base_rev and link_rev are not stored in changelogv2, but are
AlphareUnsubmitted
Not Done

Maybe a small comment here saying that we need the base_rev and link_rev because they're declared but they're useless in clv2?

Alphare: Maybe a small comment here saying that we need the base_rev and link_rev because they're…
still used by some functions shared with the other revlogs.
They are supposed to contain links to other revisions,
but they always point to themselves in the case of a changelog.
*/
base_rev = pos;
link_rev = pos;
parent_1 = getbe32(data + entry_cl2_offset_parent_1);
parent_2 = getbe32(data + entry_cl2_offset_parent_2);
c_node_id = data + entry_cl2_offset_node_id;
sidedata_offset =
getbe64(data + entry_cl2_offset_sidedata_offset);
sidedata_comp_len =
getbe32(data + entry_cl2_offset_sidedata_comp_len);
data_comp_mode = data[entry_cl2_offset_all_comp_mode] & 3;
sidedata_comp_mode =
((data[entry_cl2_offset_all_comp_mode] >> 2) & 3);
rank = getbe32(data + entry_cl2_offset_rank);
} else { } else {
raise_revlog_error(); raise_revlog_error();
return NULL; return NULL;
} }
return Py_BuildValue(tuple_format, offset_flags, comp_len, uncomp_len, return Py_BuildValue(tuple_format, offset_flags, comp_len, uncomp_len,
base_rev, link_rev, parent_1, parent_2, c_node_id, base_rev, link_rev, parent_1, parent_2, c_node_id,
self->nodelen, sidedata_offset, sidedata_comp_len, self->nodelen, sidedata_offset, sidedata_comp_len,
data_comp_mode, sidedata_comp_mode, rank_unknown); data_comp_mode, sidedata_comp_mode, rank);
} }
/* /*
* Pack header information in binary * Pack header information in binary
*/ */
static PyObject *index_pack_header(indexObject *self, PyObject *args) static PyObject *index_pack_header(indexObject *self, PyObject *args)
{ {
int header; int header;
char out[4]; char out[4];
return NULL; return NULL;
data = index_deref(self, pos); data = index_deref(self, pos);
if (self->format_version == format_v1) { if (self->format_version == format_v1) {
node_id = data + entry_v1_offset_node_id; node_id = data + entry_v1_offset_node_id;
} else if (self->format_version == format_v2) { } else if (self->format_version == format_v2) {
node_id = data + entry_v2_offset_node_id; node_id = data + entry_v2_offset_node_id;
} else if (self->format_version == format_cl2) {
node_id = data + entry_cl2_offset_node_id;
} else { } else {
raise_revlog_error(); raise_revlog_error();
return NULL; return NULL;
} }
return data ? node_id : NULL; return data ? node_id : NULL;
} }
c_node_id_len); c_node_id_len);
putbe64(sidedata_offset, putbe64(sidedata_offset,
data + entry_v2_offset_sidedata_offset); data + entry_v2_offset_sidedata_offset);
putbe32(sidedata_comp_len, putbe32(sidedata_comp_len,
data + entry_v2_offset_sidedata_comp_len); data + entry_v2_offset_sidedata_comp_len);
comp_field = data_comp_mode & 3; comp_field = data_comp_mode & 3;
comp_field = comp_field | (sidedata_comp_mode & 3) << 2; comp_field = comp_field | (sidedata_comp_mode & 3) << 2;
data[entry_v2_offset_all_comp_mode] = comp_field; data[entry_v2_offset_all_comp_mode] = comp_field;
} else if (self->format_version == format_cl2) {
putbe32(offset_flags >> 32, data + entry_cl2_offset_high);
putbe32(offset_flags & 0xffffffffU,
data + entry_cl2_offset_offset_flags);
putbe32(comp_len, data + entry_cl2_offset_comp_len);
putbe32(uncomp_len, data + entry_cl2_offset_uncomp_len);
putbe32(parent_1, data + entry_cl2_offset_parent_1);
putbe32(parent_2, data + entry_cl2_offset_parent_2);
memcpy(data + entry_cl2_offset_node_id, c_node_id,
c_node_id_len);
putbe64(sidedata_offset,
data + entry_cl2_offset_sidedata_offset);
putbe32(sidedata_comp_len,
data + entry_cl2_offset_sidedata_comp_len);
comp_field = data_comp_mode & 3;
comp_field = comp_field | (sidedata_comp_mode & 3) << 2;
data[entry_cl2_offset_all_comp_mode] = comp_field;
} else { } else {
AlphareUnsubmitted
Not Done

For posterity: this was missing putbe32(rank, data + entry_cl2_offset_rank); here

Alphare: For posterity: this was missing `putbe32(rank, data + entry_cl2_offset_rank);` here
raise_revlog_error(); raise_revlog_error();
return NULL; return NULL;
} }
if (self->ntinitialized) if (self->ntinitialized)
nt_insert(&self->nt, c_node_id, rev); nt_insert(&self->nt, c_node_id, rev);
Py_CLEAR(self->headrevs); Py_CLEAR(self->headrevs);
PyExc_IndexError, PyExc_IndexError,
"cannot rewrite entries outside of this transaction"); "cannot rewrite entries outside of this transaction");
return NULL; return NULL;
} }
/* Find the newly added node, offset from the "already on-disk" length /* Find the newly added node, offset from the "already on-disk" length
*/ */
data = self->added + self->entry_size * (rev - self->length); data = self->added + self->entry_size * (rev - self->length);
if (self->format_version == format_v2) {
putbe64(offset_flags, data + entry_v2_offset_high); putbe64(offset_flags, data + entry_v2_offset_high);
putbe64(sidedata_offset, data + entry_v2_offset_sidedata_offset); putbe64(sidedata_offset,
putbe32(sidedata_comp_len, data + entry_v2_offset_sidedata_comp_len); data + entry_v2_offset_sidedata_offset);
putbe32(sidedata_comp_len,
data + entry_v2_offset_sidedata_comp_len);
data[entry_v2_offset_all_comp_mode] = data[entry_v2_offset_all_comp_mode] =
(data[entry_v2_offset_all_comp_mode] & ~(3 << 2)) | (data[entry_v2_offset_all_comp_mode] & ~(3 << 2)) |
((comp_mode & 3) << 2); ((comp_mode & 3) << 2);
} else if (self->format_version == format_cl2) {
putbe64(offset_flags, data + entry_cl2_offset_high);
putbe64(sidedata_offset,
data + entry_cl2_offset_sidedata_offset);
putbe32(sidedata_comp_len,
data + entry_cl2_offset_sidedata_comp_len);
data[entry_cl2_offset_all_comp_mode] =
(data[entry_cl2_offset_all_comp_mode] & ~(3 << 2)) |
((comp_mode & 3) << 2);
} else {
raise_revlog_error();
return NULL;
}
Py_RETURN_NONE; Py_RETURN_NONE;
} }
static PyObject *index_stats(indexObject *self) static PyObject *index_stats(indexObject *self)
{ {
PyObject *obj = PyDict_New(); PyObject *obj = PyDict_New();
PyObject *s = NULL; PyObject *s = NULL;
data = index_deref(self, rev); data = index_deref(self, rev);
if (data == NULL) if (data == NULL)
return -2; return -2;
if (self->format_version == format_v1) { if (self->format_version == format_v1) {
result = getbe32(data + entry_v1_offset_base_rev); result = getbe32(data + entry_v1_offset_base_rev);
} else if (self->format_version == format_v2) { } else if (self->format_version == format_v2) {
result = getbe32(data + entry_v2_offset_base_rev); result = getbe32(data + entry_v2_offset_base_rev);
} else if (self->format_version == format_cl2) {
return rev;
} else { } else {
raise_revlog_error(); raise_revlog_error();
return -1; return -1;
} }
if (result > rev) { if (result > rev) {
PyErr_Format( PyErr_Format(
PyExc_ValueError, PyExc_ValueError,