( )⚙ D12136 revlog: extract entry byte offsets into named constants

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D12136

revlog: extract entry byte offsets into named constants
ClosedPublic

Authored by pacien on Feb 7 2022, 6:42 AM.

Details

Reviewers
Alphare
Group Reviewers
hg-reviewers
Commits
rHGa9364de9be29: revlog: extract entry byte offsets into named constants
Summary

Labelling the fields pointed by the given offsets shared by revlog v1 and v2.

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

pacien created this revision.Feb 7 2022, 6:42 AM
joerg.sonnenberger added a subscriber: joerg.sonnenberger.Feb 7 2022, 7:00 AM

This is reasonable, but I didn't double-check that all cases are identical.

Alphare accepted this revision.Feb 7 2022, 9:09 AM
This revision is now accepted and ready to land.Feb 7 2022, 9:09 AM
pacien added a commit: rHGa9364de9be29: revlog: extract entry byte offsets into named constants.Feb 8 2022, 10:44 AM
Closed by commit rHGa9364de9be29: revlog: extract entry byte offsets into named constants (authored by pacien). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mmercurial/cext/revlog.c (102 lines)

Diff 32080

mercurial/cext/revlog.c

static const long v1_entry_size = 64; static const long v1_entry_size = 64;
/* A Revlogv2 index entry is 96 bytes long. */ /* A Revlogv2 index entry is 96 bytes long. */
static const long v2_entry_size = 96; static const long v2_entry_size = 96;
static const long format_v1 = 1; /* Internal only, could be any number */ static const long format_v1 = 1; /* Internal only, could be any number */
static const long format_v2 = 2; /* Internal only, could be any number */ static const long format_v2 = 2; /* Internal only, could be any number */
static const long entry_offset_high = 0;
static const long entry_offset_offset_flags = 4;
static const long entry_offset_comp_len = 8;
static const long entry_offset_uncomp_len = 12;
static const long entry_offset_base_rev = 16;
static const long entry_offset_link_rev = 20;
static const long entry_offset_parent_1 = 24;
static const long entry_offset_parent_2 = 28;
static const long entry_offset_node_id = 32;
static const long entry_offset_sidedata_offset = 64;
static const long entry_offset_sidedata_comp_len = 72;
static const long entry_offset_all_comp_mode = 76;
static const long entry_offset_padding_start = 77;
static const char comp_mode_inline = 2; static const char comp_mode_inline = 2;
static const char rank_unknown = -1; static const char rank_unknown = -1;
static void raise_revlog_error(void) static void raise_revlog_error(void)
{ {
PyObject *mod = NULL, *dict = NULL, *errclass = NULL; PyObject *mod = NULL, *dict = NULL, *errclass = NULL;
mod = PyImport_ImportModule("mercurial.error"); mod = PyImport_ImportModule("mercurial.error");
* The specified rev must be valid and must not be nullrev. A returned * The specified rev must be valid and must not be nullrev. A returned
* parent revision may be nullrev, but is guaranteed to be in valid range. * parent revision may be nullrev, but is guaranteed to be in valid range.
*/ */
static inline int index_get_parents(indexObject *self, Py_ssize_t rev, int *ps, static inline int index_get_parents(indexObject *self, Py_ssize_t rev, int *ps,
int maxrev) int maxrev)
{ {
const char *data = index_deref(self, rev); const char *data = index_deref(self, rev);
ps[0] = getbe32(data + 24); ps[0] = getbe32(data + entry_offset_parent_1);
ps[1] = getbe32(data + 28); ps[1] = getbe32(data + entry_offset_parent_2);
/* If index file is corrupted, ps[] may point to invalid revisions. So /* If index file is corrupted, ps[] may point to invalid revisions. So
* there is a risk of buffer overflow to trust them unconditionally. */ * there is a risk of buffer overflow to trust them unconditionally. */
if (ps[0] < -1 || ps[0] > maxrev || ps[1] < -1 || ps[1] > maxrev) { if (ps[0] < -1 || ps[0] > maxrev || ps[1] < -1 || ps[1] > maxrev) {
PyErr_SetString(PyExc_ValueError, "parent out of range"); PyErr_SetString(PyExc_ValueError, "parent out of range");
return -1; return -1;
} }
return 0; return 0;
{ {
const char *data; const char *data;
uint64_t offset; uint64_t offset;
if (rev == nullrev) if (rev == nullrev)
return 0; return 0;
data = index_deref(self, rev); data = index_deref(self, rev);
offset = getbe32(data + 4); offset = getbe32(data + entry_offset_offset_flags);
if (rev == 0) { if (rev == 0) {
/* mask out version number for the first entry */ /* mask out version number for the first entry */
offset &= 0xFFFF; offset &= 0xFFFF;
} else { } else {
uint32_t offset_high = getbe32(data); uint32_t offset_high = getbe32(data + entry_offset_high);
offset |= ((uint64_t)offset_high) << 32; offset |= ((uint64_t)offset_high) << 32;
} }
return (int64_t)(offset >> 16); return (int64_t)(offset >> 16);
} }
static inline int index_get_length(indexObject *self, Py_ssize_t rev) static inline int index_get_length(indexObject *self, Py_ssize_t rev)
{ {
const char *data; const char *data;
int tmp; int tmp;
if (rev == nullrev) if (rev == nullrev)
return 0; return 0;
data = index_deref(self, rev); data = index_deref(self, rev);
tmp = (int)getbe32(data + 8); tmp = (int)getbe32(data + entry_offset_comp_len);
if (tmp < 0) { if (tmp < 0) {
PyErr_Format(PyExc_OverflowError, PyErr_Format(PyExc_OverflowError,
"revlog entry size out of bound (%d)", tmp); "revlog entry size out of bound (%d)", tmp);
return -1; return -1;
} }
return tmp; return tmp;
} }
PyErr_SetString(PyExc_IndexError, "revlog index out of range"); PyErr_SetString(PyExc_IndexError, "revlog index out of range");
return NULL; return NULL;
} }
data = index_deref(self, pos); data = index_deref(self, pos);
if (data == NULL) if (data == NULL)
return NULL; return NULL;
offset_flags = getbe32(data + 4); offset_flags = getbe32(data + entry_offset_offset_flags);
/* /*
* The first entry on-disk needs the version number masked out, * The first entry on-disk needs the version number masked out,
* but this doesn't apply if entries are added to an empty index. * but this doesn't apply if entries are added to an empty index.
*/ */
if (self->length && pos == 0) if (self->length && pos == 0)
offset_flags &= 0xFFFF; offset_flags &= 0xFFFF;
else { else {
uint32_t offset_high = getbe32(data); uint32_t offset_high = getbe32(data + entry_offset_high);
offset_flags |= ((uint64_t)offset_high) << 32; offset_flags |= ((uint64_t)offset_high) << 32;
} }
comp_len = getbe32(data + 8); comp_len = getbe32(data + entry_offset_comp_len);
uncomp_len = getbe32(data + 12); uncomp_len = getbe32(data + entry_offset_uncomp_len);
base_rev = getbe32(data + 16); base_rev = getbe32(data + entry_offset_base_rev);
link_rev = getbe32(data + 20); link_rev = getbe32(data + entry_offset_link_rev);
parent_1 = getbe32(data + 24); parent_1 = getbe32(data + entry_offset_parent_1);
parent_2 = getbe32(data + 28); parent_2 = getbe32(data + entry_offset_parent_2);
c_node_id = data + 32; c_node_id = data + entry_offset_node_id;
if (self->format_version == format_v1) { if (self->format_version == format_v1) {
sidedata_offset = 0; sidedata_offset = 0;
sidedata_comp_len = 0; sidedata_comp_len = 0;
data_comp_mode = comp_mode_inline; data_comp_mode = comp_mode_inline;
sidedata_comp_mode = comp_mode_inline; sidedata_comp_mode = comp_mode_inline;
} else { } else {
sidedata_offset = getbe64(data + 64); sidedata_offset = getbe64(data + entry_offset_sidedata_offset);
sidedata_comp_len = getbe32(data + 72); sidedata_comp_len =
data_comp_mode = data[76] & 3; getbe32(data + entry_offset_sidedata_comp_len);
sidedata_comp_mode = ((data[76] >> 2) & 3); data_comp_mode = data[entry_offset_all_comp_mode] & 3;
sidedata_comp_mode =
((data[entry_offset_all_comp_mode] >> 2) & 3);
} }
return Py_BuildValue(tuple_format, offset_flags, comp_len, uncomp_len, return Py_BuildValue(tuple_format, offset_flags, comp_len, uncomp_len,
base_rev, link_rev, parent_1, parent_2, c_node_id, base_rev, link_rev, parent_1, parent_2, c_node_id,
self->nodelen, sidedata_offset, sidedata_comp_len, self->nodelen, sidedata_offset, sidedata_comp_len,
data_comp_mode, sidedata_comp_mode, rank_unknown); data_comp_mode, sidedata_comp_mode, rank_unknown);
} }
/* /*
if (pos == nullrev) if (pos == nullrev)
return nullid; return nullid;
if (pos >= length) if (pos >= length)
return NULL; return NULL;
data = index_deref(self, pos); data = index_deref(self, pos);
return data ? data + 32 : NULL; return data ? data + entry_offset_node_id : NULL;
} }
/* /*
* Return the hash of the node corresponding to the given rev. The * Return the hash of the node corresponding to the given rev. The
* rev is assumed to be existing. If not, an exception is set. * rev is assumed to be existing. If not, an exception is set.
*/ */
static const char *index_node_existing(indexObject *self, Py_ssize_t pos) static const char *index_node_existing(indexObject *self, Py_ssize_t pos)
{ {
self->added, new_added_length * self->entry_size); self->added, new_added_length * self->entry_size);
if (!new_added) if (!new_added)
return PyErr_NoMemory(); return PyErr_NoMemory();
self->added = new_added; self->added = new_added;
self->added_length = new_added_length; self->added_length = new_added_length;
} }
rev = self->length + self->new_length; rev = self->length + self->new_length;
data = self->added + self->entry_size * self->new_length++; data = self->added + self->entry_size * self->new_length++;
putbe32(offset_flags >> 32, data); putbe32(offset_flags >> 32, data + entry_offset_high);
putbe32(offset_flags & 0xffffffffU, data + 4); putbe32(offset_flags & 0xffffffffU, data + entry_offset_offset_flags);
putbe32(comp_len, data + 8); putbe32(comp_len, data + entry_offset_comp_len);
putbe32(uncomp_len, data + 12); putbe32(uncomp_len, data + entry_offset_uncomp_len);
putbe32(base_rev, data + 16); putbe32(base_rev, data + entry_offset_base_rev);
putbe32(link_rev, data + 20); putbe32(link_rev, data + entry_offset_link_rev);
putbe32(parent_1, data + 24); putbe32(parent_1, data + entry_offset_parent_1);
putbe32(parent_2, data + 28); putbe32(parent_2, data + entry_offset_parent_2);
memcpy(data + 32, c_node_id, c_node_id_len); memcpy(data + entry_offset_node_id, c_node_id, c_node_id_len);
/* Padding since SHA-1 is only 20 bytes for now */ /* Padding since SHA-1 is only 20 bytes for now */
memset(data + 32 + c_node_id_len, 0, 32 - c_node_id_len); memset(data + entry_offset_node_id + c_node_id_len, 0,
entry_offset_node_id - c_node_id_len);
if (self->format_version == format_v2) { if (self->format_version == format_v2) {
putbe64(sidedata_offset, data + 64); putbe64(sidedata_offset, data + entry_offset_sidedata_offset);
putbe32(sidedata_comp_len, data + 72); putbe32(sidedata_comp_len,
data + entry_offset_sidedata_comp_len);
comp_field = data_comp_mode & 3; comp_field = data_comp_mode & 3;
comp_field = comp_field | (sidedata_comp_mode & 3) << 2; comp_field = comp_field | (sidedata_comp_mode & 3) << 2;
data[76] = comp_field; data[entry_offset_all_comp_mode] = comp_field;
/* Padding for 96 bytes alignment */ /* Padding for 96 bytes alignment */
memset(data + 77, 0, self->entry_size - 77); memset(data + entry_offset_padding_start, 0,
self->entry_size - entry_offset_padding_start);
} }
if (self->ntinitialized) if (self->ntinitialized)
nt_insert(&self->nt, c_node_id, rev); nt_insert(&self->nt, c_node_id, rev);
Py_CLEAR(self->headrevs); Py_CLEAR(self->headrevs);
Py_RETURN_NONE; Py_RETURN_NONE;
} }
PyExc_IndexError, PyExc_IndexError,
"cannot rewrite entries outside of this transaction"); "cannot rewrite entries outside of this transaction");
return NULL; return NULL;
} }
/* Find the newly added node, offset from the "already on-disk" length /* Find the newly added node, offset from the "already on-disk" length
*/ */
data = self->added + self->entry_size * (rev - self->length); data = self->added + self->entry_size * (rev - self->length);
putbe64(offset_flags, data); putbe64(offset_flags, data + entry_offset_high);
putbe64(sidedata_offset, data + 64); putbe64(sidedata_offset, data + entry_offset_sidedata_offset);
putbe32(sidedata_comp_len, data + 72); putbe32(sidedata_comp_len, data + entry_offset_sidedata_comp_len);
data[76] = (data[76] & ~(3 << 2)) | ((comp_mode & 3) << 2); data[entry_offset_all_comp_mode] =
(data[entry_offset_all_comp_mode] & ~(3 << 2)) |
((comp_mode & 3) << 2);
Py_RETURN_NONE; Py_RETURN_NONE;
} }
static PyObject *index_stats(indexObject *self) static PyObject *index_stats(indexObject *self)
{ {
PyObject *obj = PyDict_New(); PyObject *obj = PyDict_New();
PyObject *s = NULL; PyObject *s = NULL;
static inline int index_baserev(indexObject *self, int rev) static inline int index_baserev(indexObject *self, int rev)
{ {
const char *data; const char *data;
int result; int result;
data = index_deref(self, rev); data = index_deref(self, rev);
if (data == NULL) if (data == NULL)
return -2; return -2;
result = getbe32(data + 16); result = getbe32(data + entry_offset_base_rev);
if (result > rev) { if (result > rev) {
PyErr_Format( PyErr_Format(
PyExc_ValueError, PyExc_ValueError,
"corrupted revlog, revision base above revision: %d, %d", "corrupted revlog, revision base above revision: %d, %d",
rev, result); rev, result);
return -2; return -2;
} }
Py_ssize_t pos = 0; Py_ssize_t pos = 0;
Py_ssize_t end = self->buf.len; Py_ssize_t end = self->buf.len;
long incr = self->entry_size; long incr = self->entry_size;
Py_ssize_t len = 0; Py_ssize_t len = 0;
while (pos + self->entry_size <= end && pos >= 0) { while (pos + self->entry_size <= end && pos >= 0) {
uint32_t comp_len, sidedata_comp_len = 0; uint32_t comp_len, sidedata_comp_len = 0;
/* 3rd element of header is length of compressed inline data */ /* 3rd element of header is length of compressed inline data */
comp_len = getbe32(data + pos + 8); comp_len = getbe32(data + pos + entry_offset_comp_len);
if (self->entry_size == v2_entry_size) { if (self->entry_size == v2_entry_size) {
sidedata_comp_len = getbe32(data + pos + 72); sidedata_comp_len = getbe32(
data + pos + entry_offset_sidedata_comp_len);
} }
incr = self->entry_size + comp_len + sidedata_comp_len; incr = self->entry_size + comp_len + sidedata_comp_len;
if (offsets) if (offsets)
offsets[len] = data + pos; offsets[len] = data + pos;
len++; len++;
pos += incr; pos += incr;
} }