This is an archive of the discontinued Mercurial Phabricator instance.

Differential D12178

revlog: implement changelogv2 packing and unpacking in C
ClosedPublic

Authored by pacien on Feb 14 2022, 1:06 PM.

Download Raw Diff

Details

Reviewers

Alphare

Group Reviewers

hg-reviewers

Commits

rHG7dd5a2c0116a: revlog: implement changelogv2 packing and unpacking in C
rHG9c7a7b063b20: revlog: implement changelogv2 packing and unpacking in C

Summary

This introduces a C implementation of changelogv2 records packing and unpacking
operations matching the pure Python counterpart, similarly to what we already
have for revlogv1 and revlogv2.

This is also necessary to access changelogv2 record fields from future Rust
code without going through the Python part, which would annihilate any
performance benefit.

Diff Detail

Repository

rHG Mercurial

Lint

Automatic diff as part of commit; lint not applicable.

Unit

Automatic diff as part of commit; unit tests not applicable.

Event Timeline

pacien created this revision.Feb 14 2022, 1:06 PM

Herald added a reviewer: hg-reviewers. · View Herald TranscriptFeb 14 2022, 1:06 PM

Herald added a subscriber: mercurial-patches. · View Herald Transcript

pacien added a child revision: D12179: revlog: register changelogv2 C implementation in parsers.Feb 14 2022, 1:06 PM

Alphare added a subscriber: Alphare.Feb 15 2022, 8:46 AM

Alphare added inline comments.

mercurial/cext/revlog.c
466	Maybe a small comment here saying that we need the base_rev and link_rev because they're declared but they're useless in clv2?

Amended manually to keep churn down

This revision is now accepted and ready to land.Feb 15 2022, 9:38 AM

pacien added a commit: rHG9c7a7b063b20: revlog: implement changelogv2 packing and unpacking in C.Feb 15 2022, 9:49 AM

Closed by commit rHG9c7a7b063b20: revlog: implement changelogv2 packing and unpacking in C (authored by pacien). · Explain Why

This revision was automatically updated to reflect the committed changes.

pacien added a commit: rHG7dd5a2c0116a: revlog: implement changelogv2 packing and unpacking in C.Feb 15 2022, 11:35 AM

Alphare added inline comments.Feb 15 2022, 11:35 AM

mercurial/cext/revlog.c
705	For posterity: this was missing `putbe32(rank, data + entry_cl2_offset_rank);` here

Revision Contents
Changeset List

			Path	Packages
M			mercurial/cext/revlog.c (104 lines)

Status	Author	Revision
Closed	pacien	D12179 revlog: register changelogv2 C implementation in parsers
Closed	pacien	D12178 revlog: implement changelogv2 packing and unpacking in C
Closed	pacien	D12177 revlog: memset whole record instead of dealing with padding
Closed	pacien	D12176 revlog: do not compute node location by hand in index_invalidate_added

Diff 32207

mercurial/cext/revlog.c

	#endif			#endif

	/* A RevlogNG v1 index entry is 64 bytes long. */			/* A RevlogNG v1 index entry is 64 bytes long. */
	static const long v1_entry_size = 64;			static const long v1_entry_size = 64;

	/* A Revlogv2 index entry is 96 bytes long. */			/* A Revlogv2 index entry is 96 bytes long. */
	static const long v2_entry_size = 96;			static const long v2_entry_size = 96;

	static const long format_v1 = 1; /* Internal only, could be any number */			static const long format_v1 = 1; /* Internal only, could be any number */
	static const long format_v2 = 2; /* Internal only, could be any number */			static const long format_v2 = 2; /* Internal only, could be any number */
				static const long format_cl2 = 3; /* Internal only, could be any number */

	static const long entry_v1_offset_high = 0;			static const long entry_v1_offset_high = 0;
	static const long entry_v1_offset_offset_flags = 4;			static const long entry_v1_offset_offset_flags = 4;
	static const long entry_v1_offset_comp_len = 8;			static const long entry_v1_offset_comp_len = 8;
	static const long entry_v1_offset_uncomp_len = 12;			static const long entry_v1_offset_uncomp_len = 12;
	static const long entry_v1_offset_base_rev = 16;			static const long entry_v1_offset_base_rev = 16;
	static const long entry_v1_offset_link_rev = 20;			static const long entry_v1_offset_link_rev = 20;
	static const long entry_v1_offset_parent_1 = 24;			static const long entry_v1_offset_parent_1 = 24;
	static const long entry_v2_offset_parent_1 = 24;			static const long entry_v2_offset_parent_1 = 24;
	static const long entry_v2_offset_parent_2 = 28;			static const long entry_v2_offset_parent_2 = 28;
	static const long entry_v2_offset_node_id = 32;			static const long entry_v2_offset_node_id = 32;
	static const long entry_v2_offset_sidedata_offset = 64;			static const long entry_v2_offset_sidedata_offset = 64;
	static const long entry_v2_offset_sidedata_comp_len = 72;			static const long entry_v2_offset_sidedata_comp_len = 72;
	static const long entry_v2_offset_all_comp_mode = 76;			static const long entry_v2_offset_all_comp_mode = 76;
	/* next free offset: 77 */			/* next free offset: 77 */

				static const long entry_cl2_offset_high = 0;
				static const long entry_cl2_offset_offset_flags = 4;
				static const long entry_cl2_offset_comp_len = 8;
				static const long entry_cl2_offset_uncomp_len = 12;
				static const long entry_cl2_offset_parent_1 = 16;
				static const long entry_cl2_offset_parent_2 = 20;
				static const long entry_cl2_offset_node_id = 24;
				static const long entry_cl2_offset_sidedata_offset = 56;
				static const long entry_cl2_offset_sidedata_comp_len = 64;
				static const long entry_cl2_offset_all_comp_mode = 68;
				static const long entry_cl2_offset_rank = 69;
				/* next free offset: 73 */

	static const char comp_mode_inline = 2;			static const char comp_mode_inline = 2;
	static const char rank_unknown = -1;			static const char rank_unknown = -1;

	static void raise_revlog_error(void)			static void raise_revlog_error(void)
	{			{
	PyObject mod = NULL, dict = NULL, *errclass = NULL;			PyObject mod = NULL, dict = NULL, *errclass = NULL;

	mod = PyImport_ImportModule("mercurial.error");			mod = PyImport_ImportModule("mercurial.error");
	const char *data = index_deref(self, rev);			const char *data = index_deref(self, rev);

	if (self->format_version == format_v1) {			if (self->format_version == format_v1) {
	ps[0] = getbe32(data + entry_v1_offset_parent_1);			ps[0] = getbe32(data + entry_v1_offset_parent_1);
	ps[1] = getbe32(data + entry_v1_offset_parent_2);			ps[1] = getbe32(data + entry_v1_offset_parent_2);
	} else if (self->format_version == format_v2) {			} else if (self->format_version == format_v2) {
	ps[0] = getbe32(data + entry_v2_offset_parent_1);			ps[0] = getbe32(data + entry_v2_offset_parent_1);
	ps[1] = getbe32(data + entry_v2_offset_parent_2);			ps[1] = getbe32(data + entry_v2_offset_parent_2);
				} else if (self->format_version == format_cl2) {
				ps[0] = getbe32(data + entry_cl2_offset_parent_1);
				ps[1] = getbe32(data + entry_cl2_offset_parent_2);
	} else {			} else {
	raise_revlog_error();			raise_revlog_error();
	return -1;			return -1;
	}			}

	/* If index file is corrupted, ps[] may point to invalid revisions. So			/* If index file is corrupted, ps[] may point to invalid revisions. So
	* there is a risk of buffer overflow to trust them unconditionally. */			* there is a risk of buffer overflow to trust them unconditionally. */
	if (ps[0] < -1 \|\| ps[0] > maxrev \|\| ps[1] < -1 \|\| ps[1] > maxrev) {			if (ps[0] < -1 \|\| ps[0] > maxrev \|\| ps[1] < -1 \|\| ps[1] > maxrev) {
	if (rev == 0) {			if (rev == 0) {
	/* mask out version number for the first entry */			/* mask out version number for the first entry */
	offset &= 0xFFFF;			offset &= 0xFFFF;
	} else {			} else {
	uint32_t offset_high =			uint32_t offset_high =
	getbe32(data + entry_v2_offset_high);			getbe32(data + entry_v2_offset_high);
	offset \|= ((uint64_t)offset_high) << 32;			offset \|= ((uint64_t)offset_high) << 32;
	}			}
				} else if (self->format_version == format_cl2) {
				uint32_t offset_high = getbe32(data + entry_cl2_offset_high);
				offset = getbe32(data + entry_cl2_offset_offset_flags);
				offset \|= ((uint64_t)offset_high) << 32;
	} else {			} else {
	raise_revlog_error();			raise_revlog_error();
	return -1;			return -1;
	}			}

	return (int64_t)(offset >> 16);			return (int64_t)(offset >> 16);
	}			}

	static inline int index_get_length(indexObject *self, Py_ssize_t rev)			static inline int index_get_length(indexObject *self, Py_ssize_t rev)
	{			{
	const char *data;			const char *data;
	int tmp;			int tmp;

	if (rev == nullrev)			if (rev == nullrev)
	return 0;			return 0;

	data = index_deref(self, rev);			data = index_deref(self, rev);

	if (self->format_version == format_v1) {			if (self->format_version == format_v1) {
	tmp = (int)getbe32(data + entry_v1_offset_comp_len);			tmp = (int)getbe32(data + entry_v1_offset_comp_len);
	} else if (self->format_version == format_v2) {			} else if (self->format_version == format_v2) {
	tmp = (int)getbe32(data + entry_v2_offset_comp_len);			tmp = (int)getbe32(data + entry_v2_offset_comp_len);
				} else if (self->format_version == format_cl2) {
				tmp = (int)getbe32(data + entry_cl2_offset_comp_len);
	} else {			} else {
	raise_revlog_error();			raise_revlog_error();
	return -1;			return -1;
	}			}
	if (tmp < 0) {			if (tmp < 0) {
	PyErr_Format(PyExc_OverflowError,			PyErr_Format(PyExc_OverflowError,
	"revlog entry size out of bound (%d)", tmp);			"revlog entry size out of bound (%d)", tmp);
	return -1;			return -1;
	* 4 bytes: parent 1 revision			* 4 bytes: parent 1 revision
	* 4 bytes: parent 2 revision			* 4 bytes: parent 2 revision
	* 32 bytes: nodeid (only 20 bytes used with SHA-1)			* 32 bytes: nodeid (only 20 bytes used with SHA-1)
	*/			*/
	static PyObject index_get(indexObject self, Py_ssize_t pos)			static PyObject index_get(indexObject self, Py_ssize_t pos)
	{			{
	uint64_t offset_flags, sidedata_offset;			uint64_t offset_flags, sidedata_offset;
	int comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2,			int comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2,
	sidedata_comp_len;			sidedata_comp_len, rank = rank_unknown;
	char data_comp_mode, sidedata_comp_mode;			char data_comp_mode, sidedata_comp_mode;
	const char *c_node_id;			const char *c_node_id;
	const char *data;			const char *data;
	Py_ssize_t length = index_length(self);			Py_ssize_t length = index_length(self);

	if (pos == nullrev) {			if (pos == nullrev) {
	Py_INCREF(self->nullentry);			Py_INCREF(self->nullentry);
	return self->nullentry;			return self->nullentry;

	sidedata_offset =			sidedata_offset =
	getbe64(data + entry_v2_offset_sidedata_offset);			getbe64(data + entry_v2_offset_sidedata_offset);
	sidedata_comp_len =			sidedata_comp_len =
	getbe32(data + entry_v2_offset_sidedata_comp_len);			getbe32(data + entry_v2_offset_sidedata_comp_len);
	data_comp_mode = data[entry_v2_offset_all_comp_mode] & 3;			data_comp_mode = data[entry_v2_offset_all_comp_mode] & 3;
	sidedata_comp_mode =			sidedata_comp_mode =
	((data[entry_v2_offset_all_comp_mode] >> 2) & 3);			((data[entry_v2_offset_all_comp_mode] >> 2) & 3);
				} else if (self->format_version == format_cl2) {
				uint32_t offset_high = getbe32(data + entry_cl2_offset_high);
				offset_flags = getbe32(data + entry_cl2_offset_offset_flags);
				offset_flags \|= ((uint64_t)offset_high) << 32;
				comp_len = getbe32(data + entry_cl2_offset_comp_len);
				uncomp_len = getbe32(data + entry_cl2_offset_uncomp_len);
				/* base_rev and link_rev are not stored in changelogv2, but are
				AlphareUnsubmitted Not Done Maybe a small comment here saying that we need the base_rev and link_rev because they're declared but they're useless in clv2? Alphare: Maybe a small comment here saying that we need the base_rev and link_rev because they're…
				still used by some functions shared with the other revlogs.
				They are supposed to contain links to other revisions,
				but they always point to themselves in the case of a changelog.
				*/
				base_rev = pos;
				link_rev = pos;
				parent_1 = getbe32(data + entry_cl2_offset_parent_1);
				parent_2 = getbe32(data + entry_cl2_offset_parent_2);
				c_node_id = data + entry_cl2_offset_node_id;
				sidedata_offset =
				getbe64(data + entry_cl2_offset_sidedata_offset);
				sidedata_comp_len =
				getbe32(data + entry_cl2_offset_sidedata_comp_len);
				data_comp_mode = data[entry_cl2_offset_all_comp_mode] & 3;
				sidedata_comp_mode =
				((data[entry_cl2_offset_all_comp_mode] >> 2) & 3);
				rank = getbe32(data + entry_cl2_offset_rank);
	} else {			} else {
	raise_revlog_error();			raise_revlog_error();
	return NULL;			return NULL;
	}			}

	return Py_BuildValue(tuple_format, offset_flags, comp_len, uncomp_len,			return Py_BuildValue(tuple_format, offset_flags, comp_len, uncomp_len,
	base_rev, link_rev, parent_1, parent_2, c_node_id,			base_rev, link_rev, parent_1, parent_2, c_node_id,
	self->nodelen, sidedata_offset, sidedata_comp_len,			self->nodelen, sidedata_offset, sidedata_comp_len,
	data_comp_mode, sidedata_comp_mode, rank_unknown);			data_comp_mode, sidedata_comp_mode, rank);
	}			}
	/*			/*
	* Pack header information in binary			* Pack header information in binary
	*/			*/
	static PyObject index_pack_header(indexObject self, PyObject *args)			static PyObject index_pack_header(indexObject self, PyObject *args)
	{			{
	int header;			int header;
	char out[4];			char out[4];
	return NULL;			return NULL;

	data = index_deref(self, pos);			data = index_deref(self, pos);

	if (self->format_version == format_v1) {			if (self->format_version == format_v1) {
	node_id = data + entry_v1_offset_node_id;			node_id = data + entry_v1_offset_node_id;
	} else if (self->format_version == format_v2) {			} else if (self->format_version == format_v2) {
	node_id = data + entry_v2_offset_node_id;			node_id = data + entry_v2_offset_node_id;
				} else if (self->format_version == format_cl2) {
				node_id = data + entry_cl2_offset_node_id;
	} else {			} else {
	raise_revlog_error();			raise_revlog_error();
	return NULL;			return NULL;
	}			}

	return data ? node_id : NULL;			return data ? node_id : NULL;
	}			}

	c_node_id_len);			c_node_id_len);
	putbe64(sidedata_offset,			putbe64(sidedata_offset,
	data + entry_v2_offset_sidedata_offset);			data + entry_v2_offset_sidedata_offset);
	putbe32(sidedata_comp_len,			putbe32(sidedata_comp_len,
	data + entry_v2_offset_sidedata_comp_len);			data + entry_v2_offset_sidedata_comp_len);
	comp_field = data_comp_mode & 3;			comp_field = data_comp_mode & 3;
	comp_field = comp_field \| (sidedata_comp_mode & 3) << 2;			comp_field = comp_field \| (sidedata_comp_mode & 3) << 2;
	data[entry_v2_offset_all_comp_mode] = comp_field;			data[entry_v2_offset_all_comp_mode] = comp_field;
				} else if (self->format_version == format_cl2) {
				putbe32(offset_flags >> 32, data + entry_cl2_offset_high);
				putbe32(offset_flags & 0xffffffffU,
				data + entry_cl2_offset_offset_flags);
				putbe32(comp_len, data + entry_cl2_offset_comp_len);
				putbe32(uncomp_len, data + entry_cl2_offset_uncomp_len);
				putbe32(parent_1, data + entry_cl2_offset_parent_1);
				putbe32(parent_2, data + entry_cl2_offset_parent_2);
				memcpy(data + entry_cl2_offset_node_id, c_node_id,
				c_node_id_len);
				putbe64(sidedata_offset,
				data + entry_cl2_offset_sidedata_offset);
				putbe32(sidedata_comp_len,
				data + entry_cl2_offset_sidedata_comp_len);
				comp_field = data_comp_mode & 3;
				comp_field = comp_field \| (sidedata_comp_mode & 3) << 2;
				data[entry_cl2_offset_all_comp_mode] = comp_field;
	} else {			} else {
				AlphareUnsubmitted Not Done For posterity: this was missing `putbe32(rank, data + entry_cl2_offset_rank);` here Alphare: For posterity: this was missing `putbe32(rank, data + entry_cl2_offset_rank);` here
	raise_revlog_error();			raise_revlog_error();
	return NULL;			return NULL;
	}			}

	if (self->ntinitialized)			if (self->ntinitialized)
	nt_insert(&self->nt, c_node_id, rev);			nt_insert(&self->nt, c_node_id, rev);

	Py_CLEAR(self->headrevs);			Py_CLEAR(self->headrevs);
	PyExc_IndexError,			PyExc_IndexError,
	"cannot rewrite entries outside of this transaction");			"cannot rewrite entries outside of this transaction");
	return NULL;			return NULL;
	}			}

	/* Find the newly added node, offset from the "already on-disk" length			/* Find the newly added node, offset from the "already on-disk" length
	*/			*/
	data = self->added + self->entry_size * (rev - self->length);			data = self->added + self->entry_size * (rev - self->length);
				if (self->format_version == format_v2) {
	putbe64(offset_flags, data + entry_v2_offset_high);			putbe64(offset_flags, data + entry_v2_offset_high);
	putbe64(sidedata_offset, data + entry_v2_offset_sidedata_offset);			putbe64(sidedata_offset,
	putbe32(sidedata_comp_len, data + entry_v2_offset_sidedata_comp_len);			data + entry_v2_offset_sidedata_offset);
				putbe32(sidedata_comp_len,
				data + entry_v2_offset_sidedata_comp_len);
	data[entry_v2_offset_all_comp_mode] =			data[entry_v2_offset_all_comp_mode] =
	(data[entry_v2_offset_all_comp_mode] & ~(3 << 2)) \|			(data[entry_v2_offset_all_comp_mode] & ~(3 << 2)) \|
	((comp_mode & 3) << 2);			((comp_mode & 3) << 2);
				} else if (self->format_version == format_cl2) {
				putbe64(offset_flags, data + entry_cl2_offset_high);
				putbe64(sidedata_offset,
				data + entry_cl2_offset_sidedata_offset);
				putbe32(sidedata_comp_len,
				data + entry_cl2_offset_sidedata_comp_len);
				data[entry_cl2_offset_all_comp_mode] =
				(data[entry_cl2_offset_all_comp_mode] & ~(3 << 2)) \|
				((comp_mode & 3) << 2);
				} else {
				raise_revlog_error();
				return NULL;
				}

	Py_RETURN_NONE;			Py_RETURN_NONE;
	}			}

	static PyObject index_stats(indexObject self)			static PyObject index_stats(indexObject self)
	{			{
	PyObject *obj = PyDict_New();			PyObject *obj = PyDict_New();
	PyObject *s = NULL;			PyObject *s = NULL;
	data = index_deref(self, rev);			data = index_deref(self, rev);
	if (data == NULL)			if (data == NULL)
	return -2;			return -2;

	if (self->format_version == format_v1) {			if (self->format_version == format_v1) {
	result = getbe32(data + entry_v1_offset_base_rev);			result = getbe32(data + entry_v1_offset_base_rev);
	} else if (self->format_version == format_v2) {			} else if (self->format_version == format_v2) {
	result = getbe32(data + entry_v2_offset_base_rev);			result = getbe32(data + entry_v2_offset_base_rev);
				} else if (self->format_version == format_cl2) {
				return rev;
	} else {			} else {
	raise_revlog_error();			raise_revlog_error();
	return -1;			return -1;
	}			}

	if (result > rev) {			if (result > rev) {
	PyErr_Format(			PyErr_Format(
	PyExc_ValueError,			PyExc_ValueError,

Diff	ID	Description	Created	Lint	Unit
Base		Base
Diff 1	32177		Feb 14 2022, 1:06 PM	★	★
Diff 2	32207	rHG9c7a7b063b202e0767a05fe9dda203843abc5a60	Feb 14 2022, 6:34 AM	★	★