nodemap: keep track of the docket for loaded data
ClosedPublic

Diff	ID	Description	Created	Lint	Unit
Base		Base
Diff 1	19303		Jan 15 2020, 9:55 AM	★	★
Diff 2	19431		Jan 17 2020, 1:06 PM	★	★
Diff 3	19766		Jan 31 2020, 10:29 AM	★	★
Diff 4	19794		Jan 31 2020, 5:59 PM	★	★
Diff 5	19838		Feb 2 2020, 3:18 AM	★	★
Diff 6	19897		Feb 4 2020, 7:25 PM	★	★
Diff 7	20118	rHG76a96e3a2bbb2dd7546e5f33eade0c21907f55fc	Jan 15 2020, 9:49 AM	★	★

Commit	Parents	Author	Summary	Date
e89c2a5cf4c2	13b4698e58d2	Pierre-Yves David		Jan 15 2020, 9:49 AM

Status	Author	Revision
Closed	marmoute	D7894 nodemap: introduce an option to use mmap to read the nodemap mapping
Closed	marmoute	D7893 nodemap: update the index with the newly written data (when appropriate)
Closed	marmoute	D7892 nodemap: never read more than the expected data amount
Closed	marmoute	D7891 nodemap: write new data from the expected current data length
Closed	marmoute	D7890 nodemap: double check the source docket when doing incremental update
Closed	marmoute	D7889 nodemap: track the total and unused amount of data in the rawdata file
Closed	marmoute	D7888 nodemap: track the maximum revision tracked in the nodemap
Closed	marmoute	D7887 nodemap: add a flag to dump the details of the docket
Closed	marmoute	D7886 nodemap: introduce append-only incremental update of the persisten data
Closed	marmoute	D7885 nodemap: keep track of the docket for loaded data
Closed	marmoute	D7884 nodemap: introduce an explicit class/object for the docket
Closed	marmoute	D7883 nodemap: keep track of the ondisk id of nodemap blocks
Closed	marmoute	D7847 nodemap: provide the on disk data to indexes who support it
Closed	marmoute	D7846 nodemap: all check that revision and nodes match in the nodemap
Closed	marmoute	D7845 nodemap: add basic checking of the on disk nodemap content
Closed	marmoute	D7844 nodemap: code to parse the persistent binary nodemap data
Closed	marmoute	D7843 nodemap: move the iteratio inside the Block object
Closed	marmoute	D7842 nodemap: use an explicit "Block" object in the reference implementation
Closed	marmoute	D7841 nodemap: add a optional `nodemap_add_full` method on indexes
Closed	marmoute	D7840 nodemap: add a (python) index class for persistent nodemap testing
Closed	marmoute	D7839 nodemap: delete older raw data file when creating a new ones
Closed	marmoute	D7838 nodemap: use an intermediate "docket" file to carry small metadata
Closed	marmoute	D7837 nodemap: only use persistent nodemap for non-inlined revlog
Closed	marmoute	D7836 nodemap: add a function to read the data from disk
Closed	marmoute	D7835 nodemap: write nodemap data on disk
Closed	marmoute	D7834 nodemap: have some python code writing a nodemap in persistent binary form

Diff 19303

mercurial/debugcommands.py

	if args['dump_new']:			if args['dump_new']:
	unfi = repo.unfiltered()			unfi = repo.unfiltered()
	cl = unfi.changelog			cl = unfi.changelog
	data = nodemap.persistent_data(cl.index)			data = nodemap.persistent_data(cl.index)
	ui.write(data)			ui.write(data)
	elif args['dump_disk']:			elif args['dump_disk']:
	unfi = repo.unfiltered()			unfi = repo.unfiltered()
	cl = unfi.changelog			cl = unfi.changelog
	data = nodemap.persisted_data(cl)			nm_data = nodemap.persisted_data(cl)
				if nm_data is not None:
				docket, data = nm_data
	ui.write(data)			ui.write(data)
	elif args['check']:			elif args['check']:
	unfi = repo.unfiltered()			unfi = repo.unfiltered()
	cl = unfi.changelog			cl = unfi.changelog
	data = nodemap.persisted_data(cl)			nm_data = nodemap.persisted_data(cl)
				if nm_data is not None:
				docket, data = nm_data
	return nodemap.check_data(ui, cl.index, data)			return nodemap.check_data(ui, cl.index, data)


	@command(			@command(
	b'debugobsolete',			b'debugobsolete',
	[			[
	(b'', b'flags', 0, _(b'markers flag')),			(b'', b'flags', 0, _(b'markers flag')),
	(			(
	b'',			b'',

mercurial/revlog.py

	self._chainbasecache = util.lrucachedict(100)			self._chainbasecache = util.lrucachedict(100)
	# 2-tuple of (offset, data) of raw data from the revlog at an offset.			# 2-tuple of (offset, data) of raw data from the revlog at an offset.
	self._chunkcache = (0, b'')			self._chunkcache = (0, b'')
	# How much data to read and cache into the raw revlog data cache.			# How much data to read and cache into the raw revlog data cache.
	self._chunkcachesize = 65536			self._chunkcachesize = 65536
	self._maxchainlen = None			self._maxchainlen = None
	self._deltabothparents = True			self._deltabothparents = True
	self.index = None			self.index = None
				self._nodemap_docket = None
	# Mapping of partial identifiers to full nodes.			# Mapping of partial identifiers to full nodes.
	self._pcache = {}			self._pcache = {}
	# Mapping of revision integer to full node.			# Mapping of revision integer to full node.
	self._compengine = b'zlib'			self._compengine = b'zlib'
	self._compengineopts = {}			self._compengineopts = {}
	self._maxdeltachainspan = -1			self._maxdeltachainspan = -1
	self._withsparseread = False			self._withsparseread = False
	self._sparserevlog = False			self._sparserevlog = False
	raise error.RevlogError(			raise error.RevlogError(
	_(b'revlog chunk cache size %r is not a power of 2')			_(b'revlog chunk cache size %r is not a power of 2')
	% self._chunkcachesize			% self._chunkcachesize
	)			)

	indexdata = b''			indexdata = b''
	self._initempty = True			self._initempty = True
	try:			try:
				nodemap_data = nodemaputil.persisted_data(self)
				if nodemap_data is not None:
				self._nodemap_docket = nodemap_data[0]
	with self._indexfp() as f:			with self._indexfp() as f:
	if (			if (
	mmapindexthreshold is not None			mmapindexthreshold is not None
	and self.opener.fstat(f).st_size >= mmapindexthreshold			and self.opener.fstat(f).st_size >= mmapindexthreshold
	):			):
	# TODO: should .close() to release resources without			# TODO: should .close() to release resources without
	# relying on Python GC			# relying on Python GC
	indexdata = util.buffer(util.mmapread(f))			indexdata = util.buffer(util.mmapread(f))
	use_nodemap = (			use_nodemap = (
	not self._inline			not self._inline
	and self.nodemap_file is not None			and self.nodemap_file is not None
	and util.safehasattr(index, 'update_nodemap_data')			and util.safehasattr(index, 'update_nodemap_data')
	)			)
	if use_nodemap:			if use_nodemap:
	nodemap_data = nodemaputil.persisted_data(self)			nodemap_data = nodemaputil.persisted_data(self)
	if nodemap_data is not None:			if nodemap_data is not None:
	index.update_nodemap_data(nodemap_data)			index.update_nodemap_data(nodemap_data[1])
	except (ValueError, IndexError):			except (ValueError, IndexError):
	raise error.RevlogError(			raise error.RevlogError(
	_(b"index %s is corrupted") % self.indexfile			_(b"index %s is corrupted") % self.indexfile
	)			)
	self.index, self._chunkcache = d			self.index, self._chunkcache = d
	if not self._chunkcache:			if not self._chunkcache:
	self._chunkclear()			self._chunkclear()
	# revnum -> (chain-length, sum-delta-length)			# revnum -> (chain-length, sum-delta-length)

mercurial/revlogutils/nodemap.py

	if version != ONDISK_VERSION:			if version != ONDISK_VERSION:
	return None			return None
	offset += S_VERSION.size			offset += S_VERSION.size
	(uid_size,) = S_HEADER.unpack(pdata[offset : offset + S_HEADER.size])			(uid_size,) = S_HEADER.unpack(pdata[offset : offset + S_HEADER.size])
	offset += S_HEADER.size			offset += S_HEADER.size
	docket = NodeMapDocket(pdata[offset : offset + uid_size])			docket = NodeMapDocket(pdata[offset : offset + uid_size])

	filename = _rawdata_filepath(revlog, docket)			filename = _rawdata_filepath(revlog, docket)
	return revlog.opener.tryread(filename)			return docket, revlog.opener.tryread(filename)


	def setup_persistent_nodemap(tr, revlog):			def setup_persistent_nodemap(tr, revlog):
	"""Install whatever is needed transaction side to persist a nodemap on disk			"""Install whatever is needed transaction side to persist a nodemap on disk

	(only actually persist the nodemap if this is relevant for this revlog)			(only actually persist the nodemap if this is relevant for this revlog)
	"""			"""
	if revlog._inline:			if revlog._inline:
	# EXP-TODO: if this is a cache, this should use a cache vfs, not a			# EXP-TODO: if this is a cache, this should use a cache vfs, not a
	# store vfs			# store vfs
	with revlog.opener(datafile, 'w') as fd:			with revlog.opener(datafile, 'w') as fd:
	fd.write(data)			fd.write(data)
	# EXP-TODO: if this is a cache, this should use a cache vfs, not a			# EXP-TODO: if this is a cache, this should use a cache vfs, not a
	# store vfs			# store vfs
	with revlog.opener(revlog.nodemap_file, 'w', atomictemp=True) as fp:			with revlog.opener(revlog.nodemap_file, 'w', atomictemp=True) as fp:
	fp.write(target_docket.serialize())			fp.write(target_docket.serialize())
				revlog._nodemap_docket = target_docket
	# EXP-TODO: if the transaction abort, we should remove the new data and			# EXP-TODO: if the transaction abort, we should remove the new data and
	# reinstall the old one.			# reinstall the old one.


	### Nodemap docket file			### Nodemap docket file
	#			#
	# The nodemap data are stored on disk using 2 files:			# The nodemap data are stored on disk using 2 files:
	#			#

This is an archive of the discontinued Mercurial Phabricator instance.

nodemap: keep track of the docket for loaded dataClosedPublic

Details

Diff Detail

Event Timeline

Revision ContentsChangeset List

Diff 19303

mercurial/debugcommands.py

mercurial/revlog.py

mercurial/revlogutils/nodemap.py

nodemap: keep track of the docket for loaded data
ClosedPublic

Revision Contents
Changeset List