This is an archive of the discontinued Mercurial Phabricator instance.

Differential D7890

nodemap: double check the source docket when doing incremental update
ClosedPublic

Authored by marmoute on Jan 15 2020, 9:56 AM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG1d2b37def017: nodemap: double check the source docket when doing incremental update
Summary

In theory, the index will have the information we expect it to have. However by
security, it seems safer to double check that the incremental data are generated
from the data currently on disk.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

marmoute created this revision.Jan 15 2020, 9:56 AM
marmoute retitled this revision from nodemapdata: double check the source docket when doing incremental update to nodemap: double check the source docket when doing incremental update.Jan 15 2020, 1:09 PM
marmoute updated this revision to Diff 19771.Jan 31 2020, 10:29 AM
marmoute updated this revision to Diff 19798.Jan 31 2020, 6:00 PM
marmoute updated this revision to Diff 19842.Feb 2 2020, 3:19 AM

small doc update on .#s[1]

marmoute updated this revision to Diff 19902.Feb 4 2020, 7:25 PM

rebase to latest default

marmoute added a commit: rHG1d2b37def017: nodemap: double check the source docket when doing incremental update.Feb 10 2020, 8:28 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG1d2b37def017: nodemap: double check the source docket when doing incremental update (authored by marmoute). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 20123

mercurial/pure/parsers.py

def nodemap_data_incremental(self): def nodemap_data_incremental(self):
"""Return bytes containing a incremental update to persistent nodemap """Return bytes containing a incremental update to persistent nodemap
This containst the data for an append-only update of the data provided This containst the data for an append-only update of the data provided
in the last call to `update_nodemap_data`. in the last call to `update_nodemap_data`.
""" """
if self._nm_root is None: if self._nm_root is None:
return None return None
docket = self._nm_docket
changed, data = nodemaputil.update_persistent_data( changed, data = nodemaputil.update_persistent_data(
self, self._nm_root, self._nm_max_idx, self._nm_rev self, self._nm_root, self._nm_max_idx, self._nm_docket.tip_rev
) )
self._nm_root = self._nm_max_idx = self._nm_rev = None
return changed, data self._nm_root = self._nm_max_idx = self._nm_docket = None
return docket, changed, data
def update_nodemap_data(self, docket, nm_data): def update_nodemap_data(self, docket, nm_data):
"""provide full block of persisted binary data for a nodemap """provide full block of persisted binary data for a nodemap
The data are expected to come from disk. See `nodemap_data_all` for a The data are expected to come from disk. See `nodemap_data_all` for a
produceur of such data.""" produceur of such data."""
if nm_data is not None: if nm_data is not None:
self._nm_root, self._nm_max_idx = nodemaputil.parse_data(nm_data) self._nm_root, self._nm_max_idx = nodemaputil.parse_data(nm_data)
if self._nm_root: if self._nm_root:
self._nm_rev = docket.tip_rev self._nm_docket = docket
else: else:
self._nm_root = self._nm_max_idx = self._nm_rev = None self._nm_root = self._nm_max_idx = self._nm_docket = None
class InlinedIndexObject(BaseIndexObject): class InlinedIndexObject(BaseIndexObject):
def __init__(self, data, inline=0): def __init__(self, data, inline=0):
self._data = data self._data = data
self._lgt = self._inline_scan(None) self._lgt = self._inline_scan(None)
self._inline_scan(self._lgt) self._inline_scan(self._lgt)
self._extra = [] self._extra = []

mercurial/revlogutils/nodemap.py

) )
if revlog.nodemap_file is None: if revlog.nodemap_file is None:
msg = "calling persist nodemap on a revlog without the feature enableb" msg = "calling persist nodemap on a revlog without the feature enableb"
raise error.ProgrammingError(msg) raise error.ProgrammingError(msg)
can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental") can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental")
ondisk_docket = revlog._nodemap_docket ondisk_docket = revlog._nodemap_docket
data = None
# first attemp an incremental update of the data # first attemp an incremental update of the data
if can_incremental and ondisk_docket is not None: if can_incremental and ondisk_docket is not None:
target_docket = revlog._nodemap_docket.copy() target_docket = revlog._nodemap_docket.copy()
data_changed_count, data = revlog.index.nodemap_data_incremental() (
src_docket,
data_changed_count,
data,
) = revlog.index.nodemap_data_incremental()
if src_docket != target_docket:
data = None
else:
datafile = _rawdata_filepath(revlog, target_docket) datafile = _rawdata_filepath(revlog, target_docket)
# EXP-TODO: if this is a cache, this should use a cache vfs, not a # EXP-TODO: if this is a cache, this should use a cache vfs, not a
# store vfs # store vfs
with revlog.opener(datafile, b'a') as fd: with revlog.opener(datafile, b'a') as fd:
fd.write(data) fd.write(data)
target_docket.data_length += len(data) target_docket.data_length += len(data)
target_docket.data_unused += data_changed_count target_docket.data_unused += data_changed_count
else:
if data is None:
# otherwise fallback to a full new export # otherwise fallback to a full new export
target_docket = NodeMapDocket() target_docket = NodeMapDocket()
datafile = _rawdata_filepath(revlog, target_docket) datafile = _rawdata_filepath(revlog, target_docket)
if util.safehasattr(revlog.index, "nodemap_data_all"): if util.safehasattr(revlog.index, "nodemap_data_all"):
data = revlog.index.nodemap_data_all() data = revlog.index.nodemap_data_all()
else: else:
data = persistent_data(revlog.index) data = persistent_data(revlog.index)
# EXP-TODO: if this is a cache, this should use a cache vfs, not a # EXP-TODO: if this is a cache, this should use a cache vfs, not a
def copy(self): def copy(self):
new = NodeMapDocket(uid=self.uid) new = NodeMapDocket(uid=self.uid)
new.tip_rev = self.tip_rev new.tip_rev = self.tip_rev
new.data_length = self.data_length new.data_length = self.data_length
new.data_unused = self.data_unused new.data_unused = self.data_unused
return new return new
def __cmp__(self, other):
if self.uid < other.uid:
return -1
if self.uid > other.uid:
return 1
elif self.data_length < other.data_length:
return -1
elif self.data_length > other.data_length:
return 1
return 0
def __eq__(self, other):
return self.uid == other.uid and self.data_length == other.data_length
def serialize(self): def serialize(self):
"""return serialized bytes for a docket using the passed uid""" """return serialized bytes for a docket using the passed uid"""
data = [] data = []
data.append(S_VERSION.pack(ONDISK_VERSION)) data.append(S_VERSION.pack(ONDISK_VERSION))
headers = ( headers = (
len(self.uid), len(self.uid),
self.tip_rev, self.tip_rev,
self.data_length, self.data_length,