This is an archive of the discontinued Mercurial Phabricator instance.

Differential D8191

nodemap: make sure on disk change get rolled back with the transaction
ClosedPublic

Authored by marmoute on Feb 28 2020, 1:55 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG01b0805534bb: nodemap: make sure on disk change get rolled back with the transaction
Summary

In case of errors, we need to rollback the change made to the persistent
nodemap.

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

marmoute created this revision.Feb 28 2020, 1:55 PM
marmoute updated this revision to Diff 20583.Mar 6 2020, 6:37 PM

eventless rebase past D8255

marmoute updated this revision to Diff 20686.Mar 10 2020, 9:36 PM

rebase above latest default up to landed-D8182

marmoute updated this revision to Diff 20742.Mar 11 2020, 2:20 PM
marmoute added a commit: rHG01b0805534bb: nodemap: make sure on disk change get rolled back with the transaction.Apr 1 2020, 11:21 AM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG01b0805534bb: nodemap: make sure on disk change get rolled back with the transaction (authored by marmoute). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 20742

mercurial/configitems.py

coreconfigitem( coreconfigitem(
b'devel', b'warn-empty-changegroup', default=False, b'devel', b'warn-empty-changegroup', default=False,
) )
coreconfigitem( coreconfigitem(
b'devel', b'legacy.exchange', default=list, b'devel', b'legacy.exchange', default=list,
) )
# TODO before getting `persistent-nodemap` out of experimental # TODO before getting `persistent-nodemap` out of experimental
# #
# * code/tests around aborted transaction
# * regenerate a new nodemap when the unused/total ration is to high # * regenerate a new nodemap when the unused/total ration is to high
# * decide for a "status" of the persistent nodemap and associated location # * decide for a "status" of the persistent nodemap and associated location
# - part of the store next the revlog itself (new requirements) # - part of the store next the revlog itself (new requirements)
# - part of the cache directory # - part of the cache directory
# - part of an `index` directory # - part of an `index` directory
# (https://www.mercurial-scm.org/wiki/ComputedIndexPlan) # (https://www.mercurial-scm.org/wiki/ComputedIndexPlan)
# * do we want to use this for more than just changelog? if so we need: # * do we want to use this for more than just changelog? if so we need:
# - simpler "pending" logic for them # - simpler "pending" logic for them

mercurial/revlogutils/nodemap.py

""" """
def __init__(self): def __init__(self):
self._postclose = {} self._postclose = {}
def addpostclose(self, callback_id, callback_func): def addpostclose(self, callback_id, callback_func):
self._postclose[callback_id] = callback_func self._postclose[callback_id] = callback_func
def registertmp(self, *args, **kwargs):
pass
def addbackup(self, *args, **kwargs):
pass
def add(self, *args, **kwargs):
pass
def update_persistent_nodemap(revlog): def update_persistent_nodemap(revlog):
"""update the persistent nodemap right now """update the persistent nodemap right now
To be used for updating the nodemap on disk outside of a normal transaction To be used for updating the nodemap on disk outside of a normal transaction
setup (eg, `debugupdatecache`). setup (eg, `debugupdatecache`).
""" """
notr = _NoTransaction() notr = _NoTransaction()
) = revlog.index.nodemap_data_incremental() ) = revlog.index.nodemap_data_incremental()
if src_docket != target_docket: if src_docket != target_docket:
data = None data = None
else: else:
datafile = _rawdata_filepath(revlog, target_docket) datafile = _rawdata_filepath(revlog, target_docket)
# EXP-TODO: if this is a cache, this should use a cache vfs, not a # EXP-TODO: if this is a cache, this should use a cache vfs, not a
# store vfs # store vfs
new_length = target_docket.data_length + len(data) new_length = target_docket.data_length + len(data)
tr.add(datafile, target_docket.data_length)
with revlog.opener(datafile, b'r+') as fd: with revlog.opener(datafile, b'r+') as fd:
fd.seek(target_docket.data_length) fd.seek(target_docket.data_length)
fd.write(data) fd.write(data)
if feed_data: if feed_data:
if use_mmap: if use_mmap:
fd.seek(0) fd.seek(0)
new_data = fd.read(new_length) new_data = fd.read(new_length)
else: else:
fd.flush() fd.flush()
new_data = util.buffer(util.mmapread(fd, new_length)) new_data = util.buffer(util.mmapread(fd, new_length))
target_docket.data_length = new_length target_docket.data_length = new_length
target_docket.data_unused += data_changed_count target_docket.data_unused += data_changed_count
if data is None: if data is None:
# otherwise fallback to a full new export # otherwise fallback to a full new export
target_docket = NodeMapDocket() target_docket = NodeMapDocket()
datafile = _rawdata_filepath(revlog, target_docket) datafile = _rawdata_filepath(revlog, target_docket)
if util.safehasattr(revlog.index, "nodemap_data_all"): if util.safehasattr(revlog.index, "nodemap_data_all"):
data = revlog.index.nodemap_data_all() data = revlog.index.nodemap_data_all()
else: else:
data = persistent_data(revlog.index) data = persistent_data(revlog.index)
# EXP-TODO: if this is a cache, this should use a cache vfs, not a # EXP-TODO: if this is a cache, this should use a cache vfs, not a
# store vfs # store vfs
tr.add(datafile, 0)
with revlog.opener(datafile, b'w+') as fd: with revlog.opener(datafile, b'w+') as fd:
fd.write(data) fd.write(data)
if feed_data: if feed_data:
if use_mmap: if use_mmap:
new_data = data new_data = data
else: else:
fd.flush() fd.flush()
new_data = util.buffer(util.mmapread(fd, len(data))) new_data = util.buffer(util.mmapread(fd, len(data)))
target_docket.data_length = len(data) target_docket.data_length = len(data)
target_docket.tip_rev = revlog.tiprev() target_docket.tip_rev = revlog.tiprev()
target_docket.tip_node = revlog.node(target_docket.tip_rev) target_docket.tip_node = revlog.node(target_docket.tip_rev)
# EXP-TODO: if this is a cache, this should use a cache vfs, not a # EXP-TODO: if this is a cache, this should use a cache vfs, not a
# store vfs # store vfs
file_path = revlog.nodemap_file file_path = revlog.nodemap_file
if pending: if pending:
file_path += b'.a' file_path += b'.a'
tr.registertmp(file_path)
else:
tr.addbackup(file_path)
with revlog.opener(file_path, b'w', atomictemp=True) as fp: with revlog.opener(file_path, b'w', atomictemp=True) as fp:
fp.write(target_docket.serialize()) fp.write(target_docket.serialize())
revlog._nodemap_docket = target_docket revlog._nodemap_docket = target_docket
if feed_data: if feed_data:
revlog.index.update_nodemap_data(target_docket, new_data) revlog.index.update_nodemap_data(target_docket, new_data)
# EXP-TODO: if the transaction abort, we should remove the new data and # EXP-TODO: if the transaction abort, we should remove the new data and
# reinstall the old one. # reinstall the old one.

tests/test-persistent-nodemap.t

data-length: 123584 (rust !) data-length: 123584 (rust !)
data-length: 123136 (no-pure no-rust !) data-length: 123136 (no-pure no-rust !)
data-unused: 448 (pure !) data-unused: 448 (pure !)
data-unused: 448 (rust !) data-unused: 448 (rust !)
data-unused: 0 (no-pure no-rust !) data-unused: 0 (no-pure no-rust !)
$ cat output.txt $ cat output.txt
Check that a failing transaction will properly revert the data
$ echo plakfe > a
$ f --size --sha256 .hg/store/00changelog-*.nd
.hg/store/00changelog-????????????????.nd: size=123584, sha256=8c6cef6fd3d3fac291968793ee19a4be6d0b8375e9508bd5c7d4a8879e8df180 (glob) (pure !)
.hg/store/00changelog-????????????????.nd: size=123584, sha256=eb9e9a4bcafdb5e1344bc8a0cbb3288b2106413b8efae6265fb8a7973d7e97f9 (glob) (rust !)
.hg/store/00changelog-????????????????.nd: size=123136, sha256=4f504f5a834db3811ced50ab3e9e80bcae3581bb0f9b13a7a9f94b7fc34bcebe (glob) (no-pure no-rust !)
$ hg ci -m a3 --config "extensions.abort=$RUNTESTDIR/testlib/crash_transaction_late.py"
transaction abort!
rollback completed
abort: This is a late abort
[255]
$ hg debugnodemap --metadata
uid: ???????????????? (glob)
tip-rev: 5005
tip-node: bae4d45c759e30f1cb1a40e1382cf0e0414154db
data-length: 123584
data-unused: 448
$ f --size --sha256 .hg/store/00changelog-*.nd
.hg/store/00changelog-????????????????.nd: size=123584, sha256=8c6cef6fd3d3fac291968793ee19a4be6d0b8375e9508bd5c7d4a8879e8df180 (glob) (pure !)
.hg/store/00changelog-????????????????.nd: size=123584, sha256=eb9e9a4bcafdb5e1344bc8a0cbb3288b2106413b8efae6265fb8a7973d7e97f9 (glob) (rust !)
.hg/store/00changelog-????????????????.nd: size=123136, sha256=4f504f5a834db3811ced50ab3e9e80bcae3581bb0f9b13a7a9f94b7fc34bcebe (glob) (no-pure no-rust !)

tests/testlib/crash_transaction_late.py

  • This file was added.
# tiny extension to abort a transaction very late during test
#
# Copyright 2020 Pierre-Yves David <pierre-yves.david@octobus.net>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
from __future__ import absolute_import
from mercurial import (
error,
transaction,
)
def abort(fp):
raise error.Abort(b"This is a late abort")
def reposetup(ui, repo):
transaction.postfinalizegenerators.add(b'late-abort')
class LateAbortRepo(repo.__class__):
def transaction(self, *args, **kwargs):
tr = super(LateAbortRepo, self).transaction(*args, **kwargs)
tr.addfilegenerator(
b'late-abort', [b'late-abort'], abort, order=9999999
)
return tr
repo.__class__ = LateAbortRepo