( )⚙ D10629 revlogv2: delay the update of the changelog docket to transaction end

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10629

revlogv2: delay the update of the changelog docket to transaction end
ClosedPublic

Authored by marmoute on May 3 2021, 7:54 AM.

Details

Reviewers
indygreg
Alphare
Group Reviewers
hg-reviewers
Commits
rHG682f09857d69: revlogv2: delay the update of the changelog docket to transaction end
Summary

This prevent external reader to see the transaction content before it is
commited. However this also prevent the hooks to see the transaction content.
We will fix this in later changesets.

We have to temporarily suppress the error output of the command ran during the
transaction as they sometimes get confused about unknown working directory and
sometimes issue message on std-err in unspecified order.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

marmoute created this revision.May 3 2021, 7:54 AM
baymax updated this revision to Diff 27472.May 3 2021, 9:32 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

marmoute updated this revision to Diff 27545.May 4 2021, 10:13 AM
baymax updated this revision to Diff 27875.May 11 2021, 1:44 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

Alphare accepted this revision.May 17 2021, 5:48 PM
This revision is now accepted and ready to land.May 17 2021, 5:48 PM

Revision Contents
Changeset List

Diff 28029

mercurial/changelog.py

@filteredrevs.setter @filteredrevs.setter
def filteredrevs(self, val): def filteredrevs(self, val):
# Ensure all updates go through this function # Ensure all updates go through this function
assert isinstance(val, frozenset) assert isinstance(val, frozenset)
self._filteredrevs = val self._filteredrevs = val
self._filteredrevs_hashcache = {} self._filteredrevs_hashcache = {}
def _write_docket(self, tr):
if not self._delayed:
super(changelog, self)._write_docket(tr)
def delayupdate(self, tr): def delayupdate(self, tr):
"""delay visibility of index updates to other readers""" """delay visibility of index updates to other readers"""
if self._docket is not None: if self._docket is None and not self._delayed:
return
if not self._delayed:
if len(self) == 0: if len(self) == 0:
self._divert = True self._divert = True
if self._realopener.exists(self._indexfile + b'.a'): if self._realopener.exists(self._indexfile + b'.a'):
self._realopener.unlink(self._indexfile + b'.a') self._realopener.unlink(self._indexfile + b'.a')
self.opener = _divertopener(self._realopener, self._indexfile) self.opener = _divertopener(self._realopener, self._indexfile)
else: else:
self._delaybuf = [] self._delaybuf = []
self.opener = _delayopener( self.opener = _delayopener(
self._realopener, self._indexfile, self._delaybuf self._realopener, self._indexfile, self._delaybuf
) )
self._delayed = True self._delayed = True
tr.addpending(b'cl-%i' % id(self), self._writepending) tr.addpending(b'cl-%i' % id(self), self._writepending)
tr.addfinalize(b'cl-%i' % id(self), self._finalize) tr.addfinalize(b'cl-%i' % id(self), self._finalize)
def _finalize(self, tr): def _finalize(self, tr):
"""finalize index updates""" """finalize index updates"""
self._delayed = False self._delayed = False
self.opener = self._realopener self.opener = self._realopener
# move redirected index data back into place # move redirected index data back into place
if self._divert: if self._docket is not None:
self._write_docket(tr)
elif self._divert:
assert not self._delaybuf assert not self._delaybuf
tmpname = self._indexfile + b".a" tmpname = self._indexfile + b".a"
nfile = self.opener.open(tmpname) nfile = self.opener.open(tmpname)
nfile.close() nfile.close()
self.opener.rename(tmpname, self._indexfile, checkambig=True) self.opener.rename(tmpname, self._indexfile, checkambig=True)
elif self._delaybuf: elif self._delaybuf:
fp = self.opener(self._indexfile, b'a', checkambig=True) fp = self.opener(self._indexfile, b'a', checkambig=True)
fp.write(b"".join(self._delaybuf)) fp.write(b"".join(self._delaybuf))

mercurial/configitems.py

) )
coreconfigitem( coreconfigitem(
b'experimental', b'experimental',
b'revisions.prefixhexnode', b'revisions.prefixhexnode',
default=False, default=False,
) )
# "out of experimental" todo list. # "out of experimental" todo list.
# #
# * properly hide uncommitted content to other process
# * expose transaction content hooks during pre-commit validation # * expose transaction content hooks during pre-commit validation
# * include management of a persistent nodemap in the main docket # * include management of a persistent nodemap in the main docket
# * enforce a "no-truncate" policy for mmap safety # * enforce a "no-truncate" policy for mmap safety
# - for censoring operation # - for censoring operation
# - for stripping operation # - for stripping operation
# - for rollback operation # - for rollback operation
# * proper streaming (race free) of the docket file # * proper streaming (race free) of the docket file
# * store the data size in the docket to simplify sidedata rewrite. # * store the data size in the docket to simplify sidedata rewrite.

mercurial/revlog.py

transaction.add(self._indexfile, dsize + isize) transaction.add(self._indexfile, dsize + isize)
else: else:
transaction.add(self._indexfile, isize) transaction.add(self._indexfile, isize)
try: try:
self._writinghandles = (ifh, dfh) self._writinghandles = (ifh, dfh)
try: try:
yield yield
if self._docket is not None: if self._docket is not None:
self._docket.write(transaction) self._write_docket(transaction)
finally: finally:
self._writinghandles = None self._writinghandles = None
finally: finally:
ifh.close() ifh.close()
finally: finally:
if dfh is not None: if dfh is not None:
dfh.close() dfh.close()
def _write_docket(self, transaction):
"""write the current docket on disk
Exist as a method to help changelog to implement transaction logic
We could also imagine using the same transaction logic for all revlog
since docket are cheap."""
self._docket.write(transaction)
def addrevision( def addrevision(
self, self,
text, text,
transaction, transaction,
link, link,
p1, p1,
p2, p2,
cachedelta=None, cachedelta=None,
if not self.hassidedata: if not self.hassidedata:
return return
# revlog formats with sidedata support does not support inline # revlog formats with sidedata support does not support inline
assert not self._inline assert not self._inline
if not helpers[1] and not helpers[2]: if not helpers[1] and not helpers[2]:
# Nothing to generate or remove # Nothing to generate or remove
return return
# changelog implement some "delayed" writing mechanism that assume that
# all index data is writen in append mode and is therefor incompatible
# with the seeked write done in this method. The use of such "delayed"
# writing will soon be removed for revlog version that support side
# data, so for now, we only keep this simple assert to highlight the
# situation.
delayed = getattr(self, '_delayed', False)
diverted = getattr(self, '_divert', False)
if delayed and not diverted:
msg = "cannot rewrite_sidedata of a delayed revlog"
raise error.ProgrammingError(msg)
new_entries = [] new_entries = []
# append the new sidedata # append the new sidedata
with self._writing(transaction): with self._writing(transaction):
ifh, dfh = self._writinghandles ifh, dfh = self._writinghandles
dfh.seek(0, os.SEEK_END) dfh.seek(0, os.SEEK_END)
current_offset = dfh.tell() current_offset = dfh.tell()
for rev in range(startrev, endrev + 1): for rev in range(startrev, endrev + 1):
entry = self.index[rev] entry = self.index[rev]

tests/test-transaction-safety.t

$ export HG_TEST_FILE_EXT_WAITING $ export HG_TEST_FILE_EXT_WAITING
$ HG_TEST_FILE_EXT_UNLOCK=$TESTTMP/sync/ext_unlock $ HG_TEST_FILE_EXT_UNLOCK=$TESTTMP/sync/ext_unlock
$ export HG_TEST_FILE_EXT_UNLOCK $ export HG_TEST_FILE_EXT_UNLOCK
$ HG_TEST_FILE_EXT_DONE=$TESTTMP/sync/ext_done $ HG_TEST_FILE_EXT_DONE=$TESTTMP/sync/ext_done
$ export HG_TEST_FILE_EXT_DONE $ export HG_TEST_FILE_EXT_DONE
$ cat << EOF > script/external.sh $ cat << EOF > script/external.sh
> #!/bin/sh > #!/bin/sh
> $RUNTESTDIR/testlib/wait-on-file 5 $HG_TEST_FILE_EXT_UNLOCK $HG_TEST_FILE_EXT_WAITING > $RUNTESTDIR/testlib/wait-on-file 5 $HG_TEST_FILE_EXT_UNLOCK $HG_TEST_FILE_EXT_WAITING
> hg log --rev 'tip' -T 'external: {rev} {desc}\n' > $TESTTMP/output/external.out > hg log --rev 'tip' -T 'external: {rev} {desc}\n' > $TESTTMP/output/external.out 2>/dev/null
> touch $HG_TEST_FILE_EXT_DONE > touch $HG_TEST_FILE_EXT_DONE
> EOF > EOF
$ chmod +x script/external.sh $ chmod +x script/external.sh
$ cat << EOF > script/internal.sh $ cat << EOF > script/internal.sh
> #!/bin/sh > #!/bin/sh
> hg log --rev 'tip' -T 'internal: {rev} {desc}\n' > $TESTTMP/output/internal.out > hg log --rev 'tip' -T 'internal: {rev} {desc}\n' > $TESTTMP/output/internal.out 2>/dev/null
> $RUNTESTDIR/testlib/wait-on-file 5 $HG_TEST_FILE_EXT_DONE $HG_TEST_FILE_EXT_UNLOCK > $RUNTESTDIR/testlib/wait-on-file 5 $HG_TEST_FILE_EXT_DONE $HG_TEST_FILE_EXT_UNLOCK
> EOF > EOF
$ chmod +x script/internal.sh $ chmod +x script/internal.sh
Automated commands: Automated commands:
$ make_one_commit() { $ make_one_commit() {
check this is true for the initial commit (inline → inline) check this is true for the initial commit (inline → inline)
----------------------------------------------------------- -----------------------------------------------------------
the repository should still be inline (for relevant format) the repository should still be inline (for relevant format)
$ make_one_commit first $ make_one_commit first
pre-commit: -1 pre-commit: -1
external: -1 (revlogv1 !) external: -1
external: 0 first (revlogv2 known-bad-output !) internal: 0 first (revlogv1 !)
internal: 0 first internal: -1 (revlogv2 known-bad-output !)
post-tr: 0 first post-tr: 0 first
#if revlogv1 #if revlogv1
$ hg debugrevlog -c | grep inline $ hg debugrevlog -c | grep inline
flags : inline flags : inline
#endif #endif
check this is true for extra commit (inline → inline) check this is true for extra commit (inline → inline)
----------------------------------------------------- -----------------------------------------------------
the repository should still be inline (for relevant format) the repository should still be inline (for relevant format)
#if revlogv1 #if revlogv1
$ hg debugrevlog -c | grep inline $ hg debugrevlog -c | grep inline
flags : inline flags : inline
#endif #endif
$ make_one_commit second $ make_one_commit second
pre-commit: 0 first pre-commit: 0 first
external: 0 first (revlogv1 !) external: 0 first
external: 1 second (revlogv2 known-bad-output !) internal: 1 second (revlogv1 !)
internal: 1 second internal: 0 first (revlogv2 known-bad-output !)
post-tr: 1 second post-tr: 1 second
#if revlogv1 #if revlogv1
$ hg debugrevlog -c | grep inline $ hg debugrevlog -c | grep inline
flags : inline flags : inline
#endif #endif
check this is true for a small pull (inline → inline) check this is true for a small pull (inline → inline)
----------------------------------------------------- -----------------------------------------------------
the repository should still be inline (for relevant format) the repository should still be inline (for relevant format)
#if revlogv1 #if revlogv1
$ hg debugrevlog -c | grep inline $ hg debugrevlog -c | grep inline
flags : inline flags : inline
#endif #endif
$ make_one_pull 3 $ make_one_pull 3
pre-commit: 1 second pre-commit: 1 second
warning: repository is unrelated warning: repository is unrelated
external: 1 second (revlogv1 !) external: 1 second
external: 5 r3 (revlogv2 known-bad-output !) internal: 5 r3 (revlogv1 !)
internal: 5 r3 internal: 1 second (revlogv2 known-bad-output !)
post-tr: 5 r3 post-tr: 5 r3
#if revlogv1 #if revlogv1
$ hg debugrevlog -c | grep inline $ hg debugrevlog -c | grep inline
flags : inline flags : inline
#endif #endif
Make a large pull (inline → no-inline) Make a large pull (inline → no-inline)
--------------------------------------- ---------------------------------------
the repository should no longer be inline (for relevant format) the repository should no longer be inline (for relevant format)
#if revlogv1 #if revlogv1
$ hg debugrevlog -c | grep inline $ hg debugrevlog -c | grep inline
flags : inline flags : inline
#endif #endif
$ make_one_pull 400 $ make_one_pull 400
pre-commit: 5 r3 pre-commit: 5 r3
external: 5 r3 (revlogv1 !) external: 5 r3
external: 402 r400 (revlogv2 known-bad-output !) internal: 402 r400 (revlogv1 !)
internal: 402 r400 internal: 5 r3 (revlogv2 known-bad-output !)
post-tr: 402 r400 post-tr: 402 r400
#if revlogv1 #if revlogv1
$ hg debugrevlog -c | grep inline $ hg debugrevlog -c | grep inline
[1] [1]
#endif #endif
check this is true for extra commit (no-inline → no-inline) check this is true for extra commit (no-inline → no-inline)
----------------------------------------------------------- -----------------------------------------------------------
the repository should no longer be inline (for relevant format) the repository should no longer be inline (for relevant format)
#if revlogv1 #if revlogv1
$ hg debugrevlog -c | grep inline $ hg debugrevlog -c | grep inline
[1] [1]
#endif #endif
$ make_one_commit third $ make_one_commit third
pre-commit: 402 r400 pre-commit: 402 r400
external: 402 r400 (revlogv1 !) external: 402 r400
external: 403 third (revlogv2 known-bad-output !) internal: 403 third (revlogv1 !)
internal: 403 third internal: 402 r400 (revlogv2 known-bad-output !)
post-tr: 403 third post-tr: 403 third
#if revlogv1 #if revlogv1
$ hg debugrevlog -c | grep inline $ hg debugrevlog -c | grep inline
[1] [1]
#endif #endif
Make a pull (not-inline → no-inline) Make a pull (not-inline → no-inline)
------------------------------------- -------------------------------------
the repository should no longer be inline (for relevant format) the repository should no longer be inline (for relevant format)
#if revlogv1 #if revlogv1
$ hg debugrevlog -c | grep inline $ hg debugrevlog -c | grep inline
[1] [1]
#endif #endif
$ make_one_pull tip $ make_one_pull tip
pre-commit: 403 third pre-commit: 403 third
external: 403 third (revlogv1 !) external: 403 third
external: 503 r500 (revlogv2 known-bad-output !) internal: 503 r500 (revlogv1 !)
internal: 503 r500 internal: 403 third (revlogv2 known-bad-output !)
post-tr: 503 r500 post-tr: 503 r500
#if revlogv1 #if revlogv1
$ hg debugrevlog -c | grep inline $ hg debugrevlog -c | grep inline
[1] [1]
#endif #endif