| indygreg | |
| Alphare |
| hg-reviewers |
The docket is now able to write pending data. We could have used a distinct
intermediate files, however keeping everything in the same file will make it
simpler to keep track of the various involved files if necessary.
However it might prove more complicated for streaming clone. This will be dealt
with later.
Note that we lifted the stderr redirection in the test since we no longer suffer
from "unkown working directory parent" message.
| Automatic diff as part of commit; lint not applicable. |
| Automatic diff as part of commit; unit tests not applicable. |
| Path | Packages | |||
|---|---|---|---|---|
| M | mercurial/changelog.py (3 lines) | |||
| M | mercurial/configitems.py (1 line) | |||
| M | mercurial/revlog.py (4 lines) | |||
| M | mercurial/revlogutils/docket.py (58 lines) | |||
| M | tests/test-transaction-safety.t (22 lines) |
| The ``00changelog.i.a`` file contains index (and possibly inline | The ``00changelog.i.a`` file contains index (and possibly inline | ||||
| revision) data for a transaction that hasn't been finalized yet. | revision) data for a transaction that hasn't been finalized yet. | ||||
| It exists in a separate file to facilitate readers (such as | It exists in a separate file to facilitate readers (such as | ||||
| hooks processes) accessing data before a transaction is finalized. | hooks processes) accessing data before a transaction is finalized. | ||||
| ``concurrencychecker`` will be passed to the revlog init function, see | ``concurrencychecker`` will be passed to the revlog init function, see | ||||
| the documentation there. | the documentation there. | ||||
| """ | """ | ||||
| revlog.revlog.__init__( | revlog.revlog.__init__( | ||||
| self, | self, | ||||
| opener, | opener, | ||||
| target=(revlog_constants.KIND_CHANGELOG, None), | target=(revlog_constants.KIND_CHANGELOG, None), | ||||
| radix=b'00changelog', | radix=b'00changelog', | ||||
| checkambig=True, | checkambig=True, | ||||
| mmaplargeindex=True, | mmaplargeindex=True, | ||||
| persistentnodemap=opener.options.get(b'persistent-nodemap', False), | persistentnodemap=opener.options.get(b'persistent-nodemap', False), | ||||
| self._delaybuf = None | self._delaybuf = None | ||||
| self._divert = False | self._divert = False | ||||
| # split when we're done | # split when we're done | ||||
| self._enforceinlinesize(tr) | self._enforceinlinesize(tr) | ||||
| def _writepending(self, tr): | def _writepending(self, tr): | ||||
| """create a file containing the unfinalized state for | """create a file containing the unfinalized state for | ||||
| pretxnchangegroup""" | pretxnchangegroup""" | ||||
| if self._docket: | |||||
| return self._docket.write(tr, pending=True) | |||||
| if self._delaybuf: | if self._delaybuf: | ||||
| # make a temporary copy of the index | # make a temporary copy of the index | ||||
| fp1 = self._realopener(self._indexfile) | fp1 = self._realopener(self._indexfile) | ||||
| pendingfilename = self._indexfile + b".a" | pendingfilename = self._indexfile + b".a" | ||||
| # register as a temp file to ensure cleanup on failure | # register as a temp file to ensure cleanup on failure | ||||
| tr.registertmp(pendingfilename) | tr.registertmp(pendingfilename) | ||||
| # write existing data | # write existing data | ||||
| fp2 = self._realopener(pendingfilename, b"w") | fp2 = self._realopener(pendingfilename, b"w") | ||||
| ) | ) | ||||
| coreconfigitem( | coreconfigitem( | ||||
| b'experimental', | b'experimental', | ||||
| b'revisions.prefixhexnode', | b'revisions.prefixhexnode', | ||||
| default=False, | default=False, | ||||
| ) | ) | ||||
| # "out of experimental" todo list. | # "out of experimental" todo list. | ||||
| # | # | ||||
| # * expose transaction content hooks during pre-commit validation | |||||
| # * include management of a persistent nodemap in the main docket | # * include management of a persistent nodemap in the main docket | ||||
| # * enforce a "no-truncate" policy for mmap safety | # * enforce a "no-truncate" policy for mmap safety | ||||
| # - for censoring operation | # - for censoring operation | ||||
| # - for stripping operation | # - for stripping operation | ||||
| # - for rollback operation | # - for rollback operation | ||||
| # * proper streaming (race free) of the docket file | # * proper streaming (race free) of the docket file | ||||
| # * store the data size in the docket to simplify sidedata rewrite. | # * store the data size in the docket to simplify sidedata rewrite. | ||||
| # * track garbage data to evemtually allow rewriting -existing- sidedata. | # * track garbage data to evemtually allow rewriting -existing- sidedata. | ||||
| if not features[b'docket']: | if not features[b'docket']: | ||||
| self._indexfile = entry_point | self._indexfile = entry_point | ||||
| index_data = entry_data | index_data = entry_data | ||||
| else: | else: | ||||
| self._docket_file = entry_point | self._docket_file = entry_point | ||||
| if self._initempty: | if self._initempty: | ||||
| self._docket = docketutil.default_docket(self, header) | self._docket = docketutil.default_docket(self, header) | ||||
| else: | else: | ||||
| self._docket = docketutil.parse_docket(self, entry_data) | self._docket = docketutil.parse_docket( | ||||
| self, entry_data, use_pending=self._trypending | |||||
| ) | |||||
| self._indexfile = self._docket.index_filepath() | self._indexfile = self._docket.index_filepath() | ||||
| index_data = b'' | index_data = b'' | ||||
| index_size = self._docket.index_end | index_size = self._docket.index_end | ||||
| if index_size > 0: | if index_size > 0: | ||||
| index_data = self._get_data( | index_data = self._get_data( | ||||
| self._indexfile, mmapindexthreshold, size=index_size | self._indexfile, mmapindexthreshold, size=index_size | ||||
| ) | ) | ||||
| if len(index_data) < index_size: | if len(index_data) < index_size: | ||||
| # * an index file, containing fixed width information about revisions, | # * an index file, containing fixed width information about revisions, | ||||
| # | # | ||||
| # * a data file, containing variable width data for these revisions, | # * a data file, containing variable width data for these revisions, | ||||
| from __future__ import absolute_import | from __future__ import absolute_import | ||||
| import struct | import struct | ||||
| from .. import ( | |||||
| error, | |||||
| ) | |||||
| from . import ( | from . import ( | ||||
| constants, | constants, | ||||
| ) | ) | ||||
| # Docket format | # Docket format | ||||
| # | # | ||||
| # * 4 bytes: revlog version | # * 4 bytes: revlog version | ||||
| # | This is mandatory as docket must be compatible with the previous | # | This is mandatory as docket must be compatible with the previous | ||||
| # | revlog index header. | # | revlog index header. | ||||
| # * 8 bytes: size of index data | # * 8 bytes: size of index data | ||||
| S_HEADER = struct.Struct(constants.INDEX_HEADER.format + 'L') | # * 8 bytes: pending size of index data | ||||
| S_HEADER = struct.Struct(constants.INDEX_HEADER.format + 'LL') | |||||
| class RevlogDocket(object): | class RevlogDocket(object): | ||||
| """metadata associated with revlog""" | """metadata associated with revlog""" | ||||
| def __init__(self, revlog, version_header=None, index_end=0): | def __init__( | ||||
| self, | |||||
| revlog, | |||||
| use_pending=False, | |||||
| version_header=None, | |||||
| index_end=0, | |||||
| pending_index_end=0, | |||||
| ): | |||||
| self._version_header = version_header | self._version_header = version_header | ||||
| self._read_only = bool(use_pending) | |||||
| self._dirty = False | self._dirty = False | ||||
| self._radix = revlog.radix | self._radix = revlog.radix | ||||
| self._path = revlog._docket_file | self._path = revlog._docket_file | ||||
| self._opener = revlog.opener | self._opener = revlog.opener | ||||
| self._index_end = index_end | # this assert should be True as long as we have a single index filename | ||||
| assert index_end <= pending_index_end | |||||
| self._initial_index_end = index_end | |||||
| self._pending_index_end = pending_index_end | |||||
| if use_pending: | |||||
| self._index_end = self._pending_index_end | |||||
| else: | |||||
| self._index_end = self._initial_index_end | |||||
| def index_filepath(self): | def index_filepath(self): | ||||
| """file path to the current index file associated to this docket""" | """file path to the current index file associated to this docket""" | ||||
| # very simplistic version at first | # very simplistic version at first | ||||
| return b"%s.idx" % self._radix | return b"%s.idx" % self._radix | ||||
| @property | @property | ||||
| def index_end(self): | def index_end(self): | ||||
| return self._index_end | return self._index_end | ||||
| @index_end.setter | @index_end.setter | ||||
| def index_end(self, new_size): | def index_end(self, new_size): | ||||
| if new_size != self._index_end: | if new_size != self._index_end: | ||||
| self._index_end = new_size | self._index_end = new_size | ||||
| self._dirty = True | self._dirty = True | ||||
| def write(self, transaction, stripping=False): | def write(self, transaction, pending=False, stripping=False): | ||||
| """write the modification of disk if any | """write the modification of disk if any | ||||
| This make the new content visible to all process""" | This make the new content visible to all process""" | ||||
| if self._dirty: | if not self._dirty: | ||||
| return False | |||||
| else: | |||||
| if self._read_only: | |||||
| msg = b'writing read-only docket: %s' | |||||
| msg %= self._path | |||||
| raise error.ProgrammingError(msg) | |||||
| if not stripping: | if not stripping: | ||||
| # XXX we could, leverage the docket while stripping. However it | # XXX we could, leverage the docket while stripping. However it | ||||
| # is not powerfull enough at the time of this comment | # is not powerfull enough at the time of this comment | ||||
| transaction.addbackup(self._path, location=b'store') | transaction.addbackup(self._path, location=b'store') | ||||
| with self._opener(self._path, mode=b'w', atomictemp=True) as f: | with self._opener(self._path, mode=b'w', atomictemp=True) as f: | ||||
| f.write(self._serialize()) | f.write(self._serialize(pending=pending)) | ||||
| self._dirty = False | # if pending we still need to the write final data eventually | ||||
| self._dirty = pending | |||||
| return True | |||||
| def _serialize(self, pending=False): | |||||
| if pending: | |||||
| official_index_end = self._initial_index_end | |||||
| else: | |||||
| official_index_end = self._index_end | |||||
| def _serialize(self): | # this assert should be True as long as we have a single index filename | ||||
| assert official_index_end <= self._index_end | |||||
| data = ( | data = ( | ||||
| self._version_header, | self._version_header, | ||||
| official_index_end, | |||||
| self._index_end, | self._index_end, | ||||
| ) | ) | ||||
| return S_HEADER.pack(*data) | return S_HEADER.pack(*data) | ||||
| def default_docket(revlog, version_header): | def default_docket(revlog, version_header): | ||||
| """given a revlog version a new docket object for the given revlog""" | """given a revlog version a new docket object for the given revlog""" | ||||
| if (version_header & 0xFFFF) != constants.REVLOGV2: | if (version_header & 0xFFFF) != constants.REVLOGV2: | ||||
| return None | return None | ||||
| docket = RevlogDocket(revlog, version_header=version_header) | docket = RevlogDocket(revlog, version_header=version_header) | ||||
| docket._dirty = True | docket._dirty = True | ||||
| return docket | return docket | ||||
| def parse_docket(revlog, data): | def parse_docket(revlog, data, use_pending=False): | ||||
| """given some docket data return a docket object for the given revlog""" | """given some docket data return a docket object for the given revlog""" | ||||
| header = S_HEADER.unpack(data[: S_HEADER.size]) | header = S_HEADER.unpack(data[: S_HEADER.size]) | ||||
| version_header, index_size = header | version_header, index_size, pending_index_size = header | ||||
| docket = RevlogDocket( | docket = RevlogDocket( | ||||
| revlog, | revlog, | ||||
| use_pending=use_pending, | |||||
| version_header=version_header, | version_header=version_header, | ||||
| index_end=index_size, | index_end=index_size, | ||||
| pending_index_end=pending_index_size, | |||||
| ) | ) | ||||
| return docket | return docket | ||||
| $ export HG_TEST_FILE_EXT_WAITING | $ export HG_TEST_FILE_EXT_WAITING | ||||
| $ HG_TEST_FILE_EXT_UNLOCK=$TESTTMP/sync/ext_unlock | $ HG_TEST_FILE_EXT_UNLOCK=$TESTTMP/sync/ext_unlock | ||||
| $ export HG_TEST_FILE_EXT_UNLOCK | $ export HG_TEST_FILE_EXT_UNLOCK | ||||
| $ HG_TEST_FILE_EXT_DONE=$TESTTMP/sync/ext_done | $ HG_TEST_FILE_EXT_DONE=$TESTTMP/sync/ext_done | ||||
| $ export HG_TEST_FILE_EXT_DONE | $ export HG_TEST_FILE_EXT_DONE | ||||
| $ cat << EOF > script/external.sh | $ cat << EOF > script/external.sh | ||||
| > #!/bin/sh | > #!/bin/sh | ||||
| > $RUNTESTDIR/testlib/wait-on-file 5 $HG_TEST_FILE_EXT_UNLOCK $HG_TEST_FILE_EXT_WAITING | > $RUNTESTDIR/testlib/wait-on-file 5 $HG_TEST_FILE_EXT_UNLOCK $HG_TEST_FILE_EXT_WAITING | ||||
| > hg log --rev 'tip' -T 'external: {rev} {desc}\n' > $TESTTMP/output/external.out 2>/dev/null | > hg log --rev 'tip' -T 'external: {rev} {desc}\n' > $TESTTMP/output/external.out | ||||
| > touch $HG_TEST_FILE_EXT_DONE | > touch $HG_TEST_FILE_EXT_DONE | ||||
| > EOF | > EOF | ||||
| $ chmod +x script/external.sh | $ chmod +x script/external.sh | ||||
| $ cat << EOF > script/internal.sh | $ cat << EOF > script/internal.sh | ||||
| > #!/bin/sh | > #!/bin/sh | ||||
| > hg log --rev 'tip' -T 'internal: {rev} {desc}\n' > $TESTTMP/output/internal.out 2>/dev/null | > hg log --rev 'tip' -T 'internal: {rev} {desc}\n' > $TESTTMP/output/internal.out | ||||
| > $RUNTESTDIR/testlib/wait-on-file 5 $HG_TEST_FILE_EXT_DONE $HG_TEST_FILE_EXT_UNLOCK | > $RUNTESTDIR/testlib/wait-on-file 5 $HG_TEST_FILE_EXT_DONE $HG_TEST_FILE_EXT_UNLOCK | ||||
| > EOF | > EOF | ||||
| $ chmod +x script/internal.sh | $ chmod +x script/internal.sh | ||||
| Automated commands: | Automated commands: | ||||
| $ make_one_commit() { | $ make_one_commit() { | ||||
| check this is true for the initial commit (inline → inline) | check this is true for the initial commit (inline → inline) | ||||
| ----------------------------------------------------------- | ----------------------------------------------------------- | ||||
| the repository should still be inline (for relevant format) | the repository should still be inline (for relevant format) | ||||
| $ make_one_commit first | $ make_one_commit first | ||||
| pre-commit: -1 | pre-commit: -1 | ||||
| external: -1 | external: -1 | ||||
| internal: 0 first (revlogv1 !) | internal: 0 first | ||||
| internal: -1 (revlogv2 known-bad-output !) | |||||
| post-tr: 0 first | post-tr: 0 first | ||||
| #if revlogv1 | #if revlogv1 | ||||
| $ hg debugrevlog -c | grep inline | $ hg debugrevlog -c | grep inline | ||||
| flags : inline | flags : inline | ||||
| #endif | #endif | ||||
| check this is true for extra commit (inline → inline) | check this is true for extra commit (inline → inline) | ||||
| ----------------------------------------------------- | ----------------------------------------------------- | ||||
| the repository should still be inline (for relevant format) | the repository should still be inline (for relevant format) | ||||
| #if revlogv1 | #if revlogv1 | ||||
| $ hg debugrevlog -c | grep inline | $ hg debugrevlog -c | grep inline | ||||
| flags : inline | flags : inline | ||||
| #endif | #endif | ||||
| $ make_one_commit second | $ make_one_commit second | ||||
| pre-commit: 0 first | pre-commit: 0 first | ||||
| external: 0 first | external: 0 first | ||||
| internal: 1 second (revlogv1 !) | internal: 1 second | ||||
| internal: 0 first (revlogv2 known-bad-output !) | |||||
| post-tr: 1 second | post-tr: 1 second | ||||
| #if revlogv1 | #if revlogv1 | ||||
| $ hg debugrevlog -c | grep inline | $ hg debugrevlog -c | grep inline | ||||
| flags : inline | flags : inline | ||||
| #endif | #endif | ||||
| flags : inline | flags : inline | ||||
| #endif | #endif | ||||
| $ make_one_pull 3 | $ make_one_pull 3 | ||||
| pre-commit: 1 second | pre-commit: 1 second | ||||
| warning: repository is unrelated | warning: repository is unrelated | ||||
| external: 1 second | external: 1 second | ||||
| internal: 5 r3 (revlogv1 !) | internal: 5 r3 | ||||
| internal: 1 second (revlogv2 known-bad-output !) | |||||
| post-tr: 5 r3 | post-tr: 5 r3 | ||||
| #if revlogv1 | #if revlogv1 | ||||
| $ hg debugrevlog -c | grep inline | $ hg debugrevlog -c | grep inline | ||||
| flags : inline | flags : inline | ||||
| #endif | #endif | ||||
| Make a large pull (inline → no-inline) | Make a large pull (inline → no-inline) | ||||
| --------------------------------------- | --------------------------------------- | ||||
| the repository should no longer be inline (for relevant format) | the repository should no longer be inline (for relevant format) | ||||
| #if revlogv1 | #if revlogv1 | ||||
| $ hg debugrevlog -c | grep inline | $ hg debugrevlog -c | grep inline | ||||
| flags : inline | flags : inline | ||||
| #endif | #endif | ||||
| $ make_one_pull 400 | $ make_one_pull 400 | ||||
| pre-commit: 5 r3 | pre-commit: 5 r3 | ||||
| external: 5 r3 | external: 5 r3 | ||||
| internal: 402 r400 (revlogv1 !) | internal: 402 r400 | ||||
| internal: 5 r3 (revlogv2 known-bad-output !) | |||||
| post-tr: 402 r400 | post-tr: 402 r400 | ||||
| #if revlogv1 | #if revlogv1 | ||||
| $ hg debugrevlog -c | grep inline | $ hg debugrevlog -c | grep inline | ||||
| [1] | [1] | ||||
| #endif | #endif | ||||
| check this is true for extra commit (no-inline → no-inline) | check this is true for extra commit (no-inline → no-inline) | ||||
| ----------------------------------------------------------- | ----------------------------------------------------------- | ||||
| the repository should no longer be inline (for relevant format) | the repository should no longer be inline (for relevant format) | ||||
| #if revlogv1 | #if revlogv1 | ||||
| $ hg debugrevlog -c | grep inline | $ hg debugrevlog -c | grep inline | ||||
| [1] | [1] | ||||
| #endif | #endif | ||||
| $ make_one_commit third | $ make_one_commit third | ||||
| pre-commit: 402 r400 | pre-commit: 402 r400 | ||||
| external: 402 r400 | external: 402 r400 | ||||
| internal: 403 third (revlogv1 !) | internal: 403 third | ||||
| internal: 402 r400 (revlogv2 known-bad-output !) | |||||
| post-tr: 403 third | post-tr: 403 third | ||||
| #if revlogv1 | #if revlogv1 | ||||
| $ hg debugrevlog -c | grep inline | $ hg debugrevlog -c | grep inline | ||||
| [1] | [1] | ||||
| #endif | #endif | ||||
| $ hg debugrevlog -c | grep inline | $ hg debugrevlog -c | grep inline | ||||
| [1] | [1] | ||||
| #endif | #endif | ||||
| $ make_one_pull tip | $ make_one_pull tip | ||||
| pre-commit: 403 third | pre-commit: 403 third | ||||
| external: 403 third | external: 403 third | ||||
| internal: 503 r500 (revlogv1 !) | internal: 503 r500 | ||||
| internal: 403 third (revlogv2 known-bad-output !) | |||||
| post-tr: 503 r500 | post-tr: 503 r500 | ||||
| #if revlogv1 | #if revlogv1 | ||||
| $ hg debugrevlog -c | grep inline | $ hg debugrevlog -c | grep inline | ||||
| [1] | [1] | ||||
| #endif | #endif | ||||