diff --git a/mercurial/configitems.py b/mercurial/configitems.py --- a/mercurial/configitems.py +++ b/mercurial/configitems.py @@ -1283,6 +1283,12 @@ ) coreconfigitem( b'format', + b'exp-dirstate-tracked-key-version', + default=0, + experimental=True, +) +coreconfigitem( + b'format', b'dotencode', default=True, ) diff --git a/mercurial/dirstate.py b/mercurial/dirstate.py --- a/mercurial/dirstate.py +++ b/mercurial/dirstate.py @@ -12,6 +12,7 @@ import errno import os import stat +import uuid from .i18n import _ from .pycompat import delattr @@ -23,6 +24,7 @@ encoding, error, match as matchmod, + node, pathutil, policy, pycompat, @@ -99,6 +101,7 @@ sparsematchfn, nodeconstants, use_dirstate_v2, + use_tracked_key=False, ): """Create a new dirstate object. @@ -107,6 +110,7 @@ the dirstate. """ self._use_dirstate_v2 = use_dirstate_v2 + self._use_tracked_key = use_tracked_key self._nodeconstants = nodeconstants self._opener = opener self._validate = validate @@ -115,11 +119,15 @@ # ntpath.join(root, '') of Python 2.7.9 does not add sep if root is # UNC path pointing to root share (issue4557) self._rootdir = pathutil.normasprefix(root) + # True is any internal state may be different self._dirty = False + # True if the set of tracked file may be different + self._dirty_tracked_set = False self._ui = ui self._filecache = {} self._parentwriters = 0 self._filename = b'dirstate' + self._filename_tk = b'dirstate-tracked-key' self._pendingfilename = b'%s.pending' % self._filename self._plchangecallbacks = {} self._origpl = None @@ -409,6 +417,7 @@ if a in self.__dict__: delattr(self, a) self._dirty = False + self._dirty_tracked_set = False self._parentwriters = 0 self._origpl = None @@ -446,6 +455,8 @@ pre_tracked = self._map.set_tracked(filename) if reset_copy: self._map.copymap.pop(filename, None) + if pre_tracked: + self._dirty_tracked_set = True return pre_tracked @requires_no_parents_change @@ -460,6 +471,7 @@ ret = self._map.set_untracked(filename) if ret: self._dirty = True + self._dirty_tracked_set = True return ret @requires_no_parents_change @@ -544,6 +556,13 @@ # this. The test agrees self._dirty = True + old_entry = self._map.get(filename) + if old_entry is None: + prev_tracked = False + else: + prev_tracked = old_entry.tracked + if prev_tracked != wc_tracked: + self._dirty_tracked_set = True self._map.reset_state( filename, @@ -702,20 +721,44 @@ if not self._dirty: return - filename = self._filename + write_key = self._use_tracked_key and self._dirty_tracked_set if tr: # delay writing in-memory changes out + if write_key: + tr.addfilegenerator( + b'dirstate-0-key-pre', + (self._filename_tk,), + lambda f: self._write_tracked_key(tr, f), + location=b'plain', + ) tr.addfilegenerator( b'dirstate-1-main', (self._filename,), lambda f: self._writedirstate(tr, f), location=b'plain', ) + if write_key: + tr.addfilegenerator( + b'dirstate-2-key-post', + (self._filename_tk,), + lambda f: self._write_tracked_key(tr, f), + location=b'plain', + ) return file = lambda f: self._opener(f, b"w", atomictemp=True, checkambig=True) + if write_key: + # we change the key-file before changing the dirstate to make sure + # reading invalidate there cache before we start writing + with file(self._filename_tk) as f: + self._write_tracked_key(tr, f) with file(self._filename) as f: self._writedirstate(tr, f) + if write_key: + # we update the key-file after writing to make sure reader have a + # key that match the newly written content + with file(self._filename_tk) as f: + self._write_tracked_key(tr, f) def addparentchangecallback(self, category, callback): """add a callback to be called when the wd parents are changed @@ -736,9 +779,13 @@ ): callback(self, self._origpl, self._pl) self._origpl = None - self._map.write(tr, st) self._dirty = False + self._dirty_tracked_set = False + + def _write_tracked_key(self, tr, f): + key = node.hex(uuid.uuid4().bytes) + f.write(b"1\n%s\n" % key) # 1 is the format version def _dirignore(self, f): if self._ignore(f): diff --git a/mercurial/helptext/config.txt b/mercurial/helptext/config.txt --- a/mercurial/helptext/config.txt +++ b/mercurial/helptext/config.txt @@ -944,6 +944,42 @@ For a more comprehensive guide, see :hg:`help internals.dirstate-v2`. +``exp-dirstate-tracked-key-version`` + Enable or disable the writing of "tracked key" file alongside the dirstate. + + That "tracked-key" can help external automations to detect changes to the + set of tracked files. + + Two values are currently supported: + - 0: no file is written (the default), + - 1: a file in version "1" is written. + + The tracked-key is written in a new `.hg/dirstate-tracked-key`. That file + contains two lines: + - the first line is the file version (currently: 1), + - the second line contains the "tracked-key". + + The tracked-key changes whenever the set of file tracked in the dirstate + changes. The general guarantee are: + - if the tracked key is identical, the set of tracked file MUST not changed, + - if the tracked key is different, the set of tracked file MIGHT differ. + + They are two "ways" to use this feature: + + 1) monitoring changes to the `.hg/dirstate-tracked-key`, if the file changes + the tracked set might have changed. + + 2) storing the value and comparing it to later value. Beware that it is + impossible to achieve atomic writing or reading of the two file involved + files (`.hg/dirstate` and `.hg/dirstate-tracked-key`). So it is needed to + read the `tracked-key` files twice: before and after reading the tracked + set. The `tracked-key` is only usable as a cache key if it had the same + value in both cases and must be discarded otherwise. + + To enforce that the `tracked-key` value can be used race-free (with double + reading as explained in (2)), the `.hg/dirstate-tracked-key` is written + twice: before and after we change the associate `.hg/dirstate` file. + ``use-persistent-nodemap`` Enable or disable the "persistent-nodemap" feature which improves performance if the Rust extensions are available. diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py --- a/mercurial/localrepo.py +++ b/mercurial/localrepo.py @@ -1278,6 +1278,7 @@ requirementsmod.BOOKMARKS_IN_STORE_REQUIREMENT, requirementsmod.CHANGELOGV2_REQUIREMENT, requirementsmod.COPIESSDC_REQUIREMENT, + requirementsmod.DIRSTATE_TRACKED_KEY_V1, requirementsmod.DIRSTATE_V2_REQUIREMENT, requirementsmod.DOTENCODE_REQUIREMENT, requirementsmod.FNCACHE_REQUIREMENT, @@ -1742,7 +1743,9 @@ """Extension point for wrapping the dirstate per-repo.""" sparsematchfn = lambda: sparse.matcher(self) v2_req = requirementsmod.DIRSTATE_V2_REQUIREMENT + tk = requirementsmod.DIRSTATE_TRACKED_KEY_V1 use_dirstate_v2 = v2_req in self.requirements + use_tracked_key = tk in self.requirements return dirstate.dirstate( self.vfs, @@ -1752,6 +1755,7 @@ sparsematchfn, self.nodeconstants, use_dirstate_v2, + use_tracked_key=use_tracked_key, ) def _dirstatevalidate(self, node): @@ -3689,6 +3693,17 @@ else: requirements.add(requirementsmod.SHARED_REQUIREMENT) + tracked_key = ui.configint(b'format', b'exp-dirstate-tracked-key-version') + if tracked_key: + if tracked_key != 1: + msg = _("ignoring unknown tracked key version: %d\n") + hint = _( + "see `hg help config.format.exp-dirstate-tracked-key-version" + ) + ui.warn(msg % tracked_key, hint=hint) + else: + requirements.add(requirementsmod.DIRSTATE_TRACKED_KEY_V1) + return requirements diff --git a/mercurial/requirements.py b/mercurial/requirements.py --- a/mercurial/requirements.py +++ b/mercurial/requirements.py @@ -18,6 +18,7 @@ STORE_REQUIREMENT = b'store' FNCACHE_REQUIREMENT = b'fncache' +DIRSTATE_TRACKED_KEY_V1 = b'exp-dirstate-tracked-key-v1' DIRSTATE_V2_REQUIREMENT = b'dirstate-v2' # When narrowing is finalized and no longer subject to format changes, @@ -96,6 +97,7 @@ SHARED_REQUIREMENT, RELATIVE_SHARED_REQUIREMENT, SHARESAFE_REQUIREMENT, + DIRSTATE_TRACKED_KEY_V1, DIRSTATE_V2_REQUIREMENT, } diff --git a/mercurial/transaction.py b/mercurial/transaction.py --- a/mercurial/transaction.py +++ b/mercurial/transaction.py @@ -30,7 +30,9 @@ # the changelog having been written). postfinalizegenerators = { b'bookmarks', + b'dirstate-0-key-pre', b'dirstate-1-main', + b'dirstate-2-key-post', } GEN_GROUP_ALL = b'all' diff --git a/tests/test-help.t b/tests/test-help.t --- a/tests/test-help.t +++ b/tests/test-help.t @@ -1599,6 +1599,8 @@ "exp-rc-dirstate-v2" + "exp-dirstate-tracked-key-version" + "use-persistent-nodemap" "use-share-safe" diff --git a/tests/test-status-tracked-key.t b/tests/test-status-tracked-key.t new file mode 100644 --- /dev/null +++ b/tests/test-status-tracked-key.t @@ -0,0 +1,163 @@ +============================== +Test the "tracked key" feature +============================== + +The tracked key feature provide a file that get updated when the set of tracked +files get updated. + +basic setup + + $ cat << EOF >> $HGRCPATH + > [format] + > exp-dirstate-tracked-key-version=1 + > EOF + + $ hg init tracked-key-test + $ cd tracked-key-test + $ hg debugbuilddag '.+10' -n + $ hg log -G -T '{rev} {desc} {files}\n' + o 10 r10 nf10 + | + o 9 r9 nf9 + | + o 8 r8 nf8 + | + o 7 r7 nf7 + | + o 6 r6 nf6 + | + o 5 r5 nf5 + | + o 4 r4 nf4 + | + o 3 r3 nf3 + | + o 2 r2 nf2 + | + o 1 r1 nf1 + | + o 0 r0 nf0 + + $ hg up tip + 11 files updated, 0 files merged, 0 files removed, 0 files unresolved + $ hg files + nf0 + nf1 + nf10 + nf2 + nf3 + nf4 + nf5 + nf6 + nf7 + nf8 + nf9 + +key-file exists +----------- + +The tracked key file should exist + + $ ls -1 .hg/dirstate* + .hg/dirstate + .hg/dirstate-tracked-key + +key-file stay the same if the tracked set is unchanged +------------------------------------------------------ + +(copy its content for later comparison) + + $ cp .hg/dirstate-tracked-key ../key-bck + $ echo foo >> nf0 + $ sleep 1 + $ hg status + M nf0 + $ diff --brief .hg/dirstate-tracked-key ../key-bck + $ hg revert -C nf0 + $ sleep 1 + $ hg status + $ diff --brief .hg/dirstate-tracked-key ../key-bck + +key-file change if the tracked set is changed manually +------------------------------------------------------ + +adding a file to tracking + + $ cp .hg/dirstate-tracked-key ../key-bck + $ echo x > x + $ hg add x + $ diff --brief .hg/dirstate-tracked-key ../key-bck + Files .hg/dirstate-tracked-key and ../key-bck differ + [1] + +remove a file from tracking +(forget) + + $ cp .hg/dirstate-tracked-key ../key-bck + $ hg forget x + $ diff --brief .hg/dirstate-tracked-key ../key-bck + Files .hg/dirstate-tracked-key and ../key-bck differ + [1] + +(remove) + + $ cp .hg/dirstate-tracked-key ../key-bck + $ hg remove nf1 + $ diff --brief .hg/dirstate-tracked-key ../key-bck + Files .hg/dirstate-tracked-key and ../key-bck differ + [1] + +key-file changes on revert (when applicable) +-------------------------------------------- + + $ cp .hg/dirstate-tracked-key ../key-bck + $ hg status + R nf1 + ? x + $ hg revert --all + undeleting nf1 + $ hg status + ? x + $ diff --brief .hg/dirstate-tracked-key ../key-bck + Files .hg/dirstate-tracked-key and ../key-bck differ + [1] + + +`hg update` does affect the key-file (when needed) +-------------------------------------------------- + +update changing the tracked set + +(removing) + + $ cp .hg/dirstate-tracked-key ../key-bck + $ hg status --rev . --rev '.#generations[-1]' + R nf10 + $ hg up '.#generations[-1]' + 0 files updated, 0 files merged, 1 files removed, 0 files unresolved + $ diff --brief .hg/dirstate-tracked-key ../key-bck + Files .hg/dirstate-tracked-key and ../key-bck differ + [1] + +(adding) + + $ cp .hg/dirstate-tracked-key ../key-bck + $ hg status --rev . --rev '.#generations[1]' + A nf10 + $ hg up '.#generations[1]' + 1 files updated, 0 files merged, 0 files removed, 0 files unresolved + $ diff --brief .hg/dirstate-tracked-key ../key-bck + Files .hg/dirstate-tracked-key and ../key-bck differ + [1] + +update not affecting the tracked set + + $ echo foo >> nf0 + $ hg commit -m foo + + $ cp .hg/dirstate-tracked-key ../key-bck + $ hg status --rev . --rev '.#generations[-1]' + M nf0 + $ hg up '.#generations[-1]' + 1 files updated, 0 files merged, 0 files removed, 0 files unresolved + $ diff --brief .hg/dirstate-tracked-key ../key-bck