diff --git a/tests/test-treemanifest-prefetch.t b/tests/test-treemanifest-prefetch.t --- a/tests/test-treemanifest-prefetch.t +++ b/tests/test-treemanifest-prefetch.t @@ -1,10 +1,27 @@ +There are three cases which are of interest in this test: + + - client remotefilelog enabled and the client repo is a shallowrepo + In this case, the expectation is that the prefetch command will be able to + prefetch both trees and files. + + - client remotefilelog enabled and the client repo is not a shallowrepo + In this case, the expectation is that the prefetch command will be able to only + prefetch trees. + + - client remotefilelog disabled + In this case, the expectation is that the prefetch command will be able to only + prefetch trees and will not be able to perform operations like repack which + require remotefilelog. + +#testcases remotefilelog.true.shallowrepo.true remotefilelog.true.shallowrepo.false remotefilelog.false + $ CACHEDIR=`pwd`/hgcache $ PYTHONPATH=$TESTDIR/..:$PYTHONPATH $ export PYTHONPATH $ . "$TESTDIR/library.sh" - $ hg init master + $ hginit master $ cd master $ mkdir dir $ echo x > dir/x @@ -19,6 +36,7 @@ > treemanifest=$TESTDIR/../treemanifest > > [remotefilelog] + > server=True > name=master > cachepath=$CACHEDIR > usefastdatapack=True @@ -32,6 +50,18 @@ > EOF $ cd .. + +#if remotefilelog.true.shallowrepo.true + $ hgcloneshallow ssh://user@dummy/master client + streaming all changes + 2 files to transfer, 749 bytes of data + transferred 749 bytes in * seconds (*) (glob) + searching for changes + no changes found + updating to branch default + 2 files updated, 0 files merged, 0 files removed, 0 files unresolved + 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob) +#else $ hg clone ssh://user@dummy/master client streaming all changes 4 files to transfer, 952 bytes of data @@ -40,6 +70,7 @@ no changes found updating to branch default 2 files updated, 0 files merged, 0 files removed, 0 files unresolved +#endif $ cd master $ hg backfilltree @@ -57,15 +88,25 @@ > usecache = False > EOF -Test prefetchtrees - $ hg prefetchtrees -r '0 + 1 + 2' +#if remotefilelog.false + $ cat >> .hg/hgrc < + > [extensions] + > remotefilelog=! + > EOF +#endif + +Test prefetch + $ hg prefetch -r '0 + 1 + 2' 6 trees fetched over * (glob) + 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !) $ ls $CACHEDIR/master/packs/manifests 29938257d506f677320d5abec8e34a1a9ed635fe.histidx 29938257d506f677320d5abec8e34a1a9ed635fe.histpack 8adc618d23082c0a5311a4bbf9ac08b9b9672471.dataidx 8adc618d23082c0a5311a4bbf9ac08b9b9672471.datapack - $ hg debugdatapack --long $CACHEDIR/master/packs/manifests/*.dataidx + $ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \ + > --long $CACHEDIR/master/packs/manifests/*.dataidx $TESTTMP/hgcache/master/packs/manifests/8adc618d23082c0a5311a4bbf9ac08b9b9672471: subdir: Node Delta Base Delta Length Blob Size @@ -91,7 +132,8 @@ Node Delta Base Delta Length Blob Size ef362f8bbe8aa457b0cfc49f200cbeb7747984ed 0000000000000000000000000000000000000000 46 (missing) - $ hg debughistorypack $CACHEDIR/master/packs/manifests/*.histidx + $ hg debughistorypack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \ + > $CACHEDIR/master/packs/manifests/*.histidx Node P1 Node P2 Node Link Node Copy From @@ -107,7 +149,8 @@ subdir Node P1 Node P2 Node Link Node Copy From ddb35f099a64 000000000000 000000000000 f15c65c6e9bd - $ hg debugdatapack --node ef362f8bbe8aa457b0cfc49f200cbeb7747984ed $CACHEDIR/master/packs/manifests/*.dataidx + $ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \ + > --node ef362f8bbe8aa457b0cfc49f200cbeb7747984ed $CACHEDIR/master/packs/manifests/*.dataidx $TESTTMP/hgcache/master/packs/manifests/8adc618d23082c0a5311a4bbf9ac08b9b9672471: @@ -126,12 +169,29 @@ Test prefetch with base node (subdir/ shouldn't show up in the pack) $ rm -rf $CACHEDIR/master - $ hg prefetchtrees -r '2' --base '1' + +#if remotefilelog.true.shallowrepo.true +Multiple trees are fetched in this case because the file prefetching code path +requires tree manifest for the base commit. + + $ hg prefetch -r '2' --base '1' + 2 trees fetched over * (glob) + 2 trees fetched over * (glob) + 3 trees fetched over * (glob) + 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) + $ ls $CACHEDIR/master/packs/manifests/*.dataidx + $TESTTMP/hgcache/master/packs/manifests/148e9eb32f473ea522c591c95be0f9e772be9675.dataidx + $TESTTMP/hgcache/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx + $TESTTMP/hgcache/master/packs/manifests/5f14647c5653622d4c2682648ec82c7193d2a9ab.dataidx +#else + $ hg prefetch -r '2' --base '1' 2 trees fetched over * (glob) $ ls $CACHEDIR/master/packs/manifests/*.dataidx $TESTTMP/hgcache/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx +#endif - $ hg debugdatapack $CACHEDIR/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx + $ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \ + > $CACHEDIR/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx $TESTTMP/hgcache/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c: dir: Node Delta Base Delta Length Blob Size @@ -160,6 +220,7 @@ dir/x | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) + 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !) $ ls $CACHEDIR/master/packs/manifests 148e9eb32f473ea522c591c95be0f9e772be9675.dataidx 148e9eb32f473ea522c591c95be0f9e772be9675.datapack @@ -170,7 +231,8 @@ e5c44a5c1bbfd8841df1c6c4b7cca54536e016db.histidx e5c44a5c1bbfd8841df1c6c4b7cca54536e016db.histpack - $ hg debugdatapack $CACHEDIR/master/packs/manifests/148e9eb32f473ea522c591c95be0f9e772be9675 + $ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \ + > $CACHEDIR/master/packs/manifests/148e9eb32f473ea522c591c95be0f9e772be9675 $TESTTMP/hgcache/master/packs/manifests/148e9eb32f473ea522c591c95be0f9e772be9675: dir: Node Delta Base Delta Length Blob Size @@ -186,7 +248,8 @@ - Note that subdir/ is not downloaded again - $ hg debugdatapack $CACHEDIR/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c + $ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \ + > $CACHEDIR/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c $TESTTMP/hgcache/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c: dir: Node Delta Base Delta Length Blob Size @@ -199,14 +262,17 @@ Test that auto prefetch scans up the changelog for base trees $ rm -rf $CACHEDIR/master - $ hg prefetchtrees -r 'tip^' + $ hg prefetch -r 'tip^' 3 trees fetched over * (glob) + 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !) $ rm -rf $CACHEDIR/master - $ hg prefetchtrees -r tip + $ hg prefetch -r tip 3 trees fetched over * (glob) + 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !) - Only 2 of the 3 trees from tip^ are downloaded as part of --stat's fetch $ hg log -r tip --stat --pager=off > /dev/null 2 trees fetched over * (glob) + 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !) Test auto prefetch during pull @@ -220,7 +286,8 @@ no changes found prefetching trees 6 trees fetched over * (glob) - $ hg debugdatapack $CACHEDIR/master/packs/manifests/*.dataidx + $ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \ + > $CACHEDIR/master/packs/manifests/*.dataidx $TESTTMP/hgcache/master/packs/manifests/8adc618d23082c0a5311a4bbf9ac08b9b9672471: subdir: Node Delta Base Delta Length Blob Size @@ -257,7 +324,8 @@ no changes found prefetching trees 3 trees fetched over * (glob) - $ hg debugdatapack $CACHEDIR/master/packs/manifests/*.dataidx + $ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \ + > $CACHEDIR/master/packs/manifests/*.dataidx $TESTTMP/hgcache/master/packs/manifests/4ee15de76c068ec1c80e3e61f2c3c476a779078a: dir: Node Delta Base Delta Length Blob Size @@ -274,8 +342,9 @@ - Prefetch commit 1 then minimally prefetch commit 2 $ rm -rf $CACHEDIR/master - $ hg prefetchtrees -r 1 + $ hg prefetch -r 1 3 trees fetched over * (glob) + 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !) $ ls $CACHEDIR/master/packs/manifests/*dataidx $TESTTMP/hgcache/master/packs/manifests/148e9eb32f473ea522c591c95be0f9e772be9675.dataidx $ hg pull --config treemanifest.pullprefetchcount=1 --traceback @@ -287,7 +356,8 @@ $ ls $CACHEDIR/master/packs/manifests/*dataidx $TESTTMP/hgcache/master/packs/manifests/148e9eb32f473ea522c591c95be0f9e772be9675.dataidx $TESTTMP/hgcache/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx - $ hg debugdatapack $CACHEDIR/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx + $ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \ + > $CACHEDIR/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx $TESTTMP/hgcache/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c: dir: Node Delta Base Delta Length Blob Size @@ -313,12 +383,14 @@ adding changesets adding manifests adding file changes - added 2 changesets with 2 changes to 1 files + added 2 changesets with 0 changes to 0 files (remotefilelog.true.shallowrepo.true !) + added 2 changesets with 2 changes to 1 files (no-remotefilelog.true.shallowrepo.true !) new changesets dece825f8add:cfacdcc4cee5 (run 'hg update' to get a working copy) prefetching trees 3 trees fetched over * (glob) - $ hg debugdatapack $CACHEDIR/master/packs/manifests/*.dataidx + $ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \ + > $CACHEDIR/master/packs/manifests/*.dataidx $TESTTMP/hgcache/master/packs/manifests/4ee15de76c068ec1c80e3e61f2c3c476a779078a: dir: Node Delta Base Delta Length Blob Size @@ -341,7 +413,8 @@ no changes found prefetching trees 2 trees fetched over * (glob) - $ hg debugdatapack $CACHEDIR/master/packs/manifests/99050e724a9236121684523ba3f4db270e62fb58.dataidx + $ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \ + > $CACHEDIR/master/packs/manifests/99050e724a9236121684523ba3f4db270e62fb58.dataidx $TESTTMP/hgcache/master/packs/manifests/99050e724a9236121684523ba3f4db270e62fb58: dir: Node Delta Base Delta Length Blob Size @@ -357,31 +430,43 @@ $ echo >> dir/x $ hg commit -m 'edit x locally' created new head + 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !) $ rm -rf $CACHEDIR/master/* $ hg cat subdir/z 3 trees fetched over * (glob) z + 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !) Test prefetch non-parent commits with no base node (should fetch minimal trees - in this case 3 trees for commit 2, and 2 for commit 4 despite it having 3 directories) $ rm -rf $CACHEDIR/master - $ hg prefetchtrees -r '2 + 4' + $ hg prefetch -r '2 + 4' 5 trees fetched over * (glob) + 3 files fetched over 1 fetches - (3 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !) Test repack option $ rm -rf $CACHEDIR/master - $ hg prefetchtrees -r '0' - 2 trees fetched over *s (glob) - $ hg prefetchtrees -r '2' - 3 trees fetched over *s (glob) + $ hg prefetch -r '0' + 2 trees fetched over * (glob) + 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !) + $ hg prefetch -r '2' + 3 trees fetched over * (glob) + 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !) - $ hg prefetchtrees -r '4' --repack - 3 trees fetched over *s (glob) +#if remotefilelog.false + $ hg prefetch -r '4' --repack + abort: repack requires remotefilelog extension + [255] +#else + $ hg prefetch -r '4' --repack + 3 trees fetched over * (glob) (running background incremental repack) + 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !) $ sleep 0.5 $ hg debugwaitonrepack $ ls_l $CACHEDIR/master/packs/manifests | grep datapack | wc -l \s*1 (re) +#endif diff --git a/treemanifest/__init__.py b/treemanifest/__init__.py --- a/treemanifest/__init__.py +++ b/treemanifest/__init__.py @@ -82,6 +82,7 @@ registrar, repair, revlog, + scmutil, sshserver, templatekw, util, @@ -90,6 +91,10 @@ from mercurial.i18n import _ from mercurial.node import bin, hex, nullid +from remotefilelog import ( + cmdtable as remotefilelogcmdtable, + resolveprefetchopts, +) from remotefilelog.contentstore import ( manifestrevlogstore, unioncontentstore, @@ -177,6 +182,22 @@ # Change manifest template output templatekw.defaulttempl['manifest'] = '{node}' + def _wrapremotefilelog(loaded): + if loaded: + remotefilelogmod = extensions.find('remotefilelog') + extensions.wrapcommand( + remotefilelogmod.cmdtable, 'prefetch', _prefetchwrapper) + else: + # There is no prefetch command to wrap around. In this case, we use + # the command table entry for prefetch in the remotefilelog to + # define the prefetch command, wrap it, and then override it + # completely. This ensures that the options to the prefetch command + # are consistent. + cmdtable['prefetch'] = remotefilelogcmdtable['prefetch'] + extensions.wrapcommand(cmdtable, 'prefetch', _overrideprefetch) + + extensions.afterloaded('remotefilelog', _wrapremotefilelog) + def showmanifest(orig, **args): """Same implementation as the upstream showmanifest, but without the 'rev' field.""" @@ -998,28 +1019,47 @@ raise AttributeError(_("%s has no property '%s'") % (type(currcls), propname)) -@command('prefetchtrees', [ - ('r', 'rev', '', _("revs to prefetch the trees for")), - ('', 'repack', False, _('run repack after prefetch')), - ('b', 'base', '', _("rev that is assumed to already be local")), - ] + commands.walkopts, _('--rev REVS PATTERN..')) -def prefetchtrees(ui, repo, *args, **opts): - revs = repo.revs(opts.get('rev')) +# Wrapper around the 'prefetch' command which also allows for prefetching the +# trees along with the files. +def _prefetchwrapper(orig, ui, repo, *pats, **opts): + # The wrapper will take care of the repacking. + repackrequested = opts.pop('repack') + + _prefetchonlytrees(repo, opts) + _prefetchonlyfiles(orig, ui, repo, *pats, **opts) + + if repackrequested: + backgroundrepack(repo, incremental=True) + +# Wrapper around the 'prefetch' command which overrides the command completely +# and only allows for prefetching trees. This is only required when the +# 'prefetch' command is not available because the remotefilelog extension is not +# loaded and we want to be able to at least prefetch trees. The wrapping just +# ensures that we get a consistent interface to the 'prefetch' command. +def _overrideprefetch(orig, ui, repo, *pats, **opts): + if opts.get('repack'): + raise error.Abort(_('repack requires remotefilelog extension')) + + _prefetchonlytrees(repo, opts) + +def _prefetchonlyfiles(orig, ui, repo, *pats, **opts): + if shallowrepo.requirement in repo.requirements: + orig(ui, repo, *pats, **opts) + +def _prefetchonlytrees(repo, opts): + opts = resolveprefetchopts(repo.ui, opts) + revs = scmutil.revrange(repo, opts.get('rev')) mfnodes = set() for rev in revs: mfnodes.add(repo[rev].manifestnode()) basemfnode = set() base = opts.get('base') - if base: + if base is not None: basemfnode.add(repo[base].manifestnode()) _prefetchtrees(repo, '', mfnodes, basemfnode, []) - # Run repack in background - if opts.get('repack'): - backgroundrepack(repo, incremental=True) - def _prefetchtrees(repo, rootdir, mfnodes, basemfnodes, directories): # If possible, use remotefilelog's more expressive fallbackpath if util.safehasattr(repo, 'fallbackpath'):