diff --git a/tests/test-treemanifest-treeonly.t b/tests/test-treemanifest-treeonly.t --- a/tests/test-treemanifest-treeonly.t +++ b/tests/test-treemanifest-treeonly.t @@ -95,6 +95,310 @@ > treeonly=True > EOF +Test working with flat-only draft commits. + +- There are no local tree packs. + $ ls_l .hg/store | grep packs + [1] + +- Viewing flat draft commit would fail when 'treemanifest.demandgenerate' is +False in treeonly mode because there is no tree manifest. + + $ hg log -vpr 'b9b574be2f5d' --config treemanifest.demandgenerate=False \ + > 2>&1 > /dev/null | tail -1 + KeyError: 'tree node not found (, 40f43426c87ba597f0d9553077c72fe06d4e2acb)' + +- Viewing a flat draft commit in treeonly mode will generate a tree manifest +for all the commits in the path from the flat draft commit to an ancestor which +has tree manifest. In this case, this implies that tree manifest will be +generated for the commit 'b9b574be2f5d' and its parent commit '9055b56f3916'. + + $ hg log -vpr 'b9b574be2f5d' + changeset: 2:b9b574be2f5d + user: test + date: Thu Jan 01 00:00:00 1970 +0000 + files: subdir/x + description: + flat only commit 1 over flat only commit 1 at level 1 + + + diff -r 9055b56f3916 -r b9b574be2f5d subdir/x + --- a/subdir/x Thu Jan 01 00:00:00 1970 +0000 + +++ b/subdir/x Thu Jan 01 00:00:00 1970 +0000 + @@ -1,2 +1,3 @@ + x + f1 + +f11 + +- Now that we have the tree manifest for commit 'b9b574be2f5d', we should be +able to view it even with 'treemanifest.demandgenerate' being False. + + $ hg log -vpr 'b9b574be2f5d' --config treemanifest.demandgenerate=False + changeset: 2:b9b574be2f5d + user: test + date: Thu Jan 01 00:00:00 1970 +0000 + files: subdir/x + description: + flat only commit 1 over flat only commit 1 at level 1 + + + diff -r 9055b56f3916 -r b9b574be2f5d subdir/x + --- a/subdir/x Thu Jan 01 00:00:00 1970 +0000 + +++ b/subdir/x Thu Jan 01 00:00:00 1970 +0000 + @@ -1,2 +1,3 @@ + x + f1 + +f11 + +- We should be able to also view the parent of commit 'b9b574be2f5d' i.e. commit +'9055b56f3916' because we now have the tree manifest for it. + + $ hg log -vpr '9055b56f3916' --config treemanifest.demandgenerate=False + changeset: 1:9055b56f3916 + user: test + date: Thu Jan 01 00:00:00 1970 +0000 + files: subdir/x + description: + flat only commit 1 at level 1 + + + diff -r 2278cc8c6ce6 -r 9055b56f3916 subdir/x + --- a/subdir/x Thu Jan 01 00:00:00 1970 +0000 + +++ b/subdir/x Thu Jan 01 00:00:00 1970 +0000 + @@ -1,1 +1,2 @@ + x + +f1 + +- Check the tree manifest for commit '9055b56f3916' and 'b9b574be2f5d'. + + $ ls_l .hg/store/packs/manifests + -r--r--r-- 1196 028534a0bedee7c3d57bd6fd459f16abe971621b.histidx + -r--r--r-- 183 028534a0bedee7c3d57bd6fd459f16abe971621b.histpack + -r--r--r-- 1106 4bb74ed3582b14a57f34a23c494496a4212af761.dataidx + -r--r--r-- 211 4bb74ed3582b14a57f34a23c494496a4212af761.datapack + -r--r--r-- 1106 5a179c0bb6419ffadbed2c826f2e17b95b05bafb.dataidx + -r--r--r-- 211 5a179c0bb6419ffadbed2c826f2e17b95b05bafb.datapack + -r--r--r-- 1196 9a66f2052faa2af4f56b178a1c2958b52b676046.histidx + -r--r--r-- 183 9a66f2052faa2af4f56b178a1c2958b52b676046.histpack + +- Tree manifest data for commit '9055b56f3916'. + + $ hg debugdatapack .hg/store/packs/manifests/5a179c0bb6419ffadbed2c826f2e17b95b05bafb.datapack + .hg/store/packs/manifests/5a179c0bb6419ffadbed2c826f2e17b95b05bafb: + subdir: + Node Delta Base Delta Length Blob Size + 33600a12f793 000000000000 43 (missing) + + (empty name): + Node Delta Base Delta Length Blob Size + 40f43426c87b 000000000000 49 (missing) + +- Tree manifest data for commit 'b9b574be2f5d'. + + $ hg debugdatapack .hg/store/packs/manifests/4bb74ed3582b14a57f34a23c494496a4212af761.datapack + .hg/store/packs/manifests/4bb74ed3582b14a57f34a23c494496a4212af761: + subdir: + Node Delta Base Delta Length Blob Size + 397e59856f06 000000000000 43 (missing) + + (empty name): + Node Delta Base Delta Length Blob Size + 53c631458e33 000000000000 49 (missing) + +- Again, this would generate the tree manifest from the corresponding flat +manifest for commit 'f7febcf0f689'. + + $ hg log -vpr 'f7febcf0f689' + changeset: 3:f7febcf0f689 + parent: 1:9055b56f3916 + user: test + date: Thu Jan 01 00:00:00 1970 +0000 + files: subdir/x + description: + flat only commit 2 over flat only commit 1 at level 1 + + + diff -r 9055b56f3916 -r f7febcf0f689 subdir/x + --- a/subdir/x Thu Jan 01 00:00:00 1970 +0000 + +++ b/subdir/x Thu Jan 01 00:00:00 1970 +0000 + @@ -1,2 +1,3 @@ + x + f1 + +f12 + + $ ls_l .hg/store/packs/manifests + -r--r--r-- 1196 028534a0bedee7c3d57bd6fd459f16abe971621b.histidx + -r--r--r-- 183 028534a0bedee7c3d57bd6fd459f16abe971621b.histpack + -r--r--r-- 1106 1074860af987f99d7c9e6d053852060e47ed05bd.dataidx + -r--r--r-- 211 1074860af987f99d7c9e6d053852060e47ed05bd.datapack + -r--r--r-- 1106 4bb74ed3582b14a57f34a23c494496a4212af761.dataidx + -r--r--r-- 211 4bb74ed3582b14a57f34a23c494496a4212af761.datapack + -r--r--r-- 1106 5a179c0bb6419ffadbed2c826f2e17b95b05bafb.dataidx + -r--r--r-- 211 5a179c0bb6419ffadbed2c826f2e17b95b05bafb.datapack + -r--r--r-- 1196 5ef2ade5f4492a25bed3204d2e5b6040e5f5f96e.histidx + -r--r--r-- 183 5ef2ade5f4492a25bed3204d2e5b6040e5f5f96e.histpack + -r--r--r-- 1196 9a66f2052faa2af4f56b178a1c2958b52b676046.histidx + -r--r--r-- 183 9a66f2052faa2af4f56b178a1c2958b52b676046.histpack + +- Tree manifest data for commit 'f7febcf0f689'. + + $ hg debugdatapack .hg/store/packs/manifests/1074860af987f99d7c9e6d053852060e47ed05bd.datapack + .hg/store/packs/manifests/1074860af987f99d7c9e6d053852060e47ed05bd: + subdir: + Node Delta Base Delta Length Blob Size + 906f17f69284 000000000000 43 (missing) + + (empty name): + Node Delta Base Delta Length Blob Size + a6875e5fbf69 000000000000 49 (missing) + +- Clean up generated tree manifests for remaining tests. + + $ rm -rf .hg/store/packs + +- Test rebasing of the flat ony commits works as expected. + + $ hg rebase -d '9055b56f3916' -s '3795bd66ca70' + rebasing 4:3795bd66ca70 "flat only commit 1 over flat only commit 2 at level 2" (tip) + merging subdir/x + warning: conflicts while merging subdir/x! (edit, then use 'hg resolve --mark') + unresolved conflicts (see hg resolve, then hg rebase --continue) + [1] + $ printf "x\nf1\nf11\nf111\n" > subdir/x + $ hg resolve --mark subdir/x + (no more unresolved files) + continue: hg rebase --continue + + $ hg rebase --continue + rebasing 4:3795bd66ca70 "flat only commit 1 over flat only commit 2 at level 2" (tip) + 1 trees fetched over * (glob) + saved backup bundle to $TESTTMP/client/.hg/strip-backup/3795bd66ca70-6fdcea5b-rebase.hg (glob) + +- Tree manifest is fetched for 'nullid' during successful rebase as it is one of +the parents of the new commit created during the rebase. + + $ ls_l $CACHEDIR/master/packs/manifests + -r--r--r-- 1196 1b69dc04d7f9d9825351f0af940c80f956e372b9.histidx + -r--r--r-- 183 1b69dc04d7f9d9825351f0af940c80f956e372b9.histpack + -r--r--r-- 1106 4d21ecb6c95e12dcf807b793cd1c55eeed861734.dataidx + -r--r--r-- 211 4d21ecb6c95e12dcf807b793cd1c55eeed861734.datapack + -r--r--r-- 1066 586af18777dca881e6116614c23ea81897ceaf9a.dataidx + -r--r--r-- 71 586af18777dca881e6116614c23ea81897ceaf9a.datapack + -r--r--r-- 1116 6104a6b3ab2f32efe4835b65d9af7d7359bc1d9b.histidx + -r--r--r-- 89 6104a6b3ab2f32efe4835b65d9af7d7359bc1d9b.histpack + +- Tree manifest for 'nullid'. + + $ hg debugdatapack $CACHEDIR/master/packs/manifests/586af18777dca881e6116614c23ea81897ceaf9a.datapack + $TESTTMP/hgcache/master/packs/manifests/586af18777dca881e6116614c23ea81897ceaf9a: + (empty name): + Node Delta Base Delta Length Blob Size + 000000000000 000000000000 0 (missing) + + +- In the local store, we expect to see the tree manifests for commits +'9055b56f3916', 'f7febcf0f689', '3795bd66ca70' and the rebased commit. + + $ ls_l .hg/store/packs/manifests + -r--r--r-- 1196 028534a0bedee7c3d57bd6fd459f16abe971621b.histidx + -r--r--r-- 183 028534a0bedee7c3d57bd6fd459f16abe971621b.histpack + -r--r--r-- 1106 1074860af987f99d7c9e6d053852060e47ed05bd.dataidx + -r--r--r-- 211 1074860af987f99d7c9e6d053852060e47ed05bd.datapack + -r--r--r-- 1106 29e9cb3fed65ea279f701bd51552fbfb0cf0a91a.dataidx + -r--r--r-- 211 29e9cb3fed65ea279f701bd51552fbfb0cf0a91a.datapack + -r--r--r-- 1106 348a64c21d4cc36cb149a14f1d3d510658ea9432.dataidx + -r--r--r-- 211 348a64c21d4cc36cb149a14f1d3d510658ea9432.datapack + -r--r--r-- 1196 3bc37051b4b3371aa5e9a05d06390f94d0fb156f.histidx + -r--r--r-- 183 3bc37051b4b3371aa5e9a05d06390f94d0fb156f.histpack + -r--r--r-- 1106 5a179c0bb6419ffadbed2c826f2e17b95b05bafb.dataidx + -r--r--r-- 211 5a179c0bb6419ffadbed2c826f2e17b95b05bafb.datapack + -r--r--r-- 1196 5ef2ade5f4492a25bed3204d2e5b6040e5f5f96e.histidx + -r--r--r-- 183 5ef2ade5f4492a25bed3204d2e5b6040e5f5f96e.histpack + -r--r--r-- 1196 d3a99c21dc531b5f9a4a6cd618258cd0c6f952e4.histidx + -r--r--r-- 183 d3a99c21dc531b5f9a4a6cd618258cd0c6f952e4.histpack + +- We already know '1074860af987f99d7c9e6d053852060e47ed05bd' is tree manifest +data for 'f7febcf0f689' and '5a179cyaw0bb6419ffadbed2c826f2e17b95b05bafb' is +tree manifest data for '9055b56f3916'. Lets check the remaining 2. + +- Tree manifest data for '3795bd66ca70'. + $ hg debugdatapack .hg/store/packs/manifests/348a64c21d4cc36cb149a14f1d3d510658ea9432.datapack + .hg/store/packs/manifests/348a64c21d4cc36cb149a14f1d3d510658ea9432: + subdir: + Node Delta Base Delta Length Blob Size + 31a1621c0fb2 000000000000 43 (missing) + + (empty name): + Node Delta Base Delta Length Blob Size + b7db2b1fa98f 000000000000 49 (missing) + +- Tree manifest data for the rebased commit. + $ hg debugdatapack .hg/store/packs/manifests/29e9cb3fed65ea279f701bd51552fbfb0cf0a91a.datapack + .hg/store/packs/manifests/29e9cb3fed65ea279f701bd51552fbfb0cf0a91a: + subdir: + Node Delta Base Delta Length Blob Size + a3d24c48d9c8 000000000000 43 (missing) + + (empty name): + Node Delta Base Delta Length Blob Size + c3066277f7a4 000000000000 49 (missing) + +- Clean up generated tree manifests for remaining tests. + + $ rm -fr $CACHEDIR/master/packs/manifests/586af18777dca881e6116614c23ea81897ceaf9a.* + $ rm -fr $CACHEDIR/master/packs/manifests/6104a6b3ab2f32efe4835b65d9af7d7359bc1d9b.* + $ rm -rf .hg/store/packs + +- Check that 'treemanifest.demandgenerate' works for repo with only local +commits. This tests that the on demand conversion of flat manifest to tree +manifest works even if no ancestor with tree manifest is found. + + $ cd .. + $ hgcloneshallow ssh://user@dummy/master localcommitrepo -q --noupdate + $ cd localcommitrepo + $ echo f1 >> file + $ hg commit -Aqm 'flat only commit 1' + $ echo f2 >> file + $ hg commit -qm 'flat only commit 2' + $ cat >> .hg/hgrc < [extensions] + > treemanifest=$TESTDIR/../treemanifest + > fastmanifest=$TESTDIR/../fastmanifest + > [treemanifest] + > treeonly=True + > EOF + $ hg log -vpr 'tip' --config treemanifest.demandgenerate=False \ + > 2>&1 > /dev/null | tail -1 + KeyError: 'tree node not found (, bf46d105de7166f0e87629abb964f6e9bfd9b4c5)' + +- The fetched tree corresponds to the tree manifest for 'nullid' as seen in the +test cases above. + + $ hg log -vpr 'tip' + 1 trees fetched over * (glob) + changeset: 2:fe34b737c20f + tag: tip + user: test + date: Thu Jan 01 00:00:00 1970 +0000 + files: file + description: + flat only commit 2 + + + diff -r b2328d663f73 -r fe34b737c20f file + --- a/file Thu Jan 01 00:00:00 1970 +0000 + +++ b/file Thu Jan 01 00:00:00 1970 +0000 + @@ -1,1 +1,2 @@ + f1 + +f2 + +- Clean up generated tree manifest corresponding to 'nullid' revision for +remaining tests. + + $ rm -fr $CACHEDIR/master/packs/manifests/586af18777dca881e6116614c23ea81897ceaf9a.* + + $ cd ../client + Make a local hybrid flat+tree draft commit $ echo h >> subdir/x $ ls_l .hg/store | grep packs diff --git a/treemanifest/__init__.py b/treemanifest/__init__.py --- a/treemanifest/__init__.py +++ b/treemanifest/__init__.py @@ -36,6 +36,14 @@ [treemanifest] demanddownload = True +Disabling `treemanifest.demandgenerate` will prevent the extension from +automatically generating tree manifest from corresponding flat manifest when +it doesn't exist locally. Note that this setting is only relevant in treeonly +mode. + + [treemanifest] + demandgenerate = True + Setting `treemanifest.pullprefetchcount` to an integer N will cause the latest N commits' manifests to be downloaded (if they aren't already). @@ -323,9 +331,17 @@ if ui.configbool('treemanifest', 'demanddownload', True): stores.append(remotedatastore) - mfl.datastore = unioncontentstore(*stores, + ondemanddatastore = ondemandtreedatastore(repo) + if (ui.configbool('treemanifest', 'treeonly') and + ui.configbool('treemanifest', 'demandgenerate', True)): + stores.append(ondemanddatastore) + + shareddatastore = unioncontentstore(*stores, writestore=localdatastore) - remotedatastore.setshared(mfl.datastore) + + remotedatastore.setshared(shareddatastore) + ondemanddatastore.setshared(shareddatastore) + mfl.datastore = shareddatastore mfl.shareddatastores = [datastore] mfl.localdatastores = [localdatastore] @@ -1623,6 +1639,15 @@ prefetchtrees(self._repo, name, [node], basemfnodes, []) self._shared.markforrefresh() +class ondemandtreedatastore(generatingdatastore): + def _gettrees(self, name, node): + repo = self._repo + with repo.wlock(), repo.lock(), repo.transaction('demandtreegen') as tr: + mfl = manifest.manifestlog(repo.svfs, repo) + tmfl = repo.manifestlog + mfctx = manifest.manifestctx(mfl, node) + _converttotree(tr, mfl, tmfl, mfctx) + def serverrepack(repo, incremental=False, options=None): packpath = repo.vfs.join('cache/packs/%s' % PACK_CATEGORY)