diff --git a/remotefilelog/repack.py b/remotefilelog/repack.py --- a/remotefilelog/repack.py +++ b/remotefilelog/repack.py @@ -47,7 +47,7 @@ repo, constants.FILEPACK_CATEGORY) _runrepack(repo, datasource, historysource, packpath, - constants.FILEPACK_CATEGORY) + constants.FILEPACK_CATEGORY, fullhistory=historysource) if repo.ui.configbool('treemanifest', 'server'): treemfmod = extensions.find('treemanifest') @@ -73,7 +73,7 @@ *lhstores, allowincomplete=True) _runrepack(repo, datasource, historysource, lpackpath, - constants.TREEPACK_CATEGORY) + constants.TREEPACK_CATEGORY, fullhistory=historysource) def incrementalrepack(repo): @@ -142,25 +142,38 @@ files = osutil.listdir(packpath, stat=True) - datapacks = _computeincrementaldatapack(repo.ui, files) - fullpaths = list(os.path.join(packpath, p) for p in datapacks) - datapacks = list(datapack.datapack(p) for p in fullpaths) + def topacks(files, constructor): + paths = list(os.path.join(packpath, p) for p in files) + packs = list(constructor(p) for p in paths) + return packs + + datapacks = topacks(_computeincrementaldatapack(repo.ui, files), + datapack.datapack) datapacks.extend(s for s in datastore if not isinstance(s, datapack.datapackstore)) - historypacks = _computeincrementalhistorypack(repo.ui, files) - fullpaths = list(os.path.join(packpath, p) for p in historypacks) - historypacks = list(historypack.historypack(p) for p in fullpaths) + historypacks = topacks(_computeincrementalhistorypack(repo.ui, files), + historypack.historypack) historypacks.extend(s for s in historystore if not isinstance(s, historypack.historypackstore)) - datasource = contentstore.unioncontentstore( - *datapacks, - allowincomplete=allowincompletedata) - historysource = metadatastore.unionmetadatastore(*historypacks, - allowincomplete=True) - - _runrepack(repo, datasource, historysource, packpath, category) + allhistoryfiles = _allpackfileswithsuffix(files, historypack.PACKSUFFIX, + historypack.INDEXSUFFIX) + allhistorypacks = topacks((f for f, mode, stat in allhistoryfiles), + historypack.historypack) + allhistorypacks.extend(s for s in historystore + if not isinstance(s, historypack.historypackstore)) + _runrepack(repo, + contentstore.unioncontentstore( + *datapacks, + allowincomplete=allowincompletedata), + metadatastore.unionmetadatastore( + *historypacks, + allowincomplete=True), + packpath, category, + fullhistory=metadatastore.unionmetadatastore( + *allhistorypacks, + allowincomplete=True)) def _computeincrementaldatapack(ui, files): """Given a set of pack files and a set of generation size limits, this @@ -202,13 +215,8 @@ historypack.PACKSUFFIX, historypack.INDEXSUFFIX, gencountlimit, repacksizelimit, maxrepackpacks) -def _computeincrementalpack(ui, files, limits, packsuffix, indexsuffix, - gencountlimit, repacksizelimit, maxrepackpacks): - # Group the packs by generation (i.e. by size) - generations = [] - for i in xrange(len(limits)): - generations.append([]) - sizes = {} +def _allpackfileswithsuffix(files, packsuffix, indexsuffix): + result = [] fileset = set(fn for fn, mode, stat in files) for filename, mode, stat in files: if not filename.endswith(packsuffix): @@ -219,7 +227,20 @@ # Don't process a pack if it doesn't have an index. if (prefix + indexsuffix) not in fileset: continue + result.append((filename[:-len(packsuffix)], mode, stat)) + return result + +def _computeincrementalpack(ui, files, limits, packsuffix, indexsuffix, + gencountlimit, repacksizelimit, maxrepackpacks): + # Group the packs by generation (i.e. by size) + generations = [] + for i in xrange(len(limits)): + generations.append([]) + + sizes = {} + for prefix, mode, stat in _allpackfileswithsuffix(files, packsuffix, + indexsuffix): size = stat.st_size sizes[prefix] = size for i, limit in enumerate(limits): @@ -257,7 +278,7 @@ return chosenpacks -def _runrepack(repo, data, history, packpath, category): +def _runrepack(repo, data, history, packpath, category, fullhistory=None): shallowutil.mkstickygroupdir(repo.ui, packpath) def isold(repo, filename, node): @@ -275,7 +296,10 @@ return filetime[0] < limit garbagecollect = repo.ui.configbool('remotefilelog', 'gcrepack') - packer = repacker(repo, data, history, category, gc=garbagecollect, + if not fullhistory: + fullhistory = history + packer = repacker(repo, data, history, fullhistory, category, + gc=garbagecollect, isold=isold) # internal config: remotefilelog.datapackversion @@ -354,10 +378,11 @@ """Class for orchestrating the repack of data and history information into a new format. """ - def __init__(self, repo, data, history, category, gc=False, isold=None): + def __init__(self, repo, data, history, fullhistory, category, gc=False, isold=None): self.repo = repo self.data = data self.history = history + self.fullhistory = fullhistory self.unit = constants.getunits(category) self.garbagecollect = gc if self.garbagecollect: @@ -409,7 +434,7 @@ ui.progress(_("building history"), i, unit='nodes', total=len(nodes)) try: - ancestors.update(self.history.getancestors(filename, node, + ancestors.update(self.fullhistory.getancestors(filename, node, known=ancestors)) except KeyError: # Since we're packing data entries, we may not have the @@ -420,6 +445,9 @@ # Order the nodes children first, so we can produce reverse deltas orderednodes = list(reversed(self._toposort(ancestors))) + if len(nohistory) > 0: + ui.warn('repackdata: %d nodes without history\n' % + len(nohistory)) orderednodes.extend(sorted(nohistory)) # Compute deltas and write to the pack diff --git a/tests/test-remotefilelog-gcrepack.t b/tests/test-remotefilelog-gcrepack.t --- a/tests/test-remotefilelog-gcrepack.t +++ b/tests/test-remotefilelog-gcrepack.t @@ -8,6 +8,7 @@ $ cat >> .hg/hgrc < [remotefilelog] > server=True + > history.maxrepackpacks=0 > EOF $ echo x > x $ hg commit -qAm x @@ -40,7 +41,7 @@ $ hg prefetch 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob) - $ hg repack + $ hg repack --config history.maxrepackpacks=0 $ sleep 0.5 $ hg debugwaitonrepack >/dev/null 2>%1 @@ -70,7 +71,7 @@ > nodettl=86400 > EOF - $ hg repack + $ hg repack --config history.maxrepackpacks=0 $ sleep 0.5 $ hg debugwaitonrepack >/dev/null 2>%1 @@ -96,7 +97,7 @@ $ hg prefetch 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob) - $ hg repack + $ hg repack --config history.maxrepackpacks=0 $ sleep 0.5 $ hg debugwaitonrepack >/dev/null 2>%1 @@ -127,7 +128,7 @@ > nodettl=$(($(date +%s) + 100)) > EOF - $ hg repack + $ hg repack --config history.maxrepackpacks=0 $ sleep 0.5 $ hg debugwaitonrepack >/dev/null 2>%1 diff --git a/tests/test-remotefilelog-lfs.t b/tests/test-remotefilelog-lfs.t --- a/tests/test-remotefilelog-lfs.t +++ b/tests/test-remotefilelog-lfs.t @@ -90,7 +90,7 @@ # lfs content could be read after repack - $ hg repack + $ hg repack --config history.maxrepackpacks=0 $ find $CACHEDIR | sort $TESTTMP/hgcache @@ -228,7 +228,7 @@ $ cd ../shallow - $ hg repack + $ hg repack --config history.maxrepackpacks=0 $ find $CACHEDIR | sort $TESTTMP/hgcache $TESTTMP/hgcache/master diff --git a/tests/test-remotefilelog-repack-fast.t b/tests/test-remotefilelog-repack-fast.t --- a/tests/test-remotefilelog-repack-fast.t +++ b/tests/test-remotefilelog-repack-fast.t @@ -44,7 +44,7 @@ $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/aee31534993a501858fb6dd96a065671922e7d51 $TESTTMP/hgcache/repos - $ hg repack + $ hg repack --config history.maxrepackpacks=0 $ find $CACHEDIR | sort $TESTTMP/hgcache @@ -87,7 +87,7 @@ $TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack $TESTTMP/hgcache/repos - $ hg repack --traceback + $ hg repack --config history.maxrepackpacks=0 --traceback $ find $CACHEDIR -type f | sort $TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histidx @@ -107,7 +107,7 @@ # Test that repacking again without new data does not delete the pack files # and did not change the pack names - $ hg repack + $ hg repack --config history.maxrepackpacks=0 $ find $CACHEDIR -type f | sort $TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histidx $TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack @@ -116,9 +116,9 @@ $TESTTMP/hgcache/repos # Run two repacks at once - $ hg repack --config "hooks.prerepack=sleep 3" & + $ hg repack --config history.maxrepackpacks=0 --config "hooks.prerepack=sleep 3" & $ sleep 1 - $ hg repack + $ hg repack --config history.maxrepackpacks=0 abort: skipping repack - another repack is already running [255] $ hg debugwaitonrepack >/dev/null 2>&1 @@ -139,7 +139,7 @@ $TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack $TESTTMP/hgcache/repos - $ hg repack --background + $ hg repack --config history.maxrepackpacks=0 --background (running background repack) $ sleep 0.5 $ hg debugwaitonrepack >/dev/null 2>&1 @@ -190,7 +190,7 @@ $ hg pull -q $ hg up -q tip 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) - $ hg repack + $ hg repack --config history.maxrepackpacks=0 $ hg log -f y -T '{desc}\n' move x to y x4 @@ -213,7 +213,7 @@ $ hg pull -q $ hg up -q tip 2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over * (glob) - $ hg repack + $ hg repack --config history.maxrepackpacks=0 $ ls $TESTTMP/hgcache/master/packs e634f60d2a9539fc595b1f4db480c64556a396c7.dataidx e634f60d2a9539fc595b1f4db480c64556a396c7.datapack @@ -245,7 +245,7 @@ # Test repacking datapack without history $ rm -rf $CACHEDIR/master/packs/*hist* - $ hg repack + $ hg repack --config history.maxrepackpacks=0 $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack x @@ -280,7 +280,7 @@ -r--r--r-- 59 5b7dec902026f0cddb0ef8acb62f27b5698494d4.datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack -r--r--r-- 90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack - $ hg repack --incremental + $ hg repack --config history.maxrepackpacks=0 --incremental $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack -r--r--r-- 59 5b7dec902026f0cddb0ef8acb62f27b5698494d4.datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack @@ -303,7 +303,7 @@ -r--r--r-- 336 094b530486dad4427a0faf6bcbc031571b99ca24.histpack -r--r--r-- 172 276d308429d0303762befa376788300f0310f90e.histpack -r--r--r-- 90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack - $ hg repack --incremental + $ hg repack --config history.maxrepackpacks=0 --incremental $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack -r--r--r-- 59 5b7dec902026f0cddb0ef8acb62f27b5698494d4.datapack -r--r--r-- 213 f3c56163b762b2931865bbbb1250b4fae09e782c.datapack @@ -311,7 +311,7 @@ -r--r--r-- 336 094b530486dad4427a0faf6bcbc031571b99ca24.histpack 1 gen3 pack, 1 gen0 pack - does nothing - $ hg repack --incremental + $ hg repack --config history.maxrepackpacks=0 --incremental $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack -r--r--r-- 59 5b7dec902026f0cddb0ef8acb62f27b5698494d4.datapack -r--r--r-- 213 f3c56163b762b2931865bbbb1250b4fae09e782c.datapack diff --git a/tests/test-remotefilelog-repack.t b/tests/test-remotefilelog-repack.t --- a/tests/test-remotefilelog-repack.t +++ b/tests/test-remotefilelog-repack.t @@ -39,7 +39,7 @@ $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/aee31534993a501858fb6dd96a065671922e7d51 $TESTTMP/hgcache/repos - $ hg repack + $ hg repack --config history.maxrepackpacks=0 $ find $CACHEDIR | sort $TESTTMP/hgcache @@ -82,7 +82,7 @@ $TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack $TESTTMP/hgcache/repos - $ hg repack --traceback + $ hg repack --config history.maxrepackpacks=0 --traceback $ find $CACHEDIR -type f | sort $TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histidx @@ -102,7 +102,7 @@ # Test that repacking again without new data does not delete the pack files # and did not change the pack names - $ hg repack + $ hg repack --config history.maxrepackpacks=0 $ find $CACHEDIR -type f | sort $TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histidx $TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack @@ -111,9 +111,9 @@ $TESTTMP/hgcache/repos # Run two repacks at once - $ hg repack --config "hooks.prerepack=sleep 3" & + $ hg repack --config history.maxrepackpacks=0 --config "hooks.prerepack=sleep 3" & $ sleep 1 - $ hg repack + $ hg repack --config history.maxrepackpacks=0 abort: skipping repack - another repack is already running [255] $ hg debugwaitonrepack >/dev/null 2>&1 @@ -134,7 +134,7 @@ $TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack $TESTTMP/hgcache/repos - $ hg repack --background + $ hg repack --config history.maxrepackpacks=0 --background (running background repack) $ sleep 0.5 $ hg debugwaitonrepack >/dev/null 2>&1 @@ -185,7 +185,7 @@ $ hg pull -q $ hg up -q tip 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) - $ hg repack + $ hg repack --config history.maxrepackpacks=0 $ hg log -f y -T '{desc}\n' move x to y x4 @@ -208,7 +208,7 @@ $ hg pull -q $ hg up -q tip 2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over * (glob) - $ hg repack + $ hg repack --config history.maxrepackpacks=0 $ ls $TESTTMP/hgcache/master/packs e634f60d2a9539fc595b1f4db480c64556a396c7.dataidx e634f60d2a9539fc595b1f4db480c64556a396c7.datapack diff --git a/tests/test-treemanifest-repack.t b/tests/test-treemanifest-repack.t --- a/tests/test-treemanifest-repack.t +++ b/tests/test-treemanifest-repack.t @@ -19,7 +19,6 @@ > [remotefilelog] > usefastdatapack=True > reponame=master - > > [fastmanifest] > usetree=True > usecache=False @@ -78,7 +77,7 @@ 23226e7a252c 000000000000 000000000000 8e83608cbe60 - Repack and reverify - $ hg repack + $ hg repack --config history.maxrepackpacks=0 $ ls_l $CACHEDIR/master/packs/manifests | grep pack -r--r--r-- 262 7535b6084226436bbdff33043969e7fa963e8428.histpack @@ -113,7 +112,7 @@ $ ls_l .hg/store/packs/manifests | grep datapack -r--r--r-- 248 5d1716bbef6e7200192de6509055d1ee31a4172c.datapack -r--r--r-- 146 cffef142da32f3e52c1779490e5d0ddac5f9b82b.datapack - $ hg repack + $ hg repack --config history.maxrepackpacks=0 $ ls_l .hg/store/packs/manifests | grep datapack -r--r--r-- 374 201094db51b761cd78352c055b3135178aadfec5.datapack @@ -126,7 +125,7 @@ -r--r--r-- 248 d7e689a91ac63385be120a118af9ce8663748f28.datapack - repack incremental does nothing here because there are so few packs - $ hg repack --incremental --config remotefilelog.data.generations=300,200 --config remotefilelog.data.repacksizelimit=300 + $ hg repack --config history.maxrepackpacks=0 --incremental --config remotefilelog.data.generations=300,200 --config remotefilelog.data.repacksizelimit=300 $ ls_l .hg/store/packs/manifests | grep datapack -r--r--r-- 374 201094db51b761cd78352c055b3135178aadfec5.datapack -r--r--r-- 248 21501384df03b8489b366c5218be639fa08830e4.datapack @@ -154,11 +153,58 @@ - repack incremental kicks in once there are a number of packs - (set the repacksizelimit so that we test that we only repack up to 1500 bytes, - and it leaves one datapack behind) - $ hg repack --incremental --config remotefilelog.data.generations=300,200 --config remotefilelog.data.repacksizelimit=1500B + $ cp -R .hg/store/packs .hg/store/packs.bak + $ hg repack --config history.maxrepackpacks=0 --incremental --config remotefilelog.data.generations=300,200 --config remotefilelog.data.repacksizelimit=1500B $ ls_l .hg/store/packs/manifests | grep datapack | wc -l .*3 (re) $ ls_l .hg/store/packs/manifests | grep datapack | grep 248 -r--r--r-- 248 *.datapack (glob) + +- Verify that full history is used to choose node order for data repack, even if +- we actually repack fewer history packs. + $ rm -r .hg/store/packs + $ mv .hg/store/packs.bak .hg/store/packs + +- history.maxrepackpacks=0 is passed to ensure no history packs are repacked. If +- any entries have missing ancestry, a debug warning is printed. + $ hg repack --config history.maxrepackpacks=0 --config remotefilelog.data.generations=300,200 --config remotefilelog.data.repacksizelimit=1500B --config remotefilelog.history.maxrepackpacks=0 --debug + +- Inspect all data packs. We except to see none with > 1 delta bases of 000000000000 +- (which would indicate we made duplicative chains.) + $ ls .hg/store/packs/manifests/*.datapack | xargs -L1 hg debugdatapack + + + Node Delta Base Delta Length + edbe899c31e7 000000000000 89 + 23d6a1af5d5b edbe899c31e7 58 + 53dd6ae0d984 23d6a1af5d5b 58 + 10d05e165f71 53dd6ae0d984 58 + fba208634b61 10d05e165f71 58 + e24ac4639900 fba208634b61 58 + 1e27a5956e61 e24ac4639900 58 + + dir + Node Delta Base Delta Length + 31a760f1fb09 000000000000 43 + d62223eee804 31a760f1fb09 55 + 6bfa2a87412d d62223eee804 55 + 984db90696cd 6bfa2a87412d 55 + 84977318a34f 984db90696cd 55 + f175c774e18b 84977318a34f 55 + + + + + + + + + + + + + + - Clean up the pile of packs we made $ hg repack