diff --git a/remotefilelog/debugcommands.py b/remotefilelog/debugcommands.py --- a/remotefilelog/debugcommands.py +++ b/remotefilelog/debugcommands.py @@ -6,6 +6,7 @@ # GNU General Public License version 2 or any later version. from __future__ import absolute_import +from collections import defaultdict from mercurial import error, filelog, revlog from mercurial.node import bin, hex, nullid, short from mercurial.i18n import _ @@ -216,9 +217,26 @@ hashlen = 14 lastfilename = None + totaldeltasize = defaultdict(lambda: 0) + totalblobsize = defaultdict(lambda: 0) + def printtotals(filename): + if not (filename in totaldeltasize or filename in totaldeltasize): + return + difference = totalblobsize[filename] - totaldeltasize[filename] + deltastr = "%0.1f%% %s" % ( + (100.0 * abs(difference) / totalblobsize[filename]), + ("smaller" if difference > 0 else "bigger")) + + ui.write("Total:%s%s %s (%s)\n" % ( + "".ljust(2 * hashlen - len("Total:")), + str(totaldeltasize[filename]).ljust(12), + str(totalblobsize[filename]).ljust(9), + deltastr + )) for filename, node, deltabase, deltalen in dpack.iterentries(): if filename != lastfilename: - ui.write("\n%s\n" % filename) + printtotals(lastfilename) + ui.write("\n%s:\n" % ('(Root)' if filename == '' else filename)) ui.write("%s%s%s%s\n" % ( "Node".ljust(hashlen), "Delta Base".ljust(hashlen), @@ -229,6 +247,8 @@ meta = dpack.getmeta(filename, node) if constants.METAKEYSIZE in meta: blobsize = meta[constants.METAKEYSIZE] + totaldeltasize[filename] += deltalen + totalblobsize[filename] += blobsize else: blobsize = "(missing)" ui.write("%s %s %s%s\n" % ( @@ -236,6 +256,9 @@ hashformatter(deltabase), str(deltalen).ljust(14), blobsize)) + if filename is not None: + printtotals(filename) + def dumpdeltachain(ui, deltachain, **opts): hashformatter = hex diff --git a/tests/test-remotefilelog-bgprefetch.t b/tests/test-remotefilelog-bgprefetch.t --- a/tests/test-remotefilelog-bgprefetch.t +++ b/tests/test-remotefilelog-bgprefetch.t @@ -153,22 +153,26 @@ $ hg debugdatapack $TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.datapack - w + w: Node Delta Base Delta Length Blob Size bb6ccd5dceaa 000000000000 2 2 + Total: 2 2 (0.0% bigger) - x + x: Node Delta Base Delta Length Blob Size ef95c5376f34 000000000000 3 3 1406e7411862 ef95c5376f34 14 2 + Total: 17 5 (240.0% bigger) - y + y: Node Delta Base Delta Length Blob Size 076f5e2225b3 000000000000 2 2 + Total: 2 2 (0.0% bigger) - z + z: Node Delta Base Delta Length Blob Size 69a1b6752270 000000000000 2 2 + Total: 2 2 (0.0% bigger) # background prefetch with repack on commit when wcprevset configured @@ -199,22 +203,26 @@ $ hg debugdatapack $TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.datapack - w + w: Node Delta Base Delta Length Blob Size bb6ccd5dceaa 000000000000 2 2 + Total: 2 2 (0.0% bigger) - x + x: Node Delta Base Delta Length Blob Size ef95c5376f34 000000000000 3 3 1406e7411862 ef95c5376f34 14 2 + Total: 17 5 (240.0% bigger) - y + y: Node Delta Base Delta Length Blob Size 076f5e2225b3 000000000000 2 2 + Total: 2 2 (0.0% bigger) - z + z: Node Delta Base Delta Length Blob Size 69a1b6752270 000000000000 2 2 + Total: 2 2 (0.0% bigger) # background prefetch with repack on rebase when wcprevset configured @@ -237,22 +245,26 @@ $ hg debugdatapack $TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.datapack - w + w: Node Delta Base Delta Length Blob Size bb6ccd5dceaa 000000000000 2 2 + Total: 2 2 (0.0% bigger) - x + x: Node Delta Base Delta Length Blob Size ef95c5376f34 000000000000 3 3 1406e7411862 ef95c5376f34 14 2 + Total: 17 5 (240.0% bigger) - y + y: Node Delta Base Delta Length Blob Size 076f5e2225b3 000000000000 2 2 + Total: 2 2 (0.0% bigger) - z + z: Node Delta Base Delta Length Blob Size 69a1b6752270 000000000000 2 2 + Total: 2 2 (0.0% bigger) # Check that foregound prefetch with no arguments blocks until background prefetches finish @@ -278,22 +290,26 @@ # Ensure that files were prefetched $ hg debugdatapack $TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.datapack - w + w: Node Delta Base Delta Length Blob Size bb6ccd5dceaa 000000000000 2 2 + Total: 2 2 (0.0% bigger) - x + x: Node Delta Base Delta Length Blob Size ef95c5376f34 000000000000 3 3 1406e7411862 ef95c5376f34 14 2 + Total: 17 5 (240.0% bigger) - y + y: Node Delta Base Delta Length Blob Size 076f5e2225b3 000000000000 2 2 + Total: 2 2 (0.0% bigger) - z + z: Node Delta Base Delta Length Blob Size 69a1b6752270 000000000000 2 2 + Total: 2 2 (0.0% bigger) # Check that foreground prefetch fetches revs specified by '. + draft() + bgprefetchrevs + pullprefetch' @@ -316,22 +332,26 @@ # Ensure that files were prefetched $ hg debugdatapack $TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.datapack - w + w: Node Delta Base Delta Length Blob Size bb6ccd5dceaa 000000000000 2 2 + Total: 2 2 (0.0% bigger) - x + x: Node Delta Base Delta Length Blob Size ef95c5376f34 000000000000 3 3 1406e7411862 ef95c5376f34 14 2 + Total: 17 5 (240.0% bigger) - y + y: Node Delta Base Delta Length Blob Size 076f5e2225b3 000000000000 2 2 + Total: 2 2 (0.0% bigger) - z + z: Node Delta Base Delta Length Blob Size 69a1b6752270 000000000000 2 2 + Total: 2 2 (0.0% bigger) # Test that if data was prefetched and repacked we dont need to prefetch it again # It ensures that Mercurial looks not only in loose files but in packs as well diff --git a/tests/test-remotefilelog-gcrepack.t b/tests/test-remotefilelog-gcrepack.t --- a/tests/test-remotefilelog-gcrepack.t +++ b/tests/test-remotefilelog-gcrepack.t @@ -52,14 +52,16 @@ $ hg debugdatapack $TESTTMP/hgcache/master/packs/f7a942a6e4673d2c7b697fdd926ca2d153831ca4.datapack - x + x: Node Delta Base Delta Length Blob Size 1406e7411862 000000000000 2 2 + Total: 2 2 (0.0% bigger) - y + y: Node Delta Base Delta Length Blob Size 50dbc4572b8e 000000000000 3 3 076f5e2225b3 50dbc4572b8e 14 2 + Total: 17 5 (240.0% bigger) # Test garbage collection during repack @@ -83,9 +85,10 @@ $ hg debugdatapack $TESTTMP/hgcache/master/packs/05baa499c6b07f2bf0ea3d2c8151da1cb86f5e33.datapack - y + y: Node Delta Base Delta Length Blob Size 50dbc4572b8e 000000000000 3 3 + Total: 3 3 (0.0% bigger) # Prefetch all data again and repack for later garbage collection @@ -108,14 +111,16 @@ $ hg debugdatapack $TESTTMP/hgcache/master/packs/f7a942a6e4673d2c7b697fdd926ca2d153831ca4.datapack - x + x: Node Delta Base Delta Length Blob Size 1406e7411862 000000000000 2 2 + Total: 2 2 (0.0% bigger) - y + y: Node Delta Base Delta Length Blob Size 50dbc4572b8e 000000000000 3 3 076f5e2225b3 50dbc4572b8e 14 2 + Total: 17 5 (240.0% bigger) # Test garbage collection during repack. Ensure that new files are not removed even though they are not in the keepset # For the purposes of the test the TTL of a file is set to current time + 100 seconds. i.e. all commits in tests have @@ -139,11 +144,13 @@ $ hg debugdatapack $TESTTMP/hgcache/master/packs/f7a942a6e4673d2c7b697fdd926ca2d153831ca4.datapack - x + x: Node Delta Base Delta Length Blob Size 1406e7411862 000000000000 2 2 + Total: 2 2 (0.0% bigger) - y + y: Node Delta Base Delta Length Blob Size 50dbc4572b8e 000000000000 3 3 076f5e2225b3 50dbc4572b8e 14 2 + Total: 17 5 (240.0% bigger) diff --git a/tests/test-remotefilelog-repack-fast.t b/tests/test-remotefilelog-repack-fast.t --- a/tests/test-remotefilelog-repack-fast.t +++ b/tests/test-remotefilelog-repack-fast.t @@ -154,18 +154,20 @@ $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack - x + x: Node Delta Base Delta Length Blob Size 1bb2e6237e03 000000000000 8 8 d4a3ed9310e5 1bb2e6237e03 12 6 aee31534993a d4a3ed9310e5 12 4 + Total: 32 18 (77.8% bigger) $ hg debugdatapack --long $TESTTMP/hgcache/master/packs/*.datapack - x + x: Node Delta Base Delta Length Blob Size 1bb2e6237e035c8f8ef508e281f1ce075bc6db72 0000000000000000000000000000000000000000 8 8 d4a3ed9310e5bd9887e3bf779da5077efab28216 1bb2e6237e035c8f8ef508e281f1ce075bc6db72 12 6 aee31534993a501858fb6dd96a065671922e7d51 d4a3ed9310e5bd9887e3bf779da5077efab28216 12 4 + Total: 32 18 (77.8% bigger) $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack --node d4a3ed9310e5bd9887e3bf779da5077efab28216 x @@ -248,15 +250,17 @@ $ hg repack $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack - x + x: Node Delta Base Delta Length Blob Size 1bb2e6237e03 000000000000 8 8 aee31534993a 000000000000 4 4 d4a3ed9310e5 000000000000 6 6 + Total: 18 18 (0.0% bigger) - y + y: Node Delta Base Delta Length Blob Size 577959738234 000000000000 70 8 + Total: 70 8 (775.0% bigger) $ hg cat -r ".^" x x diff --git a/tests/test-remotefilelog-repack.t b/tests/test-remotefilelog-repack.t --- a/tests/test-remotefilelog-repack.t +++ b/tests/test-remotefilelog-repack.t @@ -149,18 +149,20 @@ $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack - x + x: Node Delta Base Delta Length Blob Size 1bb2e6237e03 000000000000 8 8 d4a3ed9310e5 1bb2e6237e03 12 6 aee31534993a d4a3ed9310e5 12 4 + Total: 32 18 (77.8% bigger) $ hg debugdatapack --long $TESTTMP/hgcache/master/packs/*.datapack - x + x: Node Delta Base Delta Length Blob Size 1bb2e6237e035c8f8ef508e281f1ce075bc6db72 0000000000000000000000000000000000000000 8 8 d4a3ed9310e5bd9887e3bf779da5077efab28216 1bb2e6237e035c8f8ef508e281f1ce075bc6db72 12 6 aee31534993a501858fb6dd96a065671922e7d51 d4a3ed9310e5bd9887e3bf779da5077efab28216 12 4 + Total: 32 18 (77.8% bigger) $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack --node d4a3ed9310e5bd9887e3bf779da5077efab28216 x @@ -243,15 +245,17 @@ $ hg repack $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack - x + x: Node Delta Base Delta Length Blob Size 1bb2e6237e03 000000000000 8 8 aee31534993a 000000000000 4 4 d4a3ed9310e5 000000000000 6 6 + Total: 18 18 (0.0% bigger) - y + y: Node Delta Base Delta Length Blob Size 577959738234 000000000000 70 8 + Total: 70 8 (775.0% bigger) $ hg cat -r ".^" x x @@ -385,13 +389,15 @@ $ hg repack --config packs.maxchainlen=1 $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.dataidx - x + x: Node Delta Base Delta Length Blob Size 1bb2e6237e03 000000000000 8 8 d4a3ed9310e5 1bb2e6237e03 12 6 aee31534993a 000000000000 4 4 1406e7411862 aee31534993a 12 2 + Total: 36 20 (80.0% bigger) - y + y: Node Delta Base Delta Length Blob Size 577959738234 000000000000 8 8 + Total: 8 8 (0.0% bigger) diff --git a/tests/test-treemanifest-prefetch.t b/tests/test-treemanifest-prefetch.t --- a/tests/test-treemanifest-prefetch.t +++ b/tests/test-treemanifest-prefetch.t @@ -67,27 +67,27 @@ 8adc618d23082c0a5311a4bbf9ac08b9b9672471.datapack $ hg debugdatapack --long $CACHEDIR/master/packs/manifests/*.dataidx - subdir + subdir: Node Delta Base Delta Length Blob Size ddb35f099a648a43a997aef53123bce309c794fd 0000000000000000000000000000000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size 1be4ab2126dd2252dcae6be2aac2561dd3ddcda0 0000000000000000000000000000000000000000 95 (missing) - dir + dir: Node Delta Base Delta Length Blob Size a18d21674e76d6aab2edb46810b20fbdbd10fb4b 0000000000000000000000000000000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size 60a7f7acb6bb5aaf93ca7d9062931b0f6a0d6db5 0000000000000000000000000000000000000000 95 (missing) - dir + dir: Node Delta Base Delta Length Blob Size bc0c2c938b929f98b1c31a8c5994396ebb096bf0 0000000000000000000000000000000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size ef362f8bbe8aa457b0cfc49f200cbeb7747984ed 0000000000000000000000000000000000000000 46 (missing) $ hg debughistorypack $CACHEDIR/master/packs/manifests/*.histidx @@ -131,11 +131,11 @@ $ hg debugdatapack $CACHEDIR/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx - dir + dir: Node Delta Base Delta Length Blob Size a18d21674e76 000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size 60a7f7acb6bb 000000000000 95 (missing) @@ -169,26 +169,26 @@ $ hg debugdatapack $CACHEDIR/master/packs/manifests/148e9eb32f473ea522c591c95be0f9e772be9675 - dir + dir: Node Delta Base Delta Length Blob Size bc0c2c938b92 000000000000 43 (missing) - subdir + subdir: Node Delta Base Delta Length Blob Size ddb35f099a64 000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size 1be4ab2126dd 000000000000 95 (missing) - Note that subdir/ is not downloaded again $ hg debugdatapack $CACHEDIR/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c - dir + dir: Node Delta Base Delta Length Blob Size a18d21674e76 000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size 60a7f7acb6bb 000000000000 95 (missing) @@ -217,27 +217,27 @@ 6 trees fetched over * (glob) $ hg debugdatapack $CACHEDIR/master/packs/manifests/*.dataidx - subdir + subdir: Node Delta Base Delta Length Blob Size ddb35f099a64 000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size 1be4ab2126dd 000000000000 95 (missing) - dir + dir: Node Delta Base Delta Length Blob Size a18d21674e76 000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size 60a7f7acb6bb 000000000000 95 (missing) - dir + dir: Node Delta Base Delta Length Blob Size bc0c2c938b92 000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size ef362f8bbe8a 000000000000 46 (missing) @@ -253,15 +253,15 @@ 3 trees fetched over * (glob) $ hg debugdatapack $CACHEDIR/master/packs/manifests/*.dataidx - dir + dir: Node Delta Base Delta Length Blob Size a18d21674e76 000000000000 43 (missing) - subdir + subdir: Node Delta Base Delta Length Blob Size ddb35f099a64 000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size 60a7f7acb6bb 000000000000 95 (missing) @@ -282,11 +282,11 @@ $TESTTMP/hgcache/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx $ hg debugdatapack $CACHEDIR/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx - dir + dir: Node Delta Base Delta Length Blob Size a18d21674e76 000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size 60a7f7acb6bb 000000000000 95 (missing) @@ -312,15 +312,15 @@ 3 trees fetched over * (glob) $ hg debugdatapack $CACHEDIR/master/packs/manifests/*.dataidx - dir + dir: Node Delta Base Delta Length Blob Size a18d21674e76 000000000000 43 (missing) - subdir + subdir: Node Delta Base Delta Length Blob Size ddb35f099a64 000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size 60a7f7acb6bb 000000000000 95 (missing) @@ -334,11 +334,11 @@ 2 trees fetched over * (glob) $ hg debugdatapack $CACHEDIR/master/packs/manifests/99050e724a9236121684523ba3f4db270e62fb58.dataidx - dir + dir: Node Delta Base Delta Length Blob Size bf22bc15398b 000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size aa52a49be522 000000000000 95 (missing) diff --git a/tests/test-treemanifest-repack.t b/tests/test-treemanifest-repack.t --- a/tests/test-treemanifest-repack.t +++ b/tests/test-treemanifest-repack.t @@ -44,16 +44,16 @@ > done $TESTTMP/hgcache/master/packs/manifests/65df85879cdd898607ee3f323a0b61edc7de25b8.datapack - + (Root): Node Delta Base Delta Length Blob Size a0c8bcbbb45c 000000000000 43 (missing) $TESTTMP/hgcache/master/packs/manifests/bb55d9105672c45d4f82df15bd091a555ef02c79.datapack - dir + dir: Node Delta Base Delta Length Blob Size 23226e7a252c 000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size 1832e0765de9 000000000000 89 (missing) @@ -86,14 +86,16 @@ $ hg debugdatapack $CACHEDIR/master/packs/manifests/*.datapack - + (Root): Node Delta Base Delta Length Blob Size 1832e0765de9 000000000000 89 89 a0c8bcbbb45c 1832e0765de9 12 43 + Total: 101 132 (23.5% smaller) - dir + dir: Node Delta Base Delta Length Blob Size 23226e7a252c 000000000000 43 43 + Total: 43 43 (0.0% bigger) $ hg debughistorypack $CACHEDIR/master/packs/manifests/*.histpack @@ -188,14 +190,16 @@ 7535b6084226436bbdff33043969e7fa963e8428.histpack $ hg debugdatapack .hg/cache/packs/manifests/*.datapack - + (Root): Node Delta Base Delta Length Blob Size 1832e0765de9 000000000000 89 89 a0c8bcbbb45c 1832e0765de9 12 43 + Total: 101 132 (23.5% smaller) - dir + dir: Node Delta Base Delta Length Blob Size 23226e7a252c 000000000000 43 43 + Total: 43 43 (0.0% bigger) Test incremental revlog repacking # 1. Make commit that we'll need to repack @@ -227,10 +231,12 @@ $ hg repack --incremental --config treemanifest.repackstartrev=1 --config treemanifest.repackendrev=1 $ hg debugdatapack .hg/cache/packs/manifests/*.datapack - + (Root): Node Delta Base Delta Length Blob Size 1832e0765de9 000000000000 89 89 + Total: 89 89 (0.0% bigger) - dir + dir: Node Delta Base Delta Length Blob Size 23226e7a252c 000000000000 43 43 + Total: 43 43 (0.0% bigger) diff --git a/tests/test-treemanifest-server.t b/tests/test-treemanifest-server.t --- a/tests/test-treemanifest-server.t +++ b/tests/test-treemanifest-server.t @@ -49,21 +49,21 @@ $ hg debugdatapack $CACHEDIR/master/packs/manifests/*.dataidx - subdir + subdir: Node Delta Base Delta Length Blob Size bc0c2c938b92 000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size 85b359fdb09e 000000000000 49 (missing) $ hg debugdatapack .hg/store/packs/manifests/*.dataidx - subdir2 + subdir2: Node Delta Base Delta Length Blob Size ddb35f099a64 000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size 54cbf534b62b 000000000000 99 (missing) diff --git a/tests/test-treemanifest.t b/tests/test-treemanifest.t --- a/tests/test-treemanifest.t +++ b/tests/test-treemanifest.t @@ -60,15 +60,15 @@ $ hg debugdatapack $CACHEDIR/master/packs/manifests/678f597a73b2b96f2e120c84ef8a84069a250266 - + (Root): Node Delta Base Delta Length Blob Size bc0c2c938b92 000000000000 43 (missing) - subdir + subdir: Node Delta Base Delta Length Blob Size ddb35f099a64 000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size 70f2c6726cec bc0c2c938b92 61 (missing) @@ -84,11 +84,11 @@ -r--r--r-- 254 a7f7e084adff88a01cf76909345be1e56ee704a9.datapack $ hg debugdatapack .hg/store/packs/manifests/a7f7e084adff88a01cf76909345be1e56ee704a9 - subdir + subdir: Node Delta Base Delta Length Blob Size ac728a786423 000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size 7a911436836f 000000000000 92 (missing) @@ -120,19 +120,19 @@ 4:8289b85c6a30 $ hg debugdatapack .hg/store/packs/manifests/5a5fb42e99986c90ac86b57d184561c44238b7b7.datapack - subdir + subdir: Node Delta Base Delta Length Blob Size a4e2f032ee0f 000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size 0d05c20bb7eb 000000000000 92 (missing) - subdir + subdir: Node Delta Base Delta Length Blob Size ad0a48a2ec1e 000000000000 43 (missing) - + (Root): Node Delta Base Delta Length Blob Size 8289b85c6a30 000000000000 92 (missing)