diff --git a/remotefilelog/repack.py b/remotefilelog/repack.py --- a/remotefilelog/repack.py +++ b/remotefilelog/repack.py @@ -476,13 +476,17 @@ if deltabase != nullid: deltabasetext = self.data.get(filename, deltabase) original = self.data.get(filename, node) + size = len(original) delta = mdiff.textdiff(deltabasetext, original) else: delta = self.data.get(filename, node) + size = len(delta) # TODO: don't use the delta if it's larger than the fulltext # TODO: don't use the delta if the chain is already long meta = self.data.getmeta(filename, node) + if constants.METAKEYSIZE not in meta: + meta[constants.METAKEYSIZE] = size target.add(filename, node, deltabase, delta, meta) entries[node].datarepacked = True diff --git a/tests/test-remotefilelog-repack-fast.t b/tests/test-remotefilelog-repack-fast.t --- a/tests/test-remotefilelog-repack-fast.t +++ b/tests/test-remotefilelog-repack-fast.t @@ -306,7 +306,7 @@ $ hg repack --incremental $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack -r--r--r-- 59 5b7dec902026f0cddb0ef8acb62f27b5698494d4.datapack - -r--r--r-- 213 f3c56163b762b2931865bbbb1250b4fae09e782c.datapack + -r--r--r-- 225 8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack -r--r--r-- 336 094b530486dad4427a0faf6bcbc031571b99ca24.histpack @@ -314,7 +314,7 @@ $ hg repack --incremental $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack -r--r--r-- 59 5b7dec902026f0cddb0ef8acb62f27b5698494d4.datapack - -r--r--r-- 213 f3c56163b762b2931865bbbb1250b4fae09e782c.datapack + -r--r--r-- 225 8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack -r--r--r-- 336 094b530486dad4427a0faf6bcbc031571b99ca24.histpack @@ -352,7 +352,7 @@ $ hg debugwaitonrepack >/dev/null 2>&1 $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack -r--r--r-- 59 5b7dec902026f0cddb0ef8acb62f27b5698494d4.datapack - -r--r--r-- 213 f3c56163b762b2931865bbbb1250b4fae09e782c.datapack + -r--r--r-- 225 8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack -r--r--r-- 336 094b530486dad4427a0faf6bcbc031571b99ca24.histpack diff --git a/tests/test-remotefilelog-repack.t b/tests/test-remotefilelog-repack.t --- a/tests/test-remotefilelog-repack.t +++ b/tests/test-remotefilelog-repack.t @@ -301,7 +301,7 @@ $ hg repack --incremental $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack -r--r--r-- 59 5b7dec902026f0cddb0ef8acb62f27b5698494d4.datapack - -r--r--r-- 213 f3c56163b762b2931865bbbb1250b4fae09e782c.datapack + -r--r--r-- 225 8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack -r--r--r-- 336 094b530486dad4427a0faf6bcbc031571b99ca24.histpack @@ -309,7 +309,7 @@ $ hg repack --incremental $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack -r--r--r-- 59 5b7dec902026f0cddb0ef8acb62f27b5698494d4.datapack - -r--r--r-- 213 f3c56163b762b2931865bbbb1250b4fae09e782c.datapack + -r--r--r-- 225 8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack -r--r--r-- 336 094b530486dad4427a0faf6bcbc031571b99ca24.histpack @@ -347,7 +347,7 @@ $ hg debugwaitonrepack >/dev/null 2>&1 $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack -r--r--r-- 59 5b7dec902026f0cddb0ef8acb62f27b5698494d4.datapack - -r--r--r-- 213 f3c56163b762b2931865bbbb1250b4fae09e782c.datapack + -r--r--r-- 225 8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack -r--r--r-- 336 094b530486dad4427a0faf6bcbc031571b99ca24.histpack @@ -387,11 +387,11 @@ x Node Delta Base Delta Length Blob Size - 1bb2e6237e03 000000000000 8 (missing) - d4a3ed9310e5 1bb2e6237e03 12 (missing) - aee31534993a 000000000000 4 (missing) - 1406e7411862 aee31534993a 12 (missing) + 1bb2e6237e03 000000000000 8 8 + d4a3ed9310e5 1bb2e6237e03 12 6 + aee31534993a 000000000000 4 4 + 1406e7411862 aee31534993a 12 2 y Node Delta Base Delta Length Blob Size - 577959738234 000000000000 8 (missing) + 577959738234 000000000000 8 8 diff --git a/tests/test-treemanifest-prefetch.t b/tests/test-treemanifest-prefetch.t --- a/tests/test-treemanifest-prefetch.t +++ b/tests/test-treemanifest-prefetch.t @@ -68,28 +68,28 @@ $ hg debugdatapack --long $CACHEDIR/master/packs/manifests/*.dataidx subdir - Node Delta Base Delta Length - ddb35f099a648a43a997aef53123bce309c794fd 0000000000000000000000000000000000000000 43 + Node Delta Base Delta Length Blob Size + ddb35f099a648a43a997aef53123bce309c794fd 0000000000000000000000000000000000000000 43 (missing) - Node Delta Base Delta Length - 1be4ab2126dd2252dcae6be2aac2561dd3ddcda0 0000000000000000000000000000000000000000 95 + Node Delta Base Delta Length Blob Size + 1be4ab2126dd2252dcae6be2aac2561dd3ddcda0 0000000000000000000000000000000000000000 95 (missing) dir - Node Delta Base Delta Length - a18d21674e76d6aab2edb46810b20fbdbd10fb4b 0000000000000000000000000000000000000000 43 + Node Delta Base Delta Length Blob Size + a18d21674e76d6aab2edb46810b20fbdbd10fb4b 0000000000000000000000000000000000000000 43 (missing) - Node Delta Base Delta Length - 60a7f7acb6bb5aaf93ca7d9062931b0f6a0d6db5 0000000000000000000000000000000000000000 95 + Node Delta Base Delta Length Blob Size + 60a7f7acb6bb5aaf93ca7d9062931b0f6a0d6db5 0000000000000000000000000000000000000000 95 (missing) dir - Node Delta Base Delta Length - bc0c2c938b929f98b1c31a8c5994396ebb096bf0 0000000000000000000000000000000000000000 43 + Node Delta Base Delta Length Blob Size + bc0c2c938b929f98b1c31a8c5994396ebb096bf0 0000000000000000000000000000000000000000 43 (missing) - Node Delta Base Delta Length - ef362f8bbe8aa457b0cfc49f200cbeb7747984ed 0000000000000000000000000000000000000000 46 + Node Delta Base Delta Length Blob Size + ef362f8bbe8aa457b0cfc49f200cbeb7747984ed 0000000000000000000000000000000000000000 46 (missing) $ hg debughistorypack $CACHEDIR/master/packs/manifests/*.histidx @@ -132,12 +132,12 @@ $ hg debugdatapack $CACHEDIR/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx dir - Node Delta Base Delta Length - a18d21674e76 000000000000 43 + Node Delta Base Delta Length Blob Size + a18d21674e76 000000000000 43 (missing) - Node Delta Base Delta Length - 60a7f7acb6bb 000000000000 95 + Node Delta Base Delta Length Blob Size + 60a7f7acb6bb 000000000000 95 (missing) Test auto prefetch during normal access $ rm -rf $CACHEDIR/master @@ -170,27 +170,27 @@ $ hg debugdatapack $CACHEDIR/master/packs/manifests/148e9eb32f473ea522c591c95be0f9e772be9675 dir - Node Delta Base Delta Length - bc0c2c938b92 000000000000 43 + Node Delta Base Delta Length Blob Size + bc0c2c938b92 000000000000 43 (missing) subdir - Node Delta Base Delta Length - ddb35f099a64 000000000000 43 + Node Delta Base Delta Length Blob Size + ddb35f099a64 000000000000 43 (missing) - Node Delta Base Delta Length - 1be4ab2126dd 000000000000 95 + Node Delta Base Delta Length Blob Size + 1be4ab2126dd 000000000000 95 (missing) - Note that subdir/ is not downloaded again $ hg debugdatapack $CACHEDIR/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c dir - Node Delta Base Delta Length - a18d21674e76 000000000000 43 + Node Delta Base Delta Length Blob Size + a18d21674e76 000000000000 43 (missing) - Node Delta Base Delta Length - 60a7f7acb6bb 000000000000 95 + Node Delta Base Delta Length Blob Size + 60a7f7acb6bb 000000000000 95 (missing) Test that auto prefetch scans up the changelog for base trees $ rm -rf $CACHEDIR/master @@ -218,28 +218,28 @@ $ hg debugdatapack $CACHEDIR/master/packs/manifests/*.dataidx subdir - Node Delta Base Delta Length - ddb35f099a64 000000000000 43 + Node Delta Base Delta Length Blob Size + ddb35f099a64 000000000000 43 (missing) - Node Delta Base Delta Length - 1be4ab2126dd 000000000000 95 + Node Delta Base Delta Length Blob Size + 1be4ab2126dd 000000000000 95 (missing) dir - Node Delta Base Delta Length - a18d21674e76 000000000000 43 + Node Delta Base Delta Length Blob Size + a18d21674e76 000000000000 43 (missing) - Node Delta Base Delta Length - 60a7f7acb6bb 000000000000 95 + Node Delta Base Delta Length Blob Size + 60a7f7acb6bb 000000000000 95 (missing) dir - Node Delta Base Delta Length - bc0c2c938b92 000000000000 43 + Node Delta Base Delta Length Blob Size + bc0c2c938b92 000000000000 43 (missing) - Node Delta Base Delta Length - ef362f8bbe8a 000000000000 46 + Node Delta Base Delta Length Blob Size + ef362f8bbe8a 000000000000 46 (missing) $ hg strip -q -r 'draft()' @@ -254,16 +254,16 @@ $ hg debugdatapack $CACHEDIR/master/packs/manifests/*.dataidx dir - Node Delta Base Delta Length - a18d21674e76 000000000000 43 + Node Delta Base Delta Length Blob Size + a18d21674e76 000000000000 43 (missing) subdir - Node Delta Base Delta Length - ddb35f099a64 000000000000 43 + Node Delta Base Delta Length Blob Size + ddb35f099a64 000000000000 43 (missing) - Node Delta Base Delta Length - 60a7f7acb6bb 000000000000 95 + Node Delta Base Delta Length Blob Size + 60a7f7acb6bb 000000000000 95 (missing) - Prefetch commit 1 then minimally prefetch commit 2 $ rm -rf $CACHEDIR/master @@ -283,12 +283,12 @@ $ hg debugdatapack $CACHEDIR/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx dir - Node Delta Base Delta Length - a18d21674e76 000000000000 43 + Node Delta Base Delta Length Blob Size + a18d21674e76 000000000000 43 (missing) - Node Delta Base Delta Length - 60a7f7acb6bb 000000000000 95 + Node Delta Base Delta Length Blob Size + 60a7f7acb6bb 000000000000 95 (missing) Test prefetching certain revs during pull $ cd ../master @@ -313,16 +313,16 @@ $ hg debugdatapack $CACHEDIR/master/packs/manifests/*.dataidx dir - Node Delta Base Delta Length - a18d21674e76 000000000000 43 + Node Delta Base Delta Length Blob Size + a18d21674e76 000000000000 43 (missing) subdir - Node Delta Base Delta Length - ddb35f099a64 000000000000 43 + Node Delta Base Delta Length Blob Size + ddb35f099a64 000000000000 43 (missing) - Node Delta Base Delta Length - 60a7f7acb6bb 000000000000 95 + Node Delta Base Delta Length Blob Size + 60a7f7acb6bb 000000000000 95 (missing) - Test prefetching only the new tree parts for a commit who's parent tree is not - downloaded already. Note that subdir/z was not downloaded this time. @@ -335,12 +335,12 @@ $ hg debugdatapack $CACHEDIR/master/packs/manifests/99050e724a9236121684523ba3f4db270e62fb58.dataidx dir - Node Delta Base Delta Length - bf22bc15398b 000000000000 43 + Node Delta Base Delta Length Blob Size + bf22bc15398b 000000000000 43 (missing) - Node Delta Base Delta Length - aa52a49be522 000000000000 95 + Node Delta Base Delta Length Blob Size + aa52a49be522 000000000000 95 (missing) Test that prefetch refills just part of a tree when the cache is deleted diff --git a/tests/test-treemanifest-repack.t b/tests/test-treemanifest-repack.t --- a/tests/test-treemanifest-repack.t +++ b/tests/test-treemanifest-repack.t @@ -81,19 +81,19 @@ $ hg repack $ ls_l $CACHEDIR/master/packs/manifests | grep pack + -r--r--r-- 339 56e8c6f0ca2a324b8b5ca1a2730323a1b4d0793a.datapack -r--r--r-- 262 7535b6084226436bbdff33043969e7fa963e8428.histpack - -r--r--r-- 327 be1a1ae9828d9bd09ab8ba765db358f99fd09ea7.datapack $ hg debugdatapack $CACHEDIR/master/packs/manifests/*.datapack Node Delta Base Delta Length Blob Size - 1832e0765de9 000000000000 89 (missing) - a0c8bcbbb45c 1832e0765de9 12 (missing) + 1832e0765de9 000000000000 89 89 + a0c8bcbbb45c 1832e0765de9 12 43 dir Node Delta Base Delta Length Blob Size - 23226e7a252c 000000000000 43 (missing) + 23226e7a252c 000000000000 43 43 $ hg debughistorypack $CACHEDIR/master/packs/manifests/*.histpack @@ -115,32 +115,32 @@ -r--r--r-- 146 cffef142da32f3e52c1779490e5d0ddac5f9b82b.datapack $ hg repack $ ls_l .hg/store/packs/manifests | grep datapack - -r--r--r-- 374 201094db51b761cd78352c055b3135178aadfec5.datapack + -r--r--r-- 386 d15c09a9a5a13bb689bd9764455a415a20dc885e.datapack # Test incremental repacking of trees $ echo b >> dir/b && hg commit -Aqm 'modify dir/b' $ echo b >> dir/b && hg commit -Aqm 'modify dir/b' $ ls_l .hg/store/packs/manifests | grep datapack - -r--r--r-- 374 201094db51b761cd78352c055b3135178aadfec5.datapack -r--r--r-- 248 21501384df03b8489b366c5218be639fa08830e4.datapack + -r--r--r-- 386 d15c09a9a5a13bb689bd9764455a415a20dc885e.datapack -r--r--r-- 248 d7e689a91ac63385be120a118af9ce8663748f28.datapack - repack incremental does nothing here because there are so few packs $ hg repack --incremental --config remotefilelog.data.generations=300,200 --config remotefilelog.data.repacksizelimit=300 $ ls_l .hg/store/packs/manifests | grep datapack - -r--r--r-- 374 201094db51b761cd78352c055b3135178aadfec5.datapack -r--r--r-- 248 21501384df03b8489b366c5218be639fa08830e4.datapack + -r--r--r-- 386 d15c09a9a5a13bb689bd9764455a415a20dc885e.datapack -r--r--r-- 248 d7e689a91ac63385be120a118af9ce8663748f28.datapack $ echo b >> dir/b && hg commit -Aqm 'modify dir/b' $ echo b >> dir/b && hg commit -Aqm 'modify dir/b' $ echo b >> dir/b && hg commit -Aqm 'modify dir/b' $ ls_l .hg/store/packs/manifests | grep datapack - -r--r--r-- 374 201094db51b761cd78352c055b3135178aadfec5.datapack -r--r--r-- 248 21501384df03b8489b366c5218be639fa08830e4.datapack -r--r--r-- 248 347263bf1efbdb5bf7e1d1565b6b504073fb9093.datapack -r--r--r-- 248 544a3b46a61732209116ae50847ec333b75e3765.datapack -r--r--r-- 248 863908ef8149261ab0d891c2344d8e8766c39441.datapack + -r--r--r-- 386 d15c09a9a5a13bb689bd9764455a415a20dc885e.datapack -r--r--r-- 248 d7e689a91ac63385be120a118af9ce8663748f28.datapack $ cd .hg/store/packs/manifests $ cp d7e689a91ac63385be120a118af9ce8663748f28.datapack x7e689a91ac63385be120a118af9ce8663748f28.datapack