This is an archive of the discontinued Mercurial Phabricator instance.

Differential D6844

remotefilelog: replace repack lock to solve race condition
ClosedPublic

Authored by lothiraldan on Sep 12 2019, 9:40 AM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG5fadf6103790: remotefilelog: replace repack lock to solve race condition
Summary

2c74337e6483 reduced the probability of race-conditions when starting
background repack and prefetch and we saw the difference in our CI instance
with all failures disappearing except one where one call to waitonrepack seems
to returns too early.

I'm not sure what exactly goes wrong but I realized that while the prefetch
operation uses a standard Mercurial lock, the repack operation is using a
custom lock based on fcntl.flock on available platforms. As extutil.flock
fallback on traditional Mercurial locks on other platforms and the tests are
stable on my laptop, our CI environment and GCC112, I'm sending this patch to
standardize the behavior across environments.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

lothiraldan created this revision.Sep 12 2019, 9:40 AM
lothiraldan added a commit: rHG5fadf6103790: remotefilelog: replace repack lock to solve race condition.Sep 17 2019, 2:19 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG5fadf6103790: remotefilelog: replace repack lock to solve race condition (authored by lothiraldan). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 16565

hgext/remotefilelog/debugcommands.py

import os import os
import zlib import zlib
from mercurial.node import bin, hex, nullid, short from mercurial.node import bin, hex, nullid, short
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial import ( from mercurial import (
error, error,
filelog, filelog,
lock as lockmod,
node as nodemod, node as nodemod,
pycompat, pycompat,
revlog, revlog,
) )
from . import ( from . import (
constants, constants,
datapack, datapack,
extutil,
fileserverclient, fileserverclient,
historypack, historypack,
repack, repack,
shallowutil, shallowutil,
) )
def debugremotefilelog(ui, path, **opts): def debugremotefilelog(ui, path, **opts):
decompress = opts.get(r'decompress') decompress = opts.get(r'decompress')
"P2 Node".ljust(14), "P2 Node".ljust(14),
"Link Node".ljust(14), "Link Node".ljust(14),
"Copy From")) "Copy From"))
lastfilename = filename lastfilename = filename
ui.write("%s %s %s %s %s\n" % (short(node), short(p1node), ui.write("%s %s %s %s %s\n" % (short(node), short(p1node),
short(p2node), short(linknode), copyfrom)) short(p2node), short(linknode), copyfrom))
def debugwaitonrepack(repo): def debugwaitonrepack(repo):
with extutil.flock(repack.repacklockvfs(repo).join('repacklock'), ''): with lockmod.lock(repack.repacklockvfs(repo), "repacklock", timeout=-1):
return return
def debugwaitonprefetch(repo): def debugwaitonprefetch(repo):
with repo._lock(repo.svfs, "prefetchlock", True, None, with repo._lock(repo.svfs, "prefetchlock", True, None,
None, _('prefetching in %s') % repo.origroot): None, _('prefetching in %s') % repo.origroot):
pass pass

hgext/remotefilelog/repack.py

from __future__ import absolute_import from __future__ import absolute_import
import os import os
import time import time
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial.node import ( from mercurial.node import (
nullid, nullid,
short, short,
) )
from mercurial import ( from mercurial import (
encoding, encoding,
error, error,
lock as lockmod,
mdiff, mdiff,
policy, policy,
pycompat, pycompat,
scmutil, scmutil,
util, util,
vfs, vfs,
) )
from mercurial.utils import procutil from mercurial.utils import procutil
from . import ( from . import (
constants, constants,
contentstore, contentstore,
datapack, datapack,
extutil,
historypack, historypack,
metadatastore, metadatastore,
shallowutil, shallowutil,
) )
osutil = policy.importmod(r'osutil') osutil = policy.importmod(r'osutil')
class RepackAlreadyRunning(error.Abort): class RepackAlreadyRunning(error.Abort):
raise ValueError("Function 'isold' is not properly specified") raise ValueError("Function 'isold' is not properly specified")
# use (filename, node) tuple as a keepset key # use (filename, node) tuple as a keepset key
self.keepkeys = keepset(repo, lambda f, n : (f, n)) self.keepkeys = keepset(repo, lambda f, n : (f, n))
self.isold = isold self.isold = isold
def run(self, targetdata, targethistory): def run(self, targetdata, targethistory):
ledger = repackledger() ledger = repackledger()
with extutil.flock(repacklockvfs(self.repo).join("repacklock"), with lockmod.lock(repacklockvfs(self.repo), "repacklock", desc=None,
_('repacking %s') % self.repo.origroot, timeout=0): timeout=0):
self.repo.hook('prerepack') self.repo.hook('prerepack')
# Populate ledger from source # Populate ledger from source
self.data.markledger(ledger, options=self.options) self.data.markledger(ledger, options=self.options)
self.history.markledger(ledger, options=self.options) self.history.markledger(ledger, options=self.options)
# Run repack # Run repack
self.repackdata(ledger, targetdata) self.repackdata(ledger, targetdata)

tests/test-remotefilelog-bgprefetch.t

$ sleep 0.5 $ sleep 0.5
$ hg debugwaitonrepack >/dev/null 2>%1 $ hg debugwaitonrepack >/dev/null 2>%1
$ sleep 0.5 $ sleep 0.5
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/6e8633deba6e544e5f8edbd7b996d6e31a2c42ae.histidx $TESTTMP/hgcache/master/packs/6e8633deba6e544e5f8edbd7b996d6e31a2c42ae.histidx
$TESTTMP/hgcache/master/packs/6e8633deba6e544e5f8edbd7b996d6e31a2c42ae.histpack $TESTTMP/hgcache/master/packs/6e8633deba6e544e5f8edbd7b996d6e31a2c42ae.histpack
$TESTTMP/hgcache/master/packs/8ce5ab3745465ab83bba30a7b9c295e0c8404652.dataidx $TESTTMP/hgcache/master/packs/8ce5ab3745465ab83bba30a7b9c295e0c8404652.dataidx
$TESTTMP/hgcache/master/packs/8ce5ab3745465ab83bba30a7b9c295e0c8404652.datapack $TESTTMP/hgcache/master/packs/8ce5ab3745465ab83bba30a7b9c295e0c8404652.datapack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# background prefetch with repack on update when wcprevset configured # background prefetch with repack on update when wcprevset configured
$ clearcache $ clearcache
$ hg up -r 0 $ hg up -r 0
2 files updated, 0 files merged, 0 files removed, 0 files unresolved 2 files updated, 0 files merged, 0 files removed, 0 files unresolved
2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob) 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
$ sleep 1 $ sleep 1
$ hg debugwaitonrepack >/dev/null 2>%1 $ hg debugwaitonrepack >/dev/null 2>%1
$ sleep 1 $ sleep 1
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
$TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
$TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.dataidx $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.dataidx
$TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.datapack $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.datapack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Ensure that file 'w' was prefetched - it was not part of the update operation and therefore # Ensure that file 'w' was prefetched - it was not part of the update operation and therefore
# could only be downloaded by the background prefetch # could only be downloaded by the background prefetch
$ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1` $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
$TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407: $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407:
w: w:
$ sleep 1 $ sleep 1
$ hg debugwaitonrepack >/dev/null 2>%1 $ hg debugwaitonrepack >/dev/null 2>%1
$ sleep 1 $ sleep 1
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
$TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
$TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.dataidx $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.dataidx
$TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.datapack $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.datapack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Ensure that file 'w' was prefetched - it was not part of the commit operation and therefore # Ensure that file 'w' was prefetched - it was not part of the commit operation and therefore
# could only be downloaded by the background prefetch # could only be downloaded by the background prefetch
$ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1` $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
$TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407: $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407:
w: w:
$ hg up -r 2 $ hg up -r 2
3 files updated, 0 files merged, 3 files removed, 0 files unresolved 3 files updated, 0 files merged, 3 files removed, 0 files unresolved
(leaving bookmark temporary) (leaving bookmark temporary)
$ clearcache $ clearcache
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$ hg rebase -s temporary -d foo $ hg rebase -s temporary -d foo
rebasing 3:58147a5b5242 "b" (temporary tip) rebasing 3:58147a5b5242 "b" (temporary tip)
saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/58147a5b5242-c3678817-rebase.hg (glob) saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/58147a5b5242-c3678817-rebase.hg
3 files fetched over 1 fetches - (3 misses, 0.00% hit ratio) over *s (glob) 3 files fetched over 1 fetches - (3 misses, 0.00% hit ratio) over *s (glob)
$ sleep 1 $ sleep 1
$ hg debugwaitonprefetch >/dev/null 2>%1 $ hg debugwaitonprefetch >/dev/null 2>%1
$ sleep 1 $ sleep 1
$ hg debugwaitonrepack >/dev/null 2>%1 $ hg debugwaitonrepack >/dev/null 2>%1
$ sleep 1 $ sleep 1
# Ensure that file 'y' was prefetched - it was not part of the rebase operation and therefore # Ensure that file 'y' was prefetched - it was not part of the rebase operation and therefore
$ hg debugwaitonrepack >/dev/null 2>%1 $ hg debugwaitonrepack >/dev/null 2>%1
$ sleep 0.5 $ sleep 0.5
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
$TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
$TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.dataidx $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.dataidx
$TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.datapack $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.datapack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Ensure that files were prefetched # Ensure that files were prefetched
$ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1` $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
$TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407: $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407:
w: w:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
bb6ccd5dceaa 000000000000 2 2 bb6ccd5dceaa 000000000000 2 2
$ hg debugwaitonrepack >/dev/null 2>%1 $ hg debugwaitonrepack >/dev/null 2>%1
$ sleep 0.5 $ sleep 0.5
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
$TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
$TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.dataidx $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.dataidx
$TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.datapack $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.datapack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Ensure that files were prefetched # Ensure that files were prefetched
$ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1` $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
$TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407: $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407:
w: w:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
bb6ccd5dceaa 000000000000 2 2 bb6ccd5dceaa 000000000000 2 2

tests/test-remotefilelog-gc.t

$ hg gc --config remotefilelog.gcrepack=True --config remotefilelog.repackonhggc=True $ hg gc --config remotefilelog.gcrepack=True --config remotefilelog.repackonhggc=True
# Ensure that loose files are repacked # Ensure that loose files are repacked
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/320dab99b7e3f60512b97f347689625263d22cf5.dataidx $TESTTMP/hgcache/master/packs/320dab99b7e3f60512b97f347689625263d22cf5.dataidx
$TESTTMP/hgcache/master/packs/320dab99b7e3f60512b97f347689625263d22cf5.datapack $TESTTMP/hgcache/master/packs/320dab99b7e3f60512b97f347689625263d22cf5.datapack
$TESTTMP/hgcache/master/packs/837b83c1ef6485a336eb4421ac5973c0ec130fbb.histidx $TESTTMP/hgcache/master/packs/837b83c1ef6485a336eb4421ac5973c0ec130fbb.histidx
$TESTTMP/hgcache/master/packs/837b83c1ef6485a336eb4421ac5973c0ec130fbb.histpack $TESTTMP/hgcache/master/packs/837b83c1ef6485a336eb4421ac5973c0ec130fbb.histpack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Test that warning is displayed when there are no valid repos in repofile # Test that warning is displayed when there are no valid repos in repofile
$ cp $CACHEDIR/repos $CACHEDIR/repos.bak $ cp $CACHEDIR/repos $CACHEDIR/repos.bak
$ echo " " > $CACHEDIR/repos $ echo " " > $CACHEDIR/repos
$ hg gc $ hg gc
warning: no valid repos in repofile warning: no valid repos in repofile
$ mv $CACHEDIR/repos.bak $CACHEDIR/repos $ mv $CACHEDIR/repos.bak $CACHEDIR/repos
# Test that warning is displayed when the repo path is malformed # Test that warning is displayed when the repo path is malformed
$ printf "asdas\0das" >> $CACHEDIR/repos $ printf "asdas\0das" >> $CACHEDIR/repos
$ hg gc $ hg gc
abort: invalid path asdas\x00da: .*(null|NULL).* (re) abort: invalid path asdas\x00da: .*(null|NULL).* (re)
[255] [255]

tests/test-remotefilelog-repack-fast.t

$ find $CACHEDIR | sort $ find $CACHEDIR | sort
$TESTTMP/hgcache $TESTTMP/hgcache
$TESTTMP/hgcache/master $TESTTMP/hgcache/master
$TESTTMP/hgcache/master/packs $TESTTMP/hgcache/master/packs
$TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
$TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
$TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
$TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Test that the packs are readonly # Test that the packs are readonly
$ ls_l $CACHEDIR/master/packs $ ls_l $CACHEDIR/master/packs
-r--r--r-- 1145 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx -r--r--r-- 1145 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
-r--r--r-- 172 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack -r--r--r-- 172 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
-r--r--r-- 1074 b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx -r--r--r-- 1074 b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
-r--r--r-- 72 b1e0cfc7f345e408a7825e3081501959488d59ce.datapack -r--r--r-- 72 b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
-rw-r--r-- 0 repacklock
# Test that the data in the new packs is accessible # Test that the data in the new packs is accessible
$ hg cat -r . x $ hg cat -r . x
x x
x x
# Test that adding new data and repacking it results in the loose data and the # Test that adding new data and repacking it results in the loose data and the
# old packs being combined. # old packs being combined.
$ cd ../master $ cd ../master
$ echo x >> x $ echo x >> x
$ hg commit -m x3 $ hg commit -m x3
$ cd ../shallow $ cd ../shallow
$ hg pull -q $ hg pull -q
$ hg up -q tip $ hg up -q tip
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/d4a3ed9310e5bd9887e3bf779da5077efab28216 $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/d4a3ed9310e5bd9887e3bf779da5077efab28216
$TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
$TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
$TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
$TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
$ hg repack --traceback $ hg repack --traceback
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.dataidx $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.dataidx
$TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack
$TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
$TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Verify all the file data is still available # Verify all the file data is still available
$ hg cat -r . x $ hg cat -r . x
x x
x x
x x
$ hg cat -r '.^' x $ hg cat -r '.^' x
x x
x x
# Test that repacking again without new data does not delete the pack files # Test that repacking again without new data does not delete the pack files
# and did not change the pack names # and did not change the pack names
$ hg repack $ hg repack
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.dataidx $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.dataidx
$TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack
$TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
$TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Run two repacks at once # Run two repacks at once
$ hg repack --config "hooks.prerepack=sleep 3" & $ hg repack --config "hooks.prerepack=sleep 3" &
$ sleep 1 $ sleep 1
$ hg repack $ hg repack
skipping repack - another repack is already running skipping repack - another repack is already running
$ hg debugwaitonrepack >/dev/null 2>&1 $ hg debugwaitonrepack >/dev/null 2>&1
# Run repack in the background # Run repack in the background
$ cd ../master $ cd ../master
$ echo x >> x $ echo x >> x
$ hg commit -m x4 $ hg commit -m x4
$ cd ../shallow $ cd ../shallow
$ hg pull -q $ hg pull -q
$ hg up -q tip $ hg up -q tip
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1bb2e6237e035c8f8ef508e281f1ce075bc6db72 $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1bb2e6237e035c8f8ef508e281f1ce075bc6db72
$TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.dataidx $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.dataidx
$TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack
$TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
$TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
$ hg repack --background $ hg repack --background
(running background repack) (running background repack)
$ sleep 0.5 $ sleep 0.5
$ hg debugwaitonrepack >/dev/null 2>&1 $ hg debugwaitonrepack >/dev/null 2>&1
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.dataidx $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.dataidx
$TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.datapack $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.datapack
$TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histidx $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histidx
$TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histpack $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histpack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Test debug commands # Test debug commands
$ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack
$TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0: $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0:
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
$ hg up -q tip $ hg up -q tip
2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over * (glob) 2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over * (glob)
$ hg repack $ hg repack
$ ls $TESTTMP/hgcache/master/packs $ ls $TESTTMP/hgcache/master/packs
bfd60adb76018bb952e27cd23fc151bf94865d7d.histidx bfd60adb76018bb952e27cd23fc151bf94865d7d.histidx
bfd60adb76018bb952e27cd23fc151bf94865d7d.histpack bfd60adb76018bb952e27cd23fc151bf94865d7d.histpack
fb3aa57b22789ebcc45706c352e2d6af099c5816.dataidx fb3aa57b22789ebcc45706c352e2d6af099c5816.dataidx
fb3aa57b22789ebcc45706c352e2d6af099c5816.datapack fb3aa57b22789ebcc45706c352e2d6af099c5816.datapack
repacklock
$ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx $ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx
x x
Node P1 Node P2 Node Link Node Copy From Node P1 Node P2 Node Link Node Copy From
cd410a44d584 577959738234 000000000000 609547eda446 y cd410a44d584 577959738234 000000000000 609547eda446 y
1bb2e6237e03 d4a3ed9310e5 000000000000 0b03bbc9e1e7 1bb2e6237e03 d4a3ed9310e5 000000000000 0b03bbc9e1e7
d4a3ed9310e5 aee31534993a 000000000000 421535db10b6 d4a3ed9310e5 aee31534993a 000000000000 421535db10b6
aee31534993a 1406e7411862 000000000000 a89d614e2364 aee31534993a 1406e7411862 000000000000 a89d614e2364

tests/test-remotefilelog-repack.t

$ find $CACHEDIR | sort $ find $CACHEDIR | sort
$TESTTMP/hgcache $TESTTMP/hgcache
$TESTTMP/hgcache/master $TESTTMP/hgcache/master
$TESTTMP/hgcache/master/packs $TESTTMP/hgcache/master/packs
$TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
$TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
$TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
$TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Test that the packs are readonly # Test that the packs are readonly
$ ls_l $CACHEDIR/master/packs $ ls_l $CACHEDIR/master/packs
-r--r--r-- 1145 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx -r--r--r-- 1145 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
-r--r--r-- 172 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack -r--r--r-- 172 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
-r--r--r-- 1074 b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx -r--r--r-- 1074 b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
-r--r--r-- 72 b1e0cfc7f345e408a7825e3081501959488d59ce.datapack -r--r--r-- 72 b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
-rw-r--r-- 0 repacklock
# Test that the data in the new packs is accessible # Test that the data in the new packs is accessible
$ hg cat -r . x $ hg cat -r . x
x x
x x
# Test that adding new data and repacking it results in the loose data and the # Test that adding new data and repacking it results in the loose data and the
# old packs being combined. # old packs being combined.
$ cd ../master $ cd ../master
$ echo x >> x $ echo x >> x
$ hg commit -m x3 $ hg commit -m x3
$ cd ../shallow $ cd ../shallow
$ hg pull -q $ hg pull -q
$ hg up -q tip $ hg up -q tip
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/d4a3ed9310e5bd9887e3bf779da5077efab28216 $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/d4a3ed9310e5bd9887e3bf779da5077efab28216
$TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
$TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
$TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
$TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# First assert that with --packsonly, the loose object will be ignored: # First assert that with --packsonly, the loose object will be ignored:
$ hg repack --packsonly $ hg repack --packsonly
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/d4a3ed9310e5bd9887e3bf779da5077efab28216 $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/d4a3ed9310e5bd9887e3bf779da5077efab28216
$TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
$TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
$TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
$TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
$ hg repack --traceback $ hg repack --traceback
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.dataidx $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.dataidx
$TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack
$TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
$TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Verify all the file data is still available # Verify all the file data is still available
$ hg cat -r . x $ hg cat -r . x
x x
x x
x x
$ hg cat -r '.^' x $ hg cat -r '.^' x
x x
x x
# Test that repacking again without new data does not delete the pack files # Test that repacking again without new data does not delete the pack files
# and did not change the pack names # and did not change the pack names
$ hg repack $ hg repack
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.dataidx $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.dataidx
$TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack
$TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
$TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Run two repacks at once # Run two repacks at once
$ hg repack --config "hooks.prerepack=sleep 3" & $ hg repack --config "hooks.prerepack=sleep 3" &
$ sleep 1 $ sleep 1
$ hg repack $ hg repack
skipping repack - another repack is already running skipping repack - another repack is already running
$ hg debugwaitonrepack >/dev/null 2>&1 $ hg debugwaitonrepack >/dev/null 2>&1
# Run repack in the background # Run repack in the background
$ cd ../master $ cd ../master
$ echo x >> x $ echo x >> x
$ hg commit -m x4 $ hg commit -m x4
$ cd ../shallow $ cd ../shallow
$ hg pull -q $ hg pull -q
$ hg up -q tip $ hg up -q tip
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1bb2e6237e035c8f8ef508e281f1ce075bc6db72 $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1bb2e6237e035c8f8ef508e281f1ce075bc6db72
$TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.dataidx $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.dataidx
$TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack
$TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
$TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
$ hg repack --background $ hg repack --background
(running background repack) (running background repack)
$ sleep 0.5 $ sleep 0.5
$ hg debugwaitonrepack >/dev/null 2>&1 $ hg debugwaitonrepack >/dev/null 2>&1
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.dataidx $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.dataidx
$TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.datapack $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.datapack
$TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histidx $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histidx
$TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histpack $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histpack
$TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Test debug commands # Test debug commands
$ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack
$TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0: $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0:
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
$ hg up -q tip $ hg up -q tip
2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over * (glob) 2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over * (glob)
$ hg repack $ hg repack
$ ls $TESTTMP/hgcache/master/packs $ ls $TESTTMP/hgcache/master/packs
bfd60adb76018bb952e27cd23fc151bf94865d7d.histidx bfd60adb76018bb952e27cd23fc151bf94865d7d.histidx
bfd60adb76018bb952e27cd23fc151bf94865d7d.histpack bfd60adb76018bb952e27cd23fc151bf94865d7d.histpack
fb3aa57b22789ebcc45706c352e2d6af099c5816.dataidx fb3aa57b22789ebcc45706c352e2d6af099c5816.dataidx
fb3aa57b22789ebcc45706c352e2d6af099c5816.datapack fb3aa57b22789ebcc45706c352e2d6af099c5816.datapack
repacklock
$ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx $ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx
x x
Node P1 Node P2 Node Link Node Copy From Node P1 Node P2 Node Link Node Copy From
cd410a44d584 577959738234 000000000000 609547eda446 y cd410a44d584 577959738234 000000000000 609547eda446 y
1bb2e6237e03 d4a3ed9310e5 000000000000 0b03bbc9e1e7 1bb2e6237e03 d4a3ed9310e5 000000000000 0b03bbc9e1e7
d4a3ed9310e5 aee31534993a 000000000000 421535db10b6 d4a3ed9310e5 aee31534993a 000000000000 421535db10b6
aee31534993a 1406e7411862 000000000000 a89d614e2364 aee31534993a 1406e7411862 000000000000 a89d614e2364