diff --git a/remotefilelog/repack.py b/remotefilelog/repack.py --- a/remotefilelog/repack.py +++ b/remotefilelog/repack.py @@ -142,25 +142,38 @@ files = osutil.listdir(packpath, stat=True) - datapacks = _computeincrementaldatapack(repo.ui, files) - fullpaths = list(os.path.join(packpath, p) for p in datapacks) - datapacks = list(datapack.datapack(p) for p in fullpaths) + def topacks(files, constructor): + paths = list(os.path.join(packpath, p) for p in files) + packs = list(constructor(p) for p in paths) + return packs + + datapacks = topacks(_computeincrementaldatapack(repo.ui, files), + datapack.datapack) datapacks.extend(s for s in datastore if not isinstance(s, datapack.datapackstore)) - historypacks = _computeincrementalhistorypack(repo.ui, files) - fullpaths = list(os.path.join(packpath, p) for p in historypacks) - historypacks = list(historypack.historypack(p) for p in fullpaths) + historypacks = topacks(_computeincrementalhistorypack(repo.ui, files), + historypack.historypack) historypacks.extend(s for s in historystore if not isinstance(s, historypack.historypackstore)) - datasource = contentstore.unioncontentstore( - *datapacks, - allowincomplete=allowincompletedata) - historysource = metadatastore.unionmetadatastore(*historypacks, - allowincomplete=True) - - _runrepack(repo, datasource, historysource, packpath, category) + allhistoryfiles = _allpackfileswithsuffix(files, historypack.PACKSUFFIX, + historypack.INDEXSUFFIX) + allhistorypacks = topacks((f for f, mode, stat in allhistoryfiles), + historypack.historypack) + allhistorypacks.extend(s for s in historystore + if not isinstance(s, historypack.historypackstore)) + _runrepack(repo, + contentstore.unioncontentstore( + *datapacks, + allowincomplete=allowincompletedata), + metadatastore.unionmetadatastore( + *historypacks, + allowincomplete=True), + packpath, category, + fullhistory=metadatastore.unionmetadatastore( + *allhistorypacks, + allowincomplete=True)) def _computeincrementaldatapack(ui, files): """Given a set of pack files and a set of generation size limits, this @@ -202,13 +215,8 @@ historypack.PACKSUFFIX, historypack.INDEXSUFFIX, gencountlimit, repacksizelimit, maxrepackpacks) -def _computeincrementalpack(ui, files, limits, packsuffix, indexsuffix, - gencountlimit, repacksizelimit, maxrepackpacks): - # Group the packs by generation (i.e. by size) - generations = [] - for i in xrange(len(limits)): - generations.append([]) - sizes = {} +def _allpackfileswithsuffix(files, packsuffix, indexsuffix): + result = [] fileset = set(fn for fn, mode, stat in files) for filename, mode, stat in files: if not filename.endswith(packsuffix): @@ -219,7 +227,20 @@ # Don't process a pack if it doesn't have an index. if (prefix + indexsuffix) not in fileset: continue + result.append((filename[:-len(packsuffix)], mode, stat)) + return result + +def _computeincrementalpack(ui, files, limits, packsuffix, indexsuffix, + gencountlimit, repacksizelimit, maxrepackpacks): + # Group the packs by generation (i.e. by size) + generations = [] + for i in xrange(len(limits)): + generations.append([]) + + sizes = {} + for prefix, mode, stat in _allpackfileswithsuffix(files, packsuffix, + indexsuffix): size = stat.st_size sizes[prefix] = size for i, limit in enumerate(limits): @@ -257,7 +278,7 @@ return chosenpacks -def _runrepack(repo, data, history, packpath, category): +def _runrepack(repo, data, history, packpath, category, fullhistory=None): shallowutil.mkstickygroupdir(repo.ui, packpath) def isold(repo, filename, node): @@ -275,7 +296,10 @@ return filetime[0] < limit garbagecollect = repo.ui.configbool('remotefilelog', 'gcrepack') - packer = repacker(repo, data, history, category, gc=garbagecollect, + if not fullhistory: + fullhistory = history + packer = repacker(repo, data, history, fullhistory, category, + gc=garbagecollect, isold=isold) # internal config: remotefilelog.datapackversion @@ -354,10 +378,12 @@ """Class for orchestrating the repack of data and history information into a new format. """ - def __init__(self, repo, data, history, category, gc=False, isold=None): + def __init__(self, repo, data, history, fullhistory, category, gc=False, + isold=None): self.repo = repo self.data = data self.history = history + self.fullhistory = fullhistory self.unit = constants.getunits(category) self.garbagecollect = gc if self.garbagecollect: @@ -409,8 +435,8 @@ ui.progress(_("building history"), i, unit='nodes', total=len(nodes)) try: - ancestors.update(self.history.getancestors(filename, node, - known=ancestors)) + ancestors.update(self.fullhistory.getancestors(filename, + node, known=ancestors)) except KeyError: # Since we're packing data entries, we may not have the # corresponding history entries for them. It's not a big @@ -420,6 +446,9 @@ # Order the nodes children first, so we can produce reverse deltas orderednodes = list(reversed(self._toposort(ancestors))) + if len(nohistory) > 0: + ui.debug('repackdata: %d nodes without history\n' % + len(nohistory)) orderednodes.extend(sorted(nohistory)) # Compute deltas and write to the pack diff --git a/tests/test-remotefilelog-repack-fast.t b/tests/test-remotefilelog-repack-fast.t --- a/tests/test-remotefilelog-repack-fast.t +++ b/tests/test-remotefilelog-repack-fast.t @@ -360,14 +360,9 @@ $ hg debugwaitonrepack >/dev/null 2>&1 $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack -r--r--r-- 59 5b7dec902026f0cddb0ef8acb62f27b5698494d4.datapack - -r--r--r-- 65 6c499d21350d79f92fd556b4b7a902569d88e3c9.datapack - -r--r--r-- 61 817d294043bd21a3de01f807721971abe45219ce.datapack - -r--r--r-- 63 ff45add45ab3f59c4f75efc6a087d86c821219d6.datapack + -r--r--r-- 225 8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack - -r--r--r-- 254 077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack -r--r--r-- 336 094b530486dad4427a0faf6bcbc031571b99ca24.histpack - -r--r--r-- 172 276d308429d0303762befa376788300f0310f90e.histpack - -r--r--r-- 90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack Test environment variable resolution $ CACHEPATH=$TESTTMP/envcache hg prefetch --config 'remotefilelog.cachepath=$CACHEPATH'