| durham |
| Restricted Project |
Adding any repacking option during the incremental repack results in
the _computeincrementalrepack signature to change. This should help avoid that
problem.
Ran all the tests.
| Lint Skipped |
| Unit Tests Skipped |
| Path | Packages | |||
|---|---|---|---|---|
| M | remotefilelog/repack.py (85 lines) |
| allowincomplete=True), | allowincomplete=True), | ||||
| packpath, category, | packpath, category, | ||||
| fullhistory=metadatastore.unionmetadatastore( | fullhistory=metadatastore.unionmetadatastore( | ||||
| *allhistorypacks, | *allhistorypacks, | ||||
| allowincomplete=True), | allowincomplete=True), | ||||
| options=options) | options=options) | ||||
| def _computeincrementaldatapack(ui, files): | def _computeincrementaldatapack(ui, files): | ||||
| """Given a set of pack files and a set of generation size limits, this | opts = { | ||||
| function computes the list of files that should be packed as part of an | 'generations' : ui.configlist( | ||||
| incremental repack. | 'remotefilelog', 'data.generations', ['1GB', '100MB', '1MB']), | ||||
| 'gencountlimit' : ui.configint( | |||||
| It tries to strike a balance between keeping incremental repacks cheap (i.e. | 'remotefilelog', 'data.gencountlimit', 2), | ||||
| packing small things when possible, and rolling the packs up to the big ones | 'repacksizelimit' : ui.configbytes( | ||||
| over time). | 'remotefilelog', 'data.repacksizelimit', '100MB'), | ||||
| """ | 'maxrepackpacks' : ui.configint( | ||||
| generations = ui.configlist("remotefilelog", "data.generations", | 'remotefilelog', 'data.maxrepackpacks', 50), | ||||
| ['1GB', '100MB', '1MB']) | } | ||||
| generations = list(sorted((util.sizetoint(s) for s in generations), | |||||
| reverse=True)) | packfiles = _allpackfileswithsuffix( | ||||
| generations.append(0) | files, datapack.PACKSUFFIX, datapack.INDEXSUFFIX) | ||||
| return _computeincrementalpack(packfiles, opts) | |||||
| gencountlimit = ui.configint('remotefilelog', 'data.gencountlimit', 2) | |||||
| repacksizelimit = ui.configbytes('remotefilelog', 'data.repacksizelimit', | |||||
| '100MB') | |||||
| maxrepackpacks = ui.configint('remotefilelog', 'data.maxrepackpacks', 50) | |||||
| return _computeincrementalpack(ui, files, generations, datapack.PACKSUFFIX, | |||||
| datapack.INDEXSUFFIX, gencountlimit, | |||||
| repacksizelimit, maxrepackpacks) | |||||
| def _computeincrementalhistorypack(ui, files): | def _computeincrementalhistorypack(ui, files): | ||||
| generations = ui.configlist("remotefilelog", "history.generations", | opts = { | ||||
| ['100MB']) | 'generations' : ui.configlist( | ||||
| generations = list(sorted((util.sizetoint(s) for s in generations), | 'remotefilelog', 'history.generations', ['100MB']), | ||||
| reverse=True)) | 'gencountlimit' : ui.configint( | ||||
| generations.append(0) | 'remotefilelog', 'history.gencountlimit', 2), | ||||
| 'repacksizelimit' : ui.configbytes( | |||||
| gencountlimit = ui.configint('remotefilelog', 'history.gencountlimit', 2) | 'remotefilelog', 'history.repacksizelimit', '100MB'), | ||||
| repacksizelimit = ui.configbytes('remotefilelog', 'history.repacksizelimit', | 'maxrepackpacks' : ui.configint( | ||||
| '100MB') | 'remotefilelog', 'history.maxrepackpacks', 50), | ||||
| maxrepackpacks = ui.configint('remotefilelog', 'history.maxrepackpacks', 50) | } | ||||
| return _computeincrementalpack(ui, files, generations, | packfiles = _allpackfileswithsuffix( | ||||
| historypack.PACKSUFFIX, historypack.INDEXSUFFIX, gencountlimit, | files, historypack.PACKSUFFIX, historypack.INDEXSUFFIX) | ||||
| repacksizelimit, maxrepackpacks) | return _computeincrementalpack(packfiles, opts) | ||||
| def _allpackfileswithsuffix(files, packsuffix, indexsuffix): | def _allpackfileswithsuffix(files, packsuffix, indexsuffix): | ||||
| result = [] | result = [] | ||||
| fileset = set(fn for fn, mode, stat in files) | fileset = set(fn for fn, mode, stat in files) | ||||
| for filename, mode, stat in files: | for filename, mode, stat in files: | ||||
| if not filename.endswith(packsuffix): | if not filename.endswith(packsuffix): | ||||
| continue | continue | ||||
| prefix = filename[:-len(packsuffix)] | prefix = filename[:-len(packsuffix)] | ||||
| # Don't process a pack if it doesn't have an index. | # Don't process a pack if it doesn't have an index. | ||||
| if (prefix + indexsuffix) not in fileset: | if (prefix + indexsuffix) not in fileset: | ||||
| continue | continue | ||||
| result.append((prefix, mode, stat)) | result.append((prefix, mode, stat)) | ||||
| return result | return result | ||||
| def _computeincrementalpack(ui, files, limits, packsuffix, indexsuffix, | def _computeincrementalpack(files, opts): | ||||
| gencountlimit, repacksizelimit, maxrepackpacks): | """Given a set of pack files along with the configuration options, this | ||||
| function computes the list of files that should be packed as part of an | |||||
| incremental repack. | |||||
| It tries to strike a balance between keeping incremental repacks cheap (i.e. | |||||
| packing small things when possible, and rolling the packs up to the big ones | |||||
| over time). | |||||
| """ | |||||
| limits = list(sorted((util.sizetoint(s) for s in opts['generations']), | |||||
| reverse=True)) | |||||
| limits.append(0) | |||||
| # Group the packs by generation (i.e. by size) | # Group the packs by generation (i.e. by size) | ||||
| generations = [] | generations = [] | ||||
| for i in xrange(len(limits)): | for i in xrange(len(limits)): | ||||
| generations.append([]) | generations.append([]) | ||||
| sizes = {} | sizes = {} | ||||
| for prefix, mode, stat in _allpackfileswithsuffix(files, packsuffix, | for prefix, mode, stat in files: | ||||
| indexsuffix): | |||||
| size = stat.st_size | size = stat.st_size | ||||
| sizes[prefix] = size | sizes[prefix] = size | ||||
| for i, limit in enumerate(limits): | for i, limit in enumerate(limits): | ||||
| if size > limit: | if size > limit: | ||||
| generations[i].append(prefix) | generations[i].append(prefix) | ||||
| break | break | ||||
| # Steps for picking what packs to repack: | # Steps for picking what packs to repack: | ||||
| # 1. Pick the largest generation with > gencountlimit pack files. | # 1. Pick the largest generation with > gencountlimit pack files. | ||||
| # 2. Take the smallest three packs. | # 2. Take the smallest three packs. | ||||
| # 3. While total-size-of-packs < repacksizelimit: add another pack | # 3. While total-size-of-packs < repacksizelimit: add another pack | ||||
| # Find the largest generation with more than gencountlimit packs | # Find the largest generation with more than gencountlimit packs | ||||
| genpacks = [] | genpacks = [] | ||||
| for i, limit in enumerate(limits): | for i, limit in enumerate(limits): | ||||
| if len(generations[i]) > gencountlimit: | if len(generations[i]) > opts['gencountlimit']: | ||||
| # Sort to be smallest last, for easy popping later | # Sort to be smallest last, for easy popping later | ||||
| genpacks.extend(sorted(generations[i], reverse=True, | genpacks.extend(sorted(generations[i], reverse=True, | ||||
| key=lambda x: sizes[x])) | key=lambda x: sizes[x])) | ||||
| break | break | ||||
| # Take as many packs from the generation as we can | # Take as many packs from the generation as we can | ||||
| chosenpacks = genpacks[-3:] | chosenpacks = genpacks[-3:] | ||||
| genpacks = genpacks[:-3] | genpacks = genpacks[:-3] | ||||
| repacksize = sum(sizes[n] for n in chosenpacks) | repacksize = sum(sizes[n] for n in chosenpacks) | ||||
| while (repacksize < repacksizelimit and genpacks and | while (repacksize < opts['repacksizelimit'] and genpacks and | ||||
| len(chosenpacks) < maxrepackpacks): | len(chosenpacks) < opts['maxrepackpacks']): | ||||
| chosenpacks.append(genpacks.pop()) | chosenpacks.append(genpacks.pop()) | ||||
| repacksize += sizes[chosenpacks[-1]] | repacksize += sizes[chosenpacks[-1]] | ||||
| # If there aren't any good candidates for a repack, | # If there aren't any good candidates for a repack, | ||||
| # repack the two largest ones. | # repack the two largest ones. | ||||
| if not chosenpacks and len(generations[0]) > 1: | if not chosenpacks and len(generations[0]) > 1: | ||||
| chosenpacks = generations[0] | chosenpacks = generations[0] | ||||