This is an archive of the discontinued Mercurial Phabricator instance.

Differential D1350

treemanifest: improve server side incremental repack
ClosedPublic

Authored by durham on Nov 8 2017, 6:54 PM.
Tags
None
Subscribers
quark

Details

Reviewers
phillco
quark
Group Reviewers
Restricted Project
Commits
rFBHGXc93f3c014cec: treemanifest: improve server side incremental repack
Summary

Currently an incremental repack on the server will repack the entire pack files, and the new parts of the revlogs. The pack files can be very large and can take a long, long time to run. So let's use the normal incremental pack heuristics to minimize how often we have to do full repacks.

Test Plan

Added a test

Diff Detail

Repository
rFBHGX Facebook Mercurial Extensions
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

durham created this revision.Nov 8 2017, 6:54 PM
Herald added a reviewer: Restricted Project. · View Herald TranscriptNov 8 2017, 6:54 PM
durham edited the summary of this revision. (Show Details)Nov 8 2017, 6:59 PM
durham edited the test plan for this revision. (Show Details)
phillco accepted this revision.Nov 9 2017, 1:21 PM
This revision is now accepted and ready to land.Nov 9 2017, 1:21 PM
quark accepted this revision.Nov 9 2017, 1:21 PM
quark added a subscriber: quark.
quark added inline comments.
treemanifest/__init__.py
1493

Do we really need stats here? If not, maybe set stat=False, or just use os.listdir instead of osutil.

Closed by commit rFBHGXc93f3c014cec: treemanifest: improve server side incremental repack (authored by durham). · Explain WhyNov 9 2017, 1:29 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 3379

tests/test-treemanifest-repack.t

1832e0765de9 000000000000 89 89 1832e0765de9 000000000000 89 89
Total: 89 89 (0.0% bigger) Total: 89 89 (0.0% bigger)
dir: dir:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
23226e7a252c 000000000000 43 43 23226e7a252c 000000000000 43 43
Total: 43 43 (0.0% bigger) Total: 43 43 (0.0% bigger)
Test incremental repack that doesn't take all packs
$ ls_l .hg/cache/packs/manifests/ | grep datapack
-r--r--r-- 264 e9093d2d887ff14457d43338fcb3994e92051853.datapack
- Only one pack, means don't repack it. Only turn revlogs into a pack.
$ hg repack --incremental --config remotefilelog.data.generations=300,20
$ ls_l .hg/cache/packs/manifests/ | grep datapack
-r--r--r-- 264 e9093d2d887ff14457d43338fcb3994e92051853.datapack
-r--r--r-- 154 f9657fdc11d7c9847208da3f1245b38c5981df79.datapack
- Two packs doesn't meet the bar for repack. Only turn revlogs into a pack.
$ echo >> a
$ hg commit -m 'modify a'
$ hg repack --incremental --config remotefilelog.data.generations=300,20
$ ls_l .hg/cache/packs/manifests/ | grep datapack
-r--r--r-- 154 0adbde90bc92c6f23e46180a9d7885c8e2499173.datapack
-r--r--r-- 264 e9093d2d887ff14457d43338fcb3994e92051853.datapack
-r--r--r-- 154 f9657fdc11d7c9847208da3f1245b38c5981df79.datapack
- Three packs meets the bar. Repack new revlogs and old pack into one.
$ hg repack --incremental --config remotefilelog.data.generations=300,20
$ ls_l .hg/cache/packs/manifests/ | grep datapack
-r--r--r-- 496 bc6c2ebb080844d7a227dacbc847a5b375ec620c.datapack

treemanifest/__init__.py

error, error,
exchange, exchange,
extensions, extensions,
hg, hg,
localrepo, localrepo,
manifest, manifest,
mdiff, mdiff,
phases, phases,
policy,
registrar, registrar,
repair, repair,
revlog, revlog,
sshserver, sshserver,
templatekw, templatekw,
util, util,
wireproto, wireproto,
) )
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial.node import bin, hex, nullid from mercurial.node import bin, hex, nullid
from remotefilelog.contentstore import ( from remotefilelog.contentstore import (
manifestrevlogstore, manifestrevlogstore,
unioncontentstore, unioncontentstore,
) )
from remotefilelog.metadatastore import ( from remotefilelog.metadatastore import (
unionmetadatastore, unionmetadatastore,
) )
from remotefilelog.datapack import ( from remotefilelog.datapack import (
datapack,
datapackstore, datapackstore,
mutabledatapack, mutabledatapack,
) )
from remotefilelog.historypack import ( from remotefilelog.historypack import (
historypack,
historypackstore, historypackstore,
mutablehistorypack, mutablehistorypack,
) )
from remotefilelog import shallowrepo, shallowutil, wirepack from remotefilelog import shallowrepo, shallowutil, wirepack
from remotefilelog.repack import ( from remotefilelog.repack import (
_computeincrementaldatapack,
_computeincrementalhistorypack,
_runrepack, _runrepack,
_topacks,
) )
import cstore import cstore
import os import os
import struct import struct
import time import time
osutil = policy.importmod(r'osutil')
cmdtable = {} cmdtable = {}
command = registrar.command(cmdtable) command = registrar.command(cmdtable)
configtable = {} configtable = {}
configitem = registrar.configitem(configtable) configitem = registrar.configitem(configtable)
configitem('treemanifest', 'sendtrees', default=False) configitem('treemanifest', 'sendtrees', default=False)
configitem('treemanifest', 'server', default=False) configitem('treemanifest', 'server', default=False)
pass pass
def getmetrics(self): def getmetrics(self):
return {} return {}
def serverrepack(repo, incremental=False): def serverrepack(repo, incremental=False):
packpath = repo.vfs.join('cache/packs/%s' % PACK_CATEGORY) packpath = repo.vfs.join('cache/packs/%s' % PACK_CATEGORY)
dpackstore = datapackstore(repo.ui, packpath)
revlogstore = manifestrevlogstore(repo) revlogstore = manifestrevlogstore(repo)
datastore = unioncontentstore(dpackstore, revlogstore)
hpackstore = historypackstore(repo.ui, packpath) try:
histstore = unionmetadatastore(hpackstore, revlogstore) files = osutil.listdir(packpath, stat=True)
quarkUnsubmitted
Not Done

Do we really need stats here? If not, maybe set stat=False, or just use os.listdir instead of osutil.

quark: Do we really need stats here? If not, maybe set `stat=False`, or just use `os.listdir` instead…
except OSError:
files = []
# Data store
fulldatapackstore = datapackstore(repo.ui, packpath)
if incremental:
datastores = _topacks(packpath,
_computeincrementaldatapack(repo.ui, files),
datapack)
else:
datastores = [fulldatapackstore]
datastores.append(revlogstore)
datastore = unioncontentstore(*datastores)
# History store
if incremental:
historystores = _topacks(packpath,
_computeincrementalhistorypack(repo.ui, files),
historypack)
else:
historystores = [historypackstore(repo.ui, packpath)]
historystores.append(revlogstore)
histstore = unionmetadatastore(*historystores)
startrev = repo.ui.configint('treemanifest', 'repackstartrev', 0) startrev = repo.ui.configint('treemanifest', 'repackstartrev', 0)
endrev = repo.ui.configint('treemanifest', 'repackendrev', endrev = repo.ui.configint('treemanifest', 'repackendrev',
len(repo.changelog) - 1) len(repo.changelog) - 1)
if startrev == 0 and incremental: if startrev == 0 and incremental:
latestpackedlinkrev = 0 latestpackedlinkrev = 0
mfrevlog = repo.manifestlog.treemanifestlog._revlog mfrevlog = repo.manifestlog.treemanifestlog._revlog
for i in xrange(len(mfrevlog) - 1, 0, -1): for i in xrange(len(mfrevlog) - 1, 0, -1):
node = mfrevlog.node(i) node = mfrevlog.node(i)
if not dpackstore.getmissing([('', node)]): if not fulldatapackstore.getmissing([('', node)]):
latestpackedlinkrev = mfrevlog.linkrev(i) latestpackedlinkrev = mfrevlog.linkrev(i)
break break
startrev = latestpackedlinkrev + 1 startrev = latestpackedlinkrev + 1
revlogstore.setrepacklinkrevrange(startrev, endrev) revlogstore.setrepacklinkrevrange(startrev, endrev)
_runrepack(repo, datastore, histstore, packpath, PACK_CATEGORY) _runrepack(repo, datastore, histstore, packpath, PACK_CATEGORY)
def striptrees(orig, repo, tr, striprev, files): def striptrees(orig, repo, tr, striprev, files):