diff --git a/cstore/py-datapackstore.h b/cstore/py-datapackstore.h --- a/cstore/py-datapackstore.h +++ b/cstore/py-datapackstore.h @@ -143,12 +143,17 @@ Py_RETURN_NONE; } +static PyObject *datapackstore_getmetrics(py_datapackstore *self, PyObject* kw) { + return PyDict_New(); +} + // --------- DatapackStore Declaration --------- static PyMethodDef datapackstore_methods[] = { {"getdeltachain", (PyCFunction)datapackstore_getdeltachain, METH_VARARGS, ""}, {"getmissing", (PyCFunction)datapackstore_getmissing, METH_O, ""}, {"markforrefresh", (PyCFunction)datapackstore_markforrefresh, METH_NOARGS, ""}, + {"getmetrics", (PyCFunction)datapackstore_getmetrics, METH_KEYWORDS, ""}, {NULL, NULL} }; @@ -369,7 +374,7 @@ Py_RETURN_NONE; } -static PyObject *uniondatapackstore_getmetrics(py_uniondatapackstore *self) { +static PyObject *uniondatapackstore_getmetrics(py_uniondatapackstore *self, PyObject* kw) { return PyDict_New(); } @@ -380,7 +385,7 @@ {"getdeltachain", (PyCFunction)uniondatapackstore_getdeltachain, METH_VARARGS, ""}, {"getmissing", (PyCFunction)uniondatapackstore_getmissing, METH_O, ""}, {"markforrefresh", (PyCFunction)uniondatapackstore_markforrefresh, METH_NOARGS, ""}, - {"getmetrics", (PyCFunction)uniondatapackstore_getmetrics, METH_NOARGS, ""}, + {"getmetrics", (PyCFunction)uniondatapackstore_getmetrics, METH_KEYWORDS, ""}, {NULL, NULL} }; diff --git a/remotefilelog/__init__.py b/remotefilelog/__init__.py --- a/remotefilelog/__init__.py +++ b/remotefilelog/__init__.py @@ -933,6 +933,10 @@ def debughistorypack(ui, path, **opts): return debugcommands.debughistorypack(ui, path) +@command('debugpackstatus', [], 'hg debugpackstatus') +def debugpackstatus(repo, ui, **opts): + return debugcommands.debugpackstatus(repo, ui) + @command('debugkeepset', [ ], _('hg debugkeepset')) def debugkeepset(ui, repo, **opts): diff --git a/remotefilelog/basepack.py b/remotefilelog/basepack.py --- a/remotefilelog/basepack.py +++ b/remotefilelog/basepack.py @@ -191,7 +191,7 @@ count += 1 return totalsize, count - def getmetrics(self): + def getmetrics(self, verbose=False): """Returns metrics on the state of this store.""" size, count = self.gettotalsizeandcount() return { diff --git a/remotefilelog/basestore.py b/remotefilelog/basestore.py --- a/remotefilelog/basestore.py +++ b/remotefilelog/basestore.py @@ -41,6 +41,43 @@ if shared: shallowutil.mkstickygroupdir(self.ui, path) + def getnumfiles(self): + count = 0 + try: + for __, __ in self._listkeys(): + count += 1 + except Exception: + pass + return count + + def gettotalsize(self): + totalsize = 0 + try: + for fnhash, node in self._listkeys(): + fnhashhex = hex(fnhash) + if self._shared: + path = os.path.join(self._getrootpath(), fnhashhex[:2], + fnhashhex[2:], hex(node)) + else: + path = os.path.join(self._getrootpath(), fnhashhex, + hex(node)) + totalsize += os.stat(path).st_size + except Exception: + pass + + return totalsize + + def getmetrics(self, verbose=False): + if verbose: + # Calculating the two is very, very slow, so we only do so when + # requested (e.g. in `hg debugpackstatus`), and not on every comand. + return { + 'numloosefiles': self.getnumfiles(), + 'totalloosesize': self.gettotalsize(), + } + else: + return {} + def getmissing(self, keys): missing = [] for name, node in keys: @@ -155,16 +192,18 @@ return filenames + def _getrootpath(self): + if self._shared: + return os.path.join(self._path, self._reponame) + else: + return self._path + def _listkeys(self): """List all the remotefilelog keys that exist in the store. Returns a iterator of (filename hash, filecontent hash) tuples. """ - if self._shared: - path = os.path.join(self._path, self._reponame) - else: - path = self._path - + path = self._getrootpath() for root, dirs, files in os.walk(path): for filename in files: if len(filename) != 40: diff --git a/remotefilelog/contentstore.py b/remotefilelog/contentstore.py --- a/remotefilelog/contentstore.py +++ b/remotefilelog/contentstore.py @@ -93,8 +93,8 @@ pass raise KeyError((name, hex(node))) - def getmetrics(self): - metrics = [s.getmetrics() for s in self.stores] + def getmetrics(self, verbose=False): + metrics = [s.getmetrics(verbose=verbose) for s in self.stores] return shallowutil.sumdicts(*metrics) def _getpartialchain(self, name, node): @@ -222,6 +222,9 @@ def markledger(self, ledger, options=None): pass + def getmetrics(self, verbose=False): + return {} + class manifestrevlogstore(object): def __init__(self, repo): self._store = repo.store diff --git a/remotefilelog/debugcommands.py b/remotefilelog/debugcommands.py --- a/remotefilelog/debugcommands.py +++ b/remotefilelog/debugcommands.py @@ -6,7 +6,8 @@ # GNU General Public License version 2 or any later version. from __future__ import absolute_import -from mercurial import error, filelog, revlog +import itertools +from mercurial import error, filelog, revlog, util from mercurial.node import bin, hex, nullid, short from mercurial.i18n import _ from . import ( @@ -198,6 +199,7 @@ return size, firstnode, mapping def debugdatapack(ui, *paths, **opts): + failures = 0 for path in paths: if '.data' in path: path = path[:path.index('.data')] @@ -238,7 +240,6 @@ bases = {} nodes = set() - failures = 0 for filename, node, deltabase, deltalen in dpack.iterentries(): bases[node] = deltabase if node in nodes: @@ -276,9 +277,9 @@ printtotals() failures += _sanitycheck(ui, set(nodes), bases) - if failures > 1: - ui.warn(("%d failures\n" % failures)) - return 1 + if failures > 1: + ui.warn(("%d failures\n" % failures)) + return 1 def _sanitycheck(ui, nodes, bases): """ @@ -357,6 +358,70 @@ ui.write("%s %s %s %s %s\n" % (short(node), short(p1node), short(p2node), short(linknode), copyfrom)) +def debugpackstatus(ui, repo): + def format(metrics, paths=None): + paths = list(itertools.chain.from_iterable(compact(paths))) + parts = [] + if 'numpacks' in metrics: + parts.append("%d packs consuming %s" % (metrics['numpacks'], + util.bytecount(metrics['totalpacksize']))) + if 'numloosefiles' in metrics: + parts.append("%d loose files consuming %s" % ( + metrics['numloosefiles'], + util.bytecount(metrics['totalloosesize']))) + + if paths: + parts.append("in %s" % ", ".join(paths)) + return ", ".join(parts) + + def compact(list): + return [i for i in list if i is not None] + + def getpaths(store): + if store is None: + return [] + # Support union stores: + if getattr(store, 'stores', None): + return list(itertools.chain.from_iterable(compact( + [getpaths(s) for s in store.stores] + ))) + # Some stores use _path for the attribute, others path: + try: + return [store.path] + except AttributeError: + pass + try: + return [store._path] + except AttributeError: + pass + return None + + mfl = repo.manifestlog + if util.safehasattr(mfl, 'localdatastores'): + ui.write(("Local Tree Store: %s\n" % format( + shallowutil.sumdicts(*[s.getmetrics() for s in + mfl.localdatastores]), + [getpaths(p) for p in mfl.localdatastores] + ))) + ui.write(("Shared Tree Store: %s\n" % format( + shallowutil.sumdicts(*[s.getmetrics() for s in + mfl.shareddatastores]), + [getpaths(p) for p in mfl.localdatastores] + ))) + else: + ui.write(("(No Tree Store)\n")) + + if (util.safehasattr(repo, 'contentstore') and + util.safehasattr(repo, 'metadatastore')): + ui.write(("File Content Store: %s\n" % + format(repo.contentstore.getmetrics(verbose=True), + [getpaths(repo.contentstore)]))) + ui.write(("File Metadata Store: %s\n" % + format(repo.metadatastore.getmetrics(verbose=True), + [getpaths(repo.contentstore)]))) + else: + ui.write(("(No File Store)\n")) + def debugwaitonrepack(repo): with repo._lock(repo.svfs, "repacklock", True, None, None, _('repacking %s') % repo.origroot): diff --git a/remotefilelog/metadatastore.py b/remotefilelog/metadatastore.py --- a/remotefilelog/metadatastore.py +++ b/remotefilelog/metadatastore.py @@ -100,8 +100,8 @@ for store in self.stores: store.markledger(ledger, options) - def getmetrics(self): - metrics = [s.getmetrics() for s in self.stores] + def getmetrics(self, verbose=False): + metrics = [s.getmetrics(verbose=verbose) for s in self.stores] return shallowutil.sumdicts(*metrics) class remotefilelogmetadatastore(basestore.basestore): @@ -145,3 +145,6 @@ def markledger(self, ledger, options=None): pass + + def getmetrics(self, verbose=False): + return {} diff --git a/tests/test-remotefilelog-repack.t b/tests/test-remotefilelog-repack.t --- a/tests/test-remotefilelog-repack.t +++ b/tests/test-remotefilelog-repack.t @@ -31,6 +31,10 @@ # Test that repack cleans up the old files and creates new packs $ cd shallow + $ hg debugpackstatus + (No Tree Store) + File Content Store: 0 packs consuming 0 bytes, 1 loose files consuming 168 bytes, in $TESTTMP/hgcache/master/packs, $TESTTMP/hgcache, $TESTTMP/shallow/.hg/store/data + File Metadata Store: 0 packs consuming 0 bytes, 1 loose files consuming 168 bytes, in $TESTTMP/hgcache/master/packs, $TESTTMP/hgcache, $TESTTMP/shallow/.hg/store/data $ find $CACHEDIR | sort $TESTTMP/hgcache $TESTTMP/hgcache/master @@ -50,6 +54,10 @@ $TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx $TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack $TESTTMP/hgcache/repos + $ hg debugpackstatus + (No Tree Store) + File Content Store: 1 packs consuming 1.12 KB, 0 loose files consuming 0 bytes, in $TESTTMP/hgcache/master/packs, $TESTTMP/hgcache, $TESTTMP/shallow/.hg/store/data + File Metadata Store: 1 packs consuming 1.29 KB, 0 loose files consuming 0 bytes, in $TESTTMP/hgcache/master/packs, $TESTTMP/hgcache, $TESTTMP/shallow/.hg/store/data # Test that the packs are readonly $ ls_l $CACHEDIR/master/packs diff --git a/tests/test-treemanifest-repack.t b/tests/test-treemanifest-repack.t --- a/tests/test-treemanifest-repack.t +++ b/tests/test-treemanifest-repack.t @@ -80,6 +80,10 @@ 23226e7a252c 000000000000 000000000000 8e83608cbe60 - Repack and reverify + $ hg debugpackstatus + Local Tree Store: 0 packs consuming 0 bytes, in $TESTTMP/client/.hg/store/packs/manifests + Shared Tree Store: 2 packs consuming 2.46 KB, in $TESTTMP/client/.hg/store/packs/manifests + (No File Store) $ hg repack $ ls_l $CACHEDIR/master/packs/manifests | grep pack @@ -110,6 +114,10 @@ dir Node P1 Node P2 Node Link Node Copy From 23226e7a252c 000000000000 000000000000 8e83608cbe60 + $ hg debugpackstatus + Local Tree Store: 0 packs consuming 0 bytes, in $TESTTMP/client/.hg/store/packs/manifests + Shared Tree Store: 1 packs consuming 1.46 KB, in $TESTTMP/client/.hg/store/packs/manifests + (No File Store) # Test repacking local manifest packs $ hg up -q 1 diff --git a/treemanifest/__init__.py b/treemanifest/__init__.py --- a/treemanifest/__init__.py +++ b/treemanifest/__init__.py @@ -1538,7 +1538,7 @@ def markledger(self, ledger, options=None): pass - def getmetrics(self): + def getmetrics(self, verbose=False): return {} def serverrepack(repo, incremental=False, options=None):