diff --git a/tests/test-treemanifest-server.t b/tests/test-treemanifest-server.t --- a/tests/test-treemanifest-server.t +++ b/tests/test-treemanifest-server.t @@ -193,3 +193,42 @@ $ hg debugindex .hg/store/meta/subdir/00manifest.i rev offset length delta linkrev nodeid p1 p2 0 0 44 -1 0 bc0c2c938b92 000000000000 000000000000 + +Test fetching from the server populates the cache + $ cd ../ + $ hgcloneshallow ssh://user@dummy/master client2 -q -U + $ cd client2 + $ cat >> .hg/hgrc < [extensions] + > treemanifest=$TESTDIR/../treemanifest + > fastmanifest=$TESTDIR/../fastmanifest + > [fastmanifest] + > usetree=True + > usecache=False + > [treemanifest] + > demanddownload=True + > sendtrees=True + > EOF + $ clearcache + $ hg status --change tip > /dev/null + 2 trees fetched over * (glob) + 2 trees fetched over * (glob) + $ ls_l ../master/.hg/cache/trees/v1/get + -rw-r--r-- 69 0b0fa4abc415aa6a46e003c61283b182ccc989b6 + -rw-r--r-- 63 bfd3db72113838ac7ebcf260374e4bf2884b3ddd + -rw-r--r-- 63 d4395b5ffa18499864439ac2b1a731ff7b7491fa + -rw-r--r-- 119 db19d9b3947b6a67f2c1a7530fb4ff1b5f52cc12 + $ ls_l ../master/.hg/cache/trees/v1/nodeinfo + -rw-r--r-- 84 0b0fa4abc415aa6a46e003c61283b182ccc989b6 + -rw-r--r-- 84 6768033e216468247bd031a0a2d9876d79818f8f + -rw-r--r-- 84 bfd3db72113838ac7ebcf260374e4bf2884b3ddd + -rw-r--r-- 84 d4395b5ffa18499864439ac2b1a731ff7b7491fa + -rw-r--r-- 84 db19d9b3947b6a67f2c1a7530fb4ff1b5f52cc12 + +- Move the revlogs away to show that the cache is answering prefetches + $ mv ../master/.hg/store/meta ../master/.hg/store/meta.bak + $ clearcache + $ hg status --change tip > /dev/null + 2 trees fetched over * (glob) + 2 trees fetched over * (glob) + $ mv ../master/.hg/store/meta.bak ../master/.hg/store/meta diff --git a/treemanifest/__init__.py b/treemanifest/__init__.py --- a/treemanifest/__init__.py +++ b/treemanifest/__init__.py @@ -63,6 +63,13 @@ [treemanifest] treeonly = True +`treemanifest.cacheserverstore` causes the treemanifest server to store a cache +of treemanifest revisions in individual files. These improve lookup speed since +we don't have to open a revlog. + + [treemanifest] + cacheserverstore = True + """ from mercurial import ( @@ -127,6 +134,7 @@ ) import cstore +import hashlib import os import shutil import struct @@ -142,6 +150,7 @@ configitem('treemanifest', 'sendtrees', default=False) configitem('treemanifest', 'server', default=False) +configitem('treemanifest', 'cacheserverstore', default=True) PACK_CATEGORY='manifests' @@ -284,6 +293,9 @@ # Data store datastore = cstore.datapackstore(packpath) revlogstore = manifestrevlogstore(repo) + if repo.ui.configbool("treemanifest", "cacheserverstore"): + revlogstore = cachestore(revlogstore, repo.cachevfs) + mfl.datastore = unioncontentstore(datastore, revlogstore) # History store @@ -1710,6 +1722,7 @@ class MissingNodesError(error.Abort): def __init__(self, nodes, message=None, hint=None): + nodes = list(nodes) nodestr = '\n'.join(hex(mfnode) for mfnode in nodes[:10]) if len(nodes) > 10: nodestr += '\n...' @@ -1719,3 +1732,100 @@ fullmessage += ' ' + message fullmessage += '\n' + nodestr super(MissingNodesError, self).__init__(fullmessage, hint=hint) + +NODEINFOFORMAT='!20s20s20sI' +NODEINFOLEN=struct.calcsize(NODEINFOFORMAT) +class cachestore(object): + def __init__(self, store, vfs, version=1): + self.store = store + self.vfs = vfs + self.version = version + + def _key(self, name, node, category): + shakey = hex(hashlib.sha1(name + node).digest()) + return os.path.join('trees', 'v' + str(self.version), category, shakey) + + def get(self, name, node): + if node == nullid: + return '' + + try: + key = self._key(name, node, 'get') + return self._read(key) + except (IOError, OSError): + data = self.store.get(name, node) + self._write(key, data) + return data + + def getdelta(self, name, node): + revision = self.get(name, node) + return (revision, name, nullid, {}) + + def getdeltachain(self, name, node): + revision = self.get(name, node) + return [(name, node, None, nullid, revision)] + + def getmeta(self, name, node): + return self.store.getmeta(name, node) + + def getmissing(self, keys): + missing = [] + for name, node in keys: + if not self.vfs.exists(self._key(name, node, 'get')): + missing.append((name, node)) + + return self.store.getmissing(missing) + + def getancestors(self, name, node, known=None): + return self.store.getancestors(name, node, known=known) + + def _serializenodeinfo(self, nodeinfo): + p1, p2, linknode, copyfrom = nodeinfo + if copyfrom is None: + copyfrom = '' + raw = struct.pack(NODEINFOFORMAT, p1, p2, linknode, len(copyfrom)) + return raw + copyfrom + + def _deserializenodeinfo(self, raw): + p1, p2, linknode, copyfromlen = struct.unpack_from(NODEINFOFORMAT, raw, + 0) + if len(raw) != NODEINFOLEN + copyfromlen: + raise IOError("invalid nodeinfo serialization: %s %s %s %s %s" % + (hex(p1), hex(p2), hex(linknode), str(copyfromlen), + raw[NODEINFOLEN:])) + return p1, p2, linknode, raw[NODEINFOLEN:NODEINFOLEN + copyfromlen] + + def _verifyvalue(self, value): + sha, value = value[:20], value[20:] + realsha = hashlib.sha1(value).digest() + if sha != realsha: + raise IOError() + return value + + def _read(self, key): + with self.vfs(key) as f: + raw = f.read() + if raw == '': + raise IOError("missing file contents: %s" % self.vfs.join(key)) + + sha, value = raw[:20], raw[20:] + realsha = hashlib.sha1(value).digest() + if sha != realsha: + raise IOError("invalid file contents: %s" % self.vfs.join(key)) + return value + + def _write(self, key, value): + with self.vfs(key, 'w+') as f: + sha = hashlib.sha1(value).digest() + f.write(sha) + f.write(value) + + def getnodeinfo(self, name, node): + key = self._key(name, node, 'nodeinfo') + try: + raw = self._read(key) + return self._deserializenodeinfo(raw) + except (IOError, OSError): + nodeinfo = self.store.getnodeinfo(name, node) + self._write(key, self._serializenodeinfo(nodeinfo)) + return nodeinfo