| indygreg |
| hg-reviewers |
I was going to change this code's use of CBOR to use our in-house CBOR code,
but discovered it's been broken for a while. This messy change gets it back to
a point where it mostly works, I think roughly as well as it ever did.
Should we keep this and fix it up the rest of the way, or dump it in favor of
the sqlite store? Would this be a good jumping-off point for some sort of
union store that could facilitate a cleanup in remotefilelog?
| Automatic diff as part of commit; lint not applicable. |
| Automatic diff as part of commit; unit tests not applicable. |
The original purpose of the simple store repo was to flush out problems with storage abstraction by building an alternate storage backend. Now that we have storage interface unit tests and the SQLite storage backend, there is definitely less of a need for the simple store backend and we could probably consider deleting it.
In D5519#81899, @indygreg wrote:The original purpose of the simple store repo was to flush out problems with storage abstraction by building an alternate storage backend. Now that we have storage interface unit tests and the SQLite storage backend, there is definitely less of a need for the simple store backend and we could probably consider deleting it.
I wonder if it's worth keeping the simpler simplestore code as an example. Probably not?
In D5519#81903, @durin42 wrote:In D5519#81899, @indygreg wrote:The original purpose of the simple store repo was to flush out problems with storage abstraction by building an alternate storage backend. Now that we have storage interface unit tests and the SQLite storage backend, there is definitely less of a need for the simple store backend and we could probably consider deleting it.
I wonder if it's worth keeping the simpler simplestore code as an example. Probably not?
With the SQLite store being a thing, probably not worth keeping even as an example. That being said, if it works and isn't hard to maintain, then why not keep it? The problem is it is difficult to maintain. You need to teach various tests about the existing of different backends because test output varies. Search for simplestore in the tests directory to see all the conditionals. Given the cost of maintaining support, I don't think it can justify its existence.
That being said, I'll probably regret this decision someday. Meh.
| def validaterev(rev): | def validaterev(rev): | ||||
| if not isinstance(rev, int): | if not isinstance(rev, int): | ||||
| raise ValueError('expected int') | raise ValueError('expected int') | ||||
| class simplestoreerror(error.StorageError): | class simplestoreerror(error.StorageError): | ||||
| pass | pass | ||||
| @interfaceutil.implementer(repository.irevisiondelta) | @interfaceutil.implementer(repository.irevisiondelta) | ||||
| @attr.s(slots=True, frozen=True) | @attr.s(slots=True) | ||||
| class simplestorerevisiondelta(object): | class simplestorerevisiondelta(object): | ||||
| node = attr.ib() | node = attr.ib() | ||||
| p1node = attr.ib() | p1node = attr.ib() | ||||
| p2node = attr.ib() | p2node = attr.ib() | ||||
| basenode = attr.ib() | basenode = attr.ib() | ||||
| linknode = attr.ib() | |||||
| flags = attr.ib() | flags = attr.ib() | ||||
| baserevisionsize = attr.ib() | baserevisionsize = attr.ib() | ||||
| revision = attr.ib() | revision = attr.ib() | ||||
| delta = attr.ib() | delta = attr.ib() | ||||
| linknode = attr.ib(default=None) | |||||
| @interfaceutil.implementer(repository.iverifyproblem) | |||||
| @attr.s(frozen=True) | |||||
| class simplefilestoreproblem(object): | |||||
| warning = attr.ib(default=None) | |||||
| error = attr.ib(default=None) | |||||
| node = attr.ib(default=None) | |||||
| @interfaceutil.implementer(repository.ifilestorage) | @interfaceutil.implementer(repository.ifilestorage) | ||||
| class filestorage(object): | class filestorage(object): | ||||
| """Implements storage for a tracked path. | """Implements storage for a tracked path. | ||||
| Data is stored in the VFS in a directory corresponding to the tracked | Data is stored in the VFS in a directory corresponding to the tracked | ||||
| path. | path. | ||||
| raise error.ProgrammingError('this should not occur') | raise error.ProgrammingError('this should not occur') | ||||
| def node(self, rev): | def node(self, rev): | ||||
| validaterev(rev) | validaterev(rev) | ||||
| return self._indexbyrev[rev][b'node'] | return self._indexbyrev[rev][b'node'] | ||||
| def hasnode(self, node): | |||||
| validatenode(node) | |||||
| return node in self._indexbynode | |||||
| def censorrevision(self, tr, censornode, tombstone=b''): | |||||
| raise NotImplementedError('TODO') | |||||
| def lookup(self, node): | def lookup(self, node): | ||||
| if isinstance(node, int): | if isinstance(node, int): | ||||
| return self.node(node) | return self.node(node) | ||||
| if len(node) == 20: | if len(node) == 20: | ||||
| self.rev(node) | self.rev(node) | ||||
| return node | return node | ||||
| def checkhash(self, text, node, p1=None, p2=None, rev=None): | def checkhash(self, text, node, p1=None, p2=None, rev=None): | ||||
| if p1 is None and p2 is None: | if p1 is None and p2 is None: | ||||
| p1, p2 = self.parents(node) | p1, p2 = self.parents(node) | ||||
| if node != storageutil.hashrevisionsha1(text, p1, p2): | if node != storageutil.hashrevisionsha1(text, p1, p2): | ||||
| raise simplestoreerror(_("integrity check failed on %s") % | raise simplestoreerror(_("integrity check failed on %s") % | ||||
| self._path) | self._path) | ||||
| def revision(self, node, raw=False): | def revision(self, nodeorrev, raw=False): | ||||
| if isinstance(nodeorrev, int): | |||||
| node = self.node(nodeorrev) | |||||
| else: | |||||
| node = nodeorrev | |||||
| validatenode(node) | validatenode(node) | ||||
| if node == nullid: | if node == nullid: | ||||
| return b'' | return b'' | ||||
| rev = self.rev(node) | rev = self.rev(node) | ||||
| flags = self._flags(rev) | flags = self._flags(rev) | ||||
| entries = self._svfs.listdir(self._storepath) | entries = self._svfs.listdir(self._storepath) | ||||
| # Strip out undo.backup.* files created as part of transaction | # Strip out undo.backup.* files created as part of transaction | ||||
| # recording. | # recording. | ||||
| entries = [f for f in entries if not f.startswith('undo.backup.')] | entries = [f for f in entries if not f.startswith('undo.backup.')] | ||||
| return [b'/'.join((self._storepath, f)) for f in entries] | return [b'/'.join((self._storepath, f)) for f in entries] | ||||
| def storageinfo(self, exclusivefiles=False, sharedfiles=False, | |||||
| revisionscount=False, trackedsize=False, | |||||
| storedsize=False): | |||||
| # TODO do a real implementation of this | |||||
| return { | |||||
| 'exclusivefiles': [], | |||||
| 'sharedfiles': [], | |||||
| 'revisionscount': len(self), | |||||
| 'trackedsize': 0, | |||||
| 'storedsize': None, | |||||
| } | |||||
| def verifyintegrity(self, state): | |||||
| state['skipread'] = set() | |||||
| for rev in self: | |||||
| node = self.node(rev) | |||||
| try: | |||||
| self.revision(node) | |||||
| except Exception as e: | |||||
| yield simplefilestoreproblem( | |||||
| error='unpacking %s: %s' % (node, e), | |||||
| node=node) | |||||
| state['skipread'].add(node) | |||||
| def emitrevisions(self, nodes, nodesorder=None, revisiondata=False, | |||||
| assumehaveparentrevisions=False, | |||||
| deltamode=repository.CG_DELTAMODE_STD): | |||||
| # TODO this will probably break on some ordering options. | |||||
| nodes = [n for n in nodes if n != nullid] | |||||
| if not nodes: | |||||
| return | |||||
| for delta in storageutil.emitrevisions( | |||||
| self, nodes, nodesorder, simplestorerevisiondelta, | |||||
| revisiondata=revisiondata, | |||||
| assumehaveparentrevisions=assumehaveparentrevisions, | |||||
| deltamode=deltamode): | |||||
| yield delta | |||||
| def add(self, text, meta, transaction, linkrev, p1, p2): | def add(self, text, meta, transaction, linkrev, p1, p2): | ||||
| if meta or text.startswith(b'\1\n'): | if meta or text.startswith(b'\1\n'): | ||||
| text = storageutil.packmeta(meta, text) | text = storageutil.packmeta(meta, text) | ||||
| return self.addrevision(text, transaction, linkrev, p1, p2) | return self.addrevision(text, transaction, linkrev, p1, p2) | ||||
| def addrevision(self, text, transaction, linkrev, p1, p2, node=None, | def addrevision(self, text, transaction, linkrev, p1, p2, node=None, | ||||
| flags=revlog.REVIDX_DEFAULT_FLAGS, cachedelta=None): | flags=revlog.REVIDX_DEFAULT_FLAGS, cachedelta=None): | ||||
| else: | else: | ||||
| text = mdiff.patch(self.revision(deltabase), delta) | text = mdiff.patch(self.revision(deltabase), delta) | ||||
| self._addrawrevision(node, text, transaction, linkrev, p1, p2, | self._addrawrevision(node, text, transaction, linkrev, p1, p2, | ||||
| flags) | flags) | ||||
| if addrevisioncb: | if addrevisioncb: | ||||
| addrevisioncb(self, node) | addrevisioncb(self, node) | ||||
| return nodes | return nodes | ||||
| def _headrevs(self): | |||||
| # Assume all revisions are heads by default. | |||||
| revishead = {rev: True for rev in self._indexbyrev} | |||||
| for rev, entry in self._indexbyrev.items(): | |||||
| # Unset head flag for all seen parents. | |||||
| revishead[self.rev(entry[b'p1'])] = False | |||||
| revishead[self.rev(entry[b'p2'])] = False | |||||
| return [rev for rev, ishead in sorted(revishead.items()) | |||||
| if ishead] | |||||
| def heads(self, start=None, stop=None): | def heads(self, start=None, stop=None): | ||||
| # This is copied from revlog.py. | # This is copied from revlog.py. | ||||
| if start is None and stop is None: | if start is None and stop is None: | ||||
| if not len(self): | if not len(self): | ||||
| return [nullid] | return [nullid] | ||||
| return [self.node(r) for r in self.headrevs()] | return [self.node(r) for r in self._headrevs()] | ||||
| if start is None: | if start is None: | ||||
| start = nullid | start = nullid | ||||
| if stop is None: | if stop is None: | ||||
| stop = [] | stop = [] | ||||
| stoprevs = set([self.rev(n) for n in stop]) | stoprevs = set([self.rev(n) for n in stop]) | ||||
| startrev = self.rev(start) | startrev = self.rev(start) | ||||
| reachable = {startrev} | reachable = {startrev} | ||||
| for pr in prevs: | for pr in prevs: | ||||
| if pr == p: | if pr == p: | ||||
| c.append(self.node(r)) | c.append(self.node(r)) | ||||
| elif p == nullrev: | elif p == nullrev: | ||||
| c.append(self.node(r)) | c.append(self.node(r)) | ||||
| return c | return c | ||||
| def getstrippoint(self, minlink): | def getstrippoint(self, minlink): | ||||
| return storageutil.resolvestripinfo( | |||||
| # This is largely a copy of revlog.getstrippoint(). | minlink, len(self) - 1, self._headrevs(), self.linkrev, | ||||
| brokenrevs = set() | self.parentrevs) | ||||
| strippoint = len(self) | |||||
| heads = {} | |||||
| futurelargelinkrevs = set() | |||||
| for head in self.heads(): | |||||
| headlinkrev = self.linkrev(self.rev(head)) | |||||
| heads[head] = headlinkrev | |||||
| if headlinkrev >= minlink: | |||||
| futurelargelinkrevs.add(headlinkrev) | |||||
| # This algorithm involves walking down the rev graph, starting at the | |||||
| # heads. Since the revs are topologically sorted according to linkrev, | |||||
| # once all head linkrevs are below the minlink, we know there are | |||||
| # no more revs that could have a linkrev greater than minlink. | |||||
| # So we can stop walking. | |||||
| while futurelargelinkrevs: | |||||
| strippoint -= 1 | |||||
| linkrev = heads.pop(strippoint) | |||||
| if linkrev < minlink: | |||||
| brokenrevs.add(strippoint) | |||||
| else: | |||||
| futurelargelinkrevs.remove(linkrev) | |||||
| for p in self.parentrevs(strippoint): | |||||
| if p != nullrev: | |||||
| plinkrev = self.linkrev(p) | |||||
| heads[p] = plinkrev | |||||
| if plinkrev >= minlink: | |||||
| futurelargelinkrevs.add(plinkrev) | |||||
| return strippoint, brokenrevs | |||||
| def strip(self, minlink, transaction): | def strip(self, minlink, transaction): | ||||
| if not len(self): | if not len(self): | ||||
| return | return | ||||
| rev, _ignored = self.getstrippoint(minlink) | rev, _ignored = self.getstrippoint(minlink) | ||||
| if rev == len(self): | if rev == len(self): | ||||
| return | return | ||||
| def file(self, f): | def file(self, f): | ||||
| return filestorage(self.svfs, f) | return filestorage(self.svfs, f) | ||||
| repo.__class__ = simplestorerepo | repo.__class__ = simplestorerepo | ||||
| def featuresetup(ui, supported): | def featuresetup(ui, supported): | ||||
| supported.add(REQUIREMENT) | supported.add(REQUIREMENT) | ||||
| def newreporequirements(orig, ui): | def newreporequirements(orig, ui, createopts): | ||||
| """Modifies default requirements for new repos to use the simple store.""" | """Modifies default requirements for new repos to use the simple store.""" | ||||
| requirements = orig(ui) | requirements = orig(ui, createopts) | ||||
| # These requirements are only used to affect creation of the store | # These requirements are only used to affect creation of the store | ||||
| # object. We have our own store. So we can remove them. | # object. We have our own store. So we can remove them. | ||||
| # TODO do this once we feel like taking the test hit. | # TODO do this once we feel like taking the test hit. | ||||
| #if 'fncache' in requirements: | #if 'fncache' in requirements: | ||||
| # requirements.remove('fncache') | # requirements.remove('fncache') | ||||
| #if 'dotencode' in requirements: | #if 'dotencode' in requirements: | ||||
| # requirements.remove('dotencode') | # requirements.remove('dotencode') | ||||
| # advertised. So suppress these warnings. | # advertised. So suppress these warnings. | ||||
| self.warnorphanstorefiles = False | self.warnorphanstorefiles = False | ||||
| def extsetup(ui): | def extsetup(ui): | ||||
| localrepo.featuresetupfuncs.add(featuresetup) | localrepo.featuresetupfuncs.add(featuresetup) | ||||
| extensions.wrapfunction(localrepo, 'newreporequirements', | extensions.wrapfunction(localrepo, 'newreporequirements', | ||||
| newreporequirements) | newreporequirements) | ||||
| extensions.wrapfunction(store, 'store', makestore) | extensions.wrapfunction(localrepo, 'makestore', makestore) | ||||
| extensions.wrapfunction(verify.verifier, '__init__', verifierinit) | extensions.wrapfunction(verify.verifier, '__init__', verifierinit) | ||||