| hg-reviewers |
There will exist common code between storage backends. It would
be nice to have a central place to put that code.
This commit attempts to create that place by creating the
"storageutil" module.
The first thing we move is revlog.hash(), which is the function for
computing the SHA-1 hash of revision fulltext and parents.
| Automatic diff as part of commit; lint not applicable. |
| Automatic diff as part of commit; unit tests not applicable. |
| Path | Packages | |||
|---|---|---|---|---|
| M | mercurial/revlog.py (33 lines) | |||
| A | M | mercurial/utils/storageutil.py (41 lines) | ||
| M | tests/simplestorerepo.py (9 lines) |
| and O(changes) merge between branches. | and O(changes) merge between branches. | ||||
| """ | """ | ||||
| from __future__ import absolute_import | from __future__ import absolute_import | ||||
| import collections | import collections | ||||
| import contextlib | import contextlib | ||||
| import errno | import errno | ||||
| import hashlib | |||||
| import os | import os | ||||
| import re | import re | ||||
| import struct | import struct | ||||
| import zlib | import zlib | ||||
| # import stuff from node for others to import from revlog | # import stuff from node for others to import from revlog | ||||
| from .node import ( | from .node import ( | ||||
| bin, | bin, | ||||
| templatefilters, | templatefilters, | ||||
| util, | util, | ||||
| ) | ) | ||||
| from .revlogutils import ( | from .revlogutils import ( | ||||
| deltas as deltautil, | deltas as deltautil, | ||||
| ) | ) | ||||
| from .utils import ( | from .utils import ( | ||||
| interfaceutil, | interfaceutil, | ||||
| storageutil, | |||||
| stringutil, | stringutil, | ||||
| ) | ) | ||||
| # blanked usage of all the name to prevent pyflakes constraints | # blanked usage of all the name to prevent pyflakes constraints | ||||
| # We need these name available in the module for extensions. | # We need these name available in the module for extensions. | ||||
| REVLOGV0 | REVLOGV0 | ||||
| REVLOGV1 | REVLOGV1 | ||||
| REVLOGV2 | REVLOGV2 | ||||
| def gettype(q): | def gettype(q): | ||||
| return int(q & 0xFFFF) | return int(q & 0xFFFF) | ||||
| def offset_type(offset, type): | def offset_type(offset, type): | ||||
| if (type & ~REVIDX_KNOWN_FLAGS) != 0: | if (type & ~REVIDX_KNOWN_FLAGS) != 0: | ||||
| raise ValueError('unknown revlog index flags') | raise ValueError('unknown revlog index flags') | ||||
| return int(int(offset) << 16 | type) | return int(int(offset) << 16 | type) | ||||
| _nullhash = hashlib.sha1(nullid) | |||||
| def hash(text, p1, p2): | |||||
| """generate a hash from the given text and its parent hashes | |||||
| This hash combines both the current file contents and its history | |||||
| in a manner that makes it easy to distinguish nodes with the same | |||||
| content in the revision graph. | |||||
| """ | |||||
| # As of now, if one of the parent node is null, p2 is null | |||||
| if p2 == nullid: | |||||
| # deep copy of a hash is faster than creating one | |||||
| s = _nullhash.copy() | |||||
| s.update(p1) | |||||
| else: | |||||
| # none of the parent nodes are nullid | |||||
| if p1 < p2: | |||||
| a = p1 | |||||
| b = p2 | |||||
| else: | |||||
| a = p2 | |||||
| b = p1 | |||||
| s = hashlib.sha1(a) | |||||
| s.update(b) | |||||
| s.update(text) | |||||
| return s.digest() | |||||
| @attr.s(slots=True, frozen=True) | @attr.s(slots=True, frozen=True) | ||||
| class _revisioninfo(object): | class _revisioninfo(object): | ||||
| """Information about a revision that allows building its fulltext | """Information about a revision that allows building its fulltext | ||||
| node: expected hash of the revision | node: expected hash of the revision | ||||
| p1, p2: parent revs of the revision | p1, p2: parent revs of the revision | ||||
| btext: built text cache consisting of a one-element list | btext: built text cache consisting of a one-element list | ||||
| cachedelta: (baserev, uncompressed_delta) or None | cachedelta: (baserev, uncompressed_delta) or None | ||||
| flags: flags associated to the revision storage | flags: flags associated to the revision storage | ||||
| return disambiguate(hexnode, length) | return disambiguate(hexnode, length) | ||||
| def cmp(self, node, text): | def cmp(self, node, text): | ||||
| """compare text with a given file revision | """compare text with a given file revision | ||||
| returns True if text is different than what is stored. | returns True if text is different than what is stored. | ||||
| """ | """ | ||||
| p1, p2 = self.parents(node) | p1, p2 = self.parents(node) | ||||
| return hash(text, p1, p2) != node | return storageutil.hashrevisionsha1(text, p1, p2) != node | ||||
| def _cachesegment(self, offset, data): | def _cachesegment(self, offset, data): | ||||
| """Add a segment to the revlog cache. | """Add a segment to the revlog cache. | ||||
| Accepts an absolute offset and the data that is at that location. | Accepts an absolute offset and the data that is at that location. | ||||
| """ | """ | ||||
| o, d = self._chunkcache | o, d = self._chunkcache | ||||
| # try to add to existing cache | # try to add to existing cache | ||||
| return text | return text | ||||
| def hash(self, text, p1, p2): | def hash(self, text, p1, p2): | ||||
| """Compute a node hash. | """Compute a node hash. | ||||
| Available as a function so that subclasses can replace the hash | Available as a function so that subclasses can replace the hash | ||||
| as needed. | as needed. | ||||
| """ | """ | ||||
| return hash(text, p1, p2) | return storageutil.hashrevisionsha1(text, p1, p2) | ||||
| def _processflags(self, text, flags, operation, raw=False): | def _processflags(self, text, flags, operation, raw=False): | ||||
| """Inspect revision data flags and applies transforms defined by | """Inspect revision data flags and applies transforms defined by | ||||
| registered flag processors. | registered flag processors. | ||||
| ``text`` - the revision data to process | ``text`` - the revision data to process | ||||
| ``flags`` - the revision flags | ``flags`` - the revision flags | ||||
| ``operation`` - the operation being performed (read or write) | ``operation`` - the operation being performed (read or write) | ||||
| # storageutil.py - Storage functionality agnostic of backend implementation. | |||||
| # | |||||
| # Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com> | |||||
| # | |||||
| # This software may be used and distributed according to the terms of the | |||||
| # GNU General Public License version 2 or any later version. | |||||
| from __future__ import absolute_import | |||||
| import hashlib | |||||
| from ..node import ( | |||||
| nullid, | |||||
| ) | |||||
| _nullhash = hashlib.sha1(nullid) | |||||
| def hashrevisionsha1(text, p1, p2): | |||||
| """Compute the SHA-1 for revision data and its parents. | |||||
| This hash combines both the current file contents and its history | |||||
| in a manner that makes it easy to distinguish nodes with the same | |||||
| content in the revision graph. | |||||
| """ | |||||
| # As of now, if one of the parent node is null, p2 is null | |||||
| if p2 == nullid: | |||||
| # deep copy of a hash is faster than creating one | |||||
| s = _nullhash.copy() | |||||
| s.update(p1) | |||||
| else: | |||||
| # none of the parent nodes are nullid | |||||
| if p1 < p2: | |||||
| a = p1 | |||||
| b = p2 | |||||
| else: | |||||
| a = p2 | |||||
| b = p1 | |||||
| s = hashlib.sha1(a) | |||||
| s.update(b) | |||||
| s.update(text) | |||||
| return s.digest() | |||||
| pycompat, | pycompat, | ||||
| repository, | repository, | ||||
| revlog, | revlog, | ||||
| store, | store, | ||||
| verify, | verify, | ||||
| ) | ) | ||||
| from mercurial.utils import ( | from mercurial.utils import ( | ||||
| interfaceutil, | interfaceutil, | ||||
| storageutil, | |||||
| ) | ) | ||||
| # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for | # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for | ||||
| # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should | # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should | ||||
| # be specifying the version(s) of Mercurial they are tested with, or | # be specifying the version(s) of Mercurial they are tested with, or | ||||
| # leave the attribute unspecified. | # leave the attribute unspecified. | ||||
| testedwith = 'ships-with-hg-core' | testedwith = 'ships-with-hg-core' | ||||
| text, vhash = writetransform(self, text) | text, vhash = writetransform(self, text) | ||||
| validatehash = validatehash and vhash | validatehash = validatehash and vhash | ||||
| return text, validatehash | return text, validatehash | ||||
| def checkhash(self, text, node, p1=None, p2=None, rev=None): | def checkhash(self, text, node, p1=None, p2=None, rev=None): | ||||
| if p1 is None and p2 is None: | if p1 is None and p2 is None: | ||||
| p1, p2 = self.parents(node) | p1, p2 = self.parents(node) | ||||
| if node != revlog.hash(text, p1, p2): | if node != storageutil.hashrevisionsha1(text, p1, p2): | ||||
| raise simplestoreerror(_("integrity check failed on %s") % | raise simplestoreerror(_("integrity check failed on %s") % | ||||
| self._path) | self._path) | ||||
| def revision(self, node, raw=False): | def revision(self, node, raw=False): | ||||
| validatenode(node) | validatenode(node) | ||||
| if node == nullid: | if node == nullid: | ||||
| return b'' | return b'' | ||||
| t = text | t = text | ||||
| if text.startswith(b'\1\n'): | if text.startswith(b'\1\n'): | ||||
| t = b'\1\n\1\n' + text | t = b'\1\n\1\n' + text | ||||
| p1, p2 = self.parents(node) | p1, p2 = self.parents(node) | ||||
| if revlog.hash(t, p1, p2) == node: | if storageutil.hashrevisionsha1(t, p1, p2) == node: | ||||
| return False | return False | ||||
| if self.iscensored(self.rev(node)): | if self.iscensored(self.rev(node)): | ||||
| return text != b'' | return text != b'' | ||||
| if self.renamed(node): | if self.renamed(node): | ||||
| t2 = self.read(node) | t2 = self.read(node) | ||||
| return t2 != text | return t2 != text | ||||
| return self.addrevision(text, transaction, linkrev, p1, p2) | return self.addrevision(text, transaction, linkrev, p1, p2) | ||||
| def addrevision(self, text, transaction, linkrev, p1, p2, node=None, | def addrevision(self, text, transaction, linkrev, p1, p2, node=None, | ||||
| flags=revlog.REVIDX_DEFAULT_FLAGS, cachedelta=None): | flags=revlog.REVIDX_DEFAULT_FLAGS, cachedelta=None): | ||||
| validatenode(p1) | validatenode(p1) | ||||
| validatenode(p2) | validatenode(p2) | ||||
| if flags: | if flags: | ||||
| node = node or revlog.hash(text, p1, p2) | node = node or storageutil.hashrevisionsha1(text, p1, p2) | ||||
| rawtext, validatehash = self._processflags(text, flags, 'write') | rawtext, validatehash = self._processflags(text, flags, 'write') | ||||
| node = node or revlog.hash(text, p1, p2) | node = node or storageutil.hashrevisionsha1(text, p1, p2) | ||||
| if node in self._indexbynode: | if node in self._indexbynode: | ||||
| return node | return node | ||||
| if validatehash: | if validatehash: | ||||
| self.checkhash(rawtext, node, p1=p1, p2=p2) | self.checkhash(rawtext, node, p1=p1, p2=p2) | ||||
| return self._addrawrevision(node, rawtext, transaction, linkrev, p1, p2, | return self._addrawrevision(node, rawtext, transaction, linkrev, p1, p2, | ||||