( )⚙ D4753 storageutil: new module for storage primitives (API)

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D4753

storageutil: new module for storage primitives (API)
ClosedPublic

Authored by indygreg on Sep 26 2018, 2:28 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHGf8eb71f9e3bd: storageutil: new module for storage primitives (API)
rHGc78e9207f46f: storageutil: new module for storage primitives (API)
Summary

There will exist common code between storage backends. It would
be nice to have a central place to put that code.

This commit attempts to create that place by creating the
"storageutil" module.

The first thing we move is revlog.hash(), which is the function for
computing the SHA-1 hash of revision fulltext and parents.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Revision Contents
Changeset List

Diff 11408

mercurial/revlog.py

and O(changes) merge between branches. and O(changes) merge between branches.
""" """
from __future__ import absolute_import from __future__ import absolute_import
import collections import collections
import contextlib import contextlib
import errno import errno
import hashlib
import os import os
import re import re
import struct import struct
import zlib import zlib
# import stuff from node for others to import from revlog # import stuff from node for others to import from revlog
from .node import ( from .node import (
bin, bin,
templatefilters, templatefilters,
util, util,
) )
from .revlogutils import ( from .revlogutils import (
deltas as deltautil, deltas as deltautil,
) )
from .utils import ( from .utils import (
interfaceutil, interfaceutil,
storageutil,
stringutil, stringutil,
) )
# blanked usage of all the name to prevent pyflakes constraints # blanked usage of all the name to prevent pyflakes constraints
# We need these name available in the module for extensions. # We need these name available in the module for extensions.
REVLOGV0 REVLOGV0
REVLOGV1 REVLOGV1
REVLOGV2 REVLOGV2
def gettype(q): def gettype(q):
return int(q & 0xFFFF) return int(q & 0xFFFF)
def offset_type(offset, type): def offset_type(offset, type):
if (type & ~REVIDX_KNOWN_FLAGS) != 0: if (type & ~REVIDX_KNOWN_FLAGS) != 0:
raise ValueError('unknown revlog index flags') raise ValueError('unknown revlog index flags')
return int(int(offset) << 16 | type) return int(int(offset) << 16 | type)
_nullhash = hashlib.sha1(nullid)
def hash(text, p1, p2):
"""generate a hash from the given text and its parent hashes
This hash combines both the current file contents and its history
in a manner that makes it easy to distinguish nodes with the same
content in the revision graph.
"""
# As of now, if one of the parent node is null, p2 is null
if p2 == nullid:
# deep copy of a hash is faster than creating one
s = _nullhash.copy()
s.update(p1)
else:
# none of the parent nodes are nullid
if p1 < p2:
a = p1
b = p2
else:
a = p2
b = p1
s = hashlib.sha1(a)
s.update(b)
s.update(text)
return s.digest()
@attr.s(slots=True, frozen=True) @attr.s(slots=True, frozen=True)
class _revisioninfo(object): class _revisioninfo(object):
"""Information about a revision that allows building its fulltext """Information about a revision that allows building its fulltext
node: expected hash of the revision node: expected hash of the revision
p1, p2: parent revs of the revision p1, p2: parent revs of the revision
btext: built text cache consisting of a one-element list btext: built text cache consisting of a one-element list
cachedelta: (baserev, uncompressed_delta) or None cachedelta: (baserev, uncompressed_delta) or None
flags: flags associated to the revision storage flags: flags associated to the revision storage
return disambiguate(hexnode, length) return disambiguate(hexnode, length)
def cmp(self, node, text): def cmp(self, node, text):
"""compare text with a given file revision """compare text with a given file revision
returns True if text is different than what is stored. returns True if text is different than what is stored.
""" """
p1, p2 = self.parents(node) p1, p2 = self.parents(node)
return hash(text, p1, p2) != node return storageutil.hashrevisionsha1(text, p1, p2) != node
def _cachesegment(self, offset, data): def _cachesegment(self, offset, data):
"""Add a segment to the revlog cache. """Add a segment to the revlog cache.
Accepts an absolute offset and the data that is at that location. Accepts an absolute offset and the data that is at that location.
""" """
o, d = self._chunkcache o, d = self._chunkcache
# try to add to existing cache # try to add to existing cache
return text return text
def hash(self, text, p1, p2): def hash(self, text, p1, p2):
"""Compute a node hash. """Compute a node hash.
Available as a function so that subclasses can replace the hash Available as a function so that subclasses can replace the hash
as needed. as needed.
""" """
return hash(text, p1, p2) return storageutil.hashrevisionsha1(text, p1, p2)
def _processflags(self, text, flags, operation, raw=False): def _processflags(self, text, flags, operation, raw=False):
"""Inspect revision data flags and applies transforms defined by """Inspect revision data flags and applies transforms defined by
registered flag processors. registered flag processors.
``text`` - the revision data to process ``text`` - the revision data to process
``flags`` - the revision flags ``flags`` - the revision flags
``operation`` - the operation being performed (read or write) ``operation`` - the operation being performed (read or write)

mercurial/utils/storageutil.py

  • This file was added.
# storageutil.py - Storage functionality agnostic of backend implementation.
#
# Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
from __future__ import absolute_import
import hashlib
from ..node import (
nullid,
)
_nullhash = hashlib.sha1(nullid)
def hashrevisionsha1(text, p1, p2):
"""Compute the SHA-1 for revision data and its parents.
This hash combines both the current file contents and its history
in a manner that makes it easy to distinguish nodes with the same
content in the revision graph.
"""
# As of now, if one of the parent node is null, p2 is null
if p2 == nullid:
# deep copy of a hash is faster than creating one
s = _nullhash.copy()
s.update(p1)
else:
# none of the parent nodes are nullid
if p1 < p2:
a = p1
b = p2
else:
a = p2
b = p1
s = hashlib.sha1(a)
s.update(b)
s.update(text)
return s.digest()

tests/simplestorerepo.py

pycompat, pycompat,
repository, repository,
revlog, revlog,
store, store,
verify, verify,
) )
from mercurial.utils import ( from mercurial.utils import (
interfaceutil, interfaceutil,
storageutil,
) )
# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
# be specifying the version(s) of Mercurial they are tested with, or # be specifying the version(s) of Mercurial they are tested with, or
# leave the attribute unspecified. # leave the attribute unspecified.
testedwith = 'ships-with-hg-core' testedwith = 'ships-with-hg-core'
text, vhash = writetransform(self, text) text, vhash = writetransform(self, text)
validatehash = validatehash and vhash validatehash = validatehash and vhash
return text, validatehash return text, validatehash
def checkhash(self, text, node, p1=None, p2=None, rev=None): def checkhash(self, text, node, p1=None, p2=None, rev=None):
if p1 is None and p2 is None: if p1 is None and p2 is None:
p1, p2 = self.parents(node) p1, p2 = self.parents(node)
if node != revlog.hash(text, p1, p2): if node != storageutil.hashrevisionsha1(text, p1, p2):
raise simplestoreerror(_("integrity check failed on %s") % raise simplestoreerror(_("integrity check failed on %s") %
self._path) self._path)
def revision(self, node, raw=False): def revision(self, node, raw=False):
validatenode(node) validatenode(node)
if node == nullid: if node == nullid:
return b'' return b''
t = text t = text
if text.startswith(b'\1\n'): if text.startswith(b'\1\n'):
t = b'\1\n\1\n' + text t = b'\1\n\1\n' + text
p1, p2 = self.parents(node) p1, p2 = self.parents(node)
if revlog.hash(t, p1, p2) == node: if storageutil.hashrevisionsha1(t, p1, p2) == node:
return False return False
if self.iscensored(self.rev(node)): if self.iscensored(self.rev(node)):
return text != b'' return text != b''
if self.renamed(node): if self.renamed(node):
t2 = self.read(node) t2 = self.read(node)
return t2 != text return t2 != text
return self.addrevision(text, transaction, linkrev, p1, p2) return self.addrevision(text, transaction, linkrev, p1, p2)
def addrevision(self, text, transaction, linkrev, p1, p2, node=None, def addrevision(self, text, transaction, linkrev, p1, p2, node=None,
flags=revlog.REVIDX_DEFAULT_FLAGS, cachedelta=None): flags=revlog.REVIDX_DEFAULT_FLAGS, cachedelta=None):
validatenode(p1) validatenode(p1)
validatenode(p2) validatenode(p2)
if flags: if flags:
node = node or revlog.hash(text, p1, p2) node = node or storageutil.hashrevisionsha1(text, p1, p2)
rawtext, validatehash = self._processflags(text, flags, 'write') rawtext, validatehash = self._processflags(text, flags, 'write')
node = node or revlog.hash(text, p1, p2) node = node or storageutil.hashrevisionsha1(text, p1, p2)
if node in self._indexbynode: if node in self._indexbynode:
return node return node
if validatehash: if validatehash:
self.checkhash(rawtext, node, p1=p1, p2=p2) self.checkhash(rawtext, node, p1=p1, p2=p2)
return self._addrawrevision(node, rawtext, transaction, linkrev, p1, p2, return self._addrawrevision(node, rawtext, transaction, linkrev, p1, p2,