This is an archive of the discontinued Mercurial Phabricator instance.

Differential D4754

storageutil: move metadata parsing and packing from revlog (API)
ClosedPublic

Authored by indygreg on Sep 26 2018, 2:28 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG3e896b51aa5d: storageutil: move metadata parsing and packing from revlog (API)
rHGa6c5a9f36716: storageutil: move metadata parsing and packing from revlog (API)
Summary

Parsing and writing of revision text metadata is likely identical
across storage backends. Let's move the code out of revlog so we
don't need to import the revlog module in order to use it.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

indygreg created this revision.Sep 26 2018, 2:28 PM
Closed by commit rHGa6c5a9f36716: storageutil: move metadata parsing and packing from revlog (API) (authored by indygreg). · Explain WhySep 26 2018, 5:12 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 11430

hgext/lfs/wrapper.py

from mercurial import ( from mercurial import (
error, error,
repository, repository,
revlog, revlog,
util, util,
) )
from mercurial.utils import ( from mercurial.utils import (
storageutil,
stringutil, stringutil,
) )
from ..largefiles import lfutil from ..largefiles import lfutil
from . import ( from . import (
blobstore, blobstore,
pointer, pointer,
# pack hg filelog metadata # pack hg filelog metadata
hgmeta = {} hgmeta = {}
for k in p.keys(): for k in p.keys():
if k.startswith('x-hg-'): if k.startswith('x-hg-'):
name = k[len('x-hg-'):] name = k[len('x-hg-'):]
hgmeta[name] = p[k] hgmeta[name] = p[k]
if hgmeta or text.startswith('\1\n'): if hgmeta or text.startswith('\1\n'):
text = revlog.packmeta(hgmeta, text) text = storageutil.packmeta(hgmeta, text)
return (text, True) return (text, True)
def writetostore(self, text): def writetostore(self, text):
# hg filelog metadata (includes rename, etc) # hg filelog metadata (includes rename, etc)
hgmeta, offset = revlog.parsemeta(text) hgmeta, offset = storageutil.parsemeta(text)
if offset and offset > 0: if offset and offset > 0:
# lfs blob does not contain hg filelog metadata # lfs blob does not contain hg filelog metadata
text = text[offset:] text = text[offset:]
# git-lfs only supports sha256 # git-lfs only supports sha256
oid = hex(hashlib.sha256(text).digest()) oid = hex(hashlib.sha256(text).digest())
self.opener.lfslocalblobstore.write(oid, text) self.opener.lfslocalblobstore.write(oid, text)
cachedelta=None, node=None, cachedelta=None, node=None,
flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds): flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds):
# The matcher isn't available if reposetup() wasn't called. # The matcher isn't available if reposetup() wasn't called.
lfstrack = self._revlog.opener.options.get('lfstrack') lfstrack = self._revlog.opener.options.get('lfstrack')
if lfstrack: if lfstrack:
textlen = len(text) textlen = len(text)
# exclude hg rename meta from file size # exclude hg rename meta from file size
meta, offset = revlog.parsemeta(text) meta, offset = storageutil.parsemeta(text)
if offset: if offset:
textlen -= offset textlen -= offset
if lfstrack(self._revlog.filename, textlen): if lfstrack(self._revlog.filename, textlen):
flags |= revlog.REVIDX_EXTSTORED flags |= revlog.REVIDX_EXTSTORED
return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta, return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
node=node, flags=flags, **kwds) node=node, flags=flags, **kwds)

mercurial/filelog.py

# filelog.py - file history class for mercurial # filelog.py - file history class for mercurial
# #
# Copyright 2005-2007 Matt Mackall <mpm@selenic.com> # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
from . import ( from . import (
error, error,
repository, repository,
revlog, revlog,
) )
from .utils import ( from .utils import (
interfaceutil, interfaceutil,
storageutil,
) )
@interfaceutil.implementer(repository.ifilestorage) @interfaceutil.implementer(repository.ifilestorage)
class filelog(object): class filelog(object):
def __init__(self, opener, path): def __init__(self, opener, path):
self._revlog = revlog.revlog(opener, self._revlog = revlog.revlog(opener,
'/'.join(('data', path + '.i')), '/'.join(('data', path + '.i')),
censorable=True) censorable=True)
t = self.revision(node) t = self.revision(node)
if not t.startswith('\1\n'): if not t.startswith('\1\n'):
return t return t
s = t.index('\1\n', 2) s = t.index('\1\n', 2)
return t[s + 2:] return t[s + 2:]
def add(self, text, meta, transaction, link, p1=None, p2=None): def add(self, text, meta, transaction, link, p1=None, p2=None):
if meta or text.startswith('\1\n'): if meta or text.startswith('\1\n'):
text = revlog.packmeta(meta, text) text = storageutil.packmeta(meta, text)
return self.addrevision(text, transaction, link, p1, p2) return self.addrevision(text, transaction, link, p1, p2)
def renamed(self, node): def renamed(self, node):
if self.parents(node)[0] != revlog.nullid: if self.parents(node)[0] != revlog.nullid:
return False return False
t = self.revision(node) t = self.revision(node)
m = revlog.parsemeta(t)[0] m = storageutil.parsemeta(t)[0]
# copy and copyrev occur in pairs. In rare cases due to bugs, # copy and copyrev occur in pairs. In rare cases due to bugs,
# one can occur without the other. # one can occur without the other.
if m and "copy" in m and "copyrev" in m: if m and "copy" in m and "copyrev" in m:
return (m["copy"], revlog.bin(m["copyrev"])) return (m["copy"], revlog.bin(m["copyrev"]))
return False return False
def size(self, rev): def size(self, rev):
"""return the size of a given revision""" """return the size of a given revision"""

mercurial/revlog.py

""" """
from __future__ import absolute_import from __future__ import absolute_import
import collections import collections
import contextlib import contextlib
import errno import errno
import os import os
import re
import struct import struct
import zlib import zlib
# import stuff from node for others to import from revlog # import stuff from node for others to import from revlog
from .node import ( from .node import (
bin, bin,
hex, hex,
nullhex, nullhex,
return False return False
ellipsisprocessor = ( ellipsisprocessor = (
ellipsisreadprocessor, ellipsisreadprocessor,
ellipsiswriteprocessor, ellipsiswriteprocessor,
ellipsisrawprocessor, ellipsisrawprocessor,
) )
_mdre = re.compile('\1\n')
def parsemeta(text):
"""return (metadatadict, metadatasize)"""
# text can be buffer, so we can't use .startswith or .index
if text[:2] != '\1\n':
return None, None
s = _mdre.search(text, 2).start()
mtext = text[2:s]
meta = {}
for l in mtext.splitlines():
k, v = l.split(": ", 1)
meta[k] = v
return meta, (s + 2)
def packmeta(meta, text):
keys = sorted(meta)
metatext = "".join("%s: %s\n" % (k, meta[k]) for k in keys)
return "\1\n%s\1\n%s" % (metatext, text)
def _censoredtext(text): def _censoredtext(text):
m, offs = parsemeta(text) m, offs = storageutil.parsemeta(text)
return m and "censored" in m return m and "censored" in m
def addflagprocessor(flag, processor): def addflagprocessor(flag, processor):
"""Register a flag processor on a revision data flag. """Register a flag processor on a revision data flag.
Invariant: Invariant:
- Flags need to be defined in REVIDX_KNOWN_FLAGS and REVIDX_FLAGS_ORDER, - Flags need to be defined in REVIDX_KNOWN_FLAGS and REVIDX_FLAGS_ORDER,
and REVIDX_RAWTEXT_CHANGING_FLAGS if they can alter rawtext. and REVIDX_RAWTEXT_CHANGING_FLAGS if they can alter rawtext.
destrevlog._deltabothparents = oldamd destrevlog._deltabothparents = oldamd
def censorrevision(self, node, tombstone=b''): def censorrevision(self, node, tombstone=b''):
if (self.version & 0xFFFF) == REVLOGV0: if (self.version & 0xFFFF) == REVLOGV0:
raise error.RevlogError(_('cannot censor with version %d revlogs') % raise error.RevlogError(_('cannot censor with version %d revlogs') %
self.version) self.version)
rev = self.rev(node) rev = self.rev(node)
tombstone = packmeta({b'censored': tombstone}, b'') tombstone = storageutil.packmeta({b'censored': tombstone}, b'')
if len(tombstone) > self.rawsize(rev): if len(tombstone) > self.rawsize(rev):
raise error.Abort(_('censor tombstone must be no longer than ' raise error.Abort(_('censor tombstone must be no longer than '
'censored data')) 'censored data'))
# Using two files instead of one makes it easy to rewrite entry-by-entry # Using two files instead of one makes it easy to rewrite entry-by-entry
idxread = self.opener(self.indexfile, 'r') idxread = self.opener(self.indexfile, 'r')
idxwrite = self.opener(self.indexfile, 'wb', atomictemp=True) idxwrite = self.opener(self.indexfile, 'wb', atomictemp=True)

mercurial/testing/storage.py

nullid, nullid,
nullrev, nullrev,
) )
from .. import ( from .. import (
error, error,
mdiff, mdiff,
revlog, revlog,
) )
from ..utils import (
storageutil,
)
class basetestcase(unittest.TestCase): class basetestcase(unittest.TestCase):
if not getattr(unittest.TestCase, r'assertRaisesRegex', False): if not getattr(unittest.TestCase, r'assertRaisesRegex', False):
assertRaisesRegex = (# camelcase-required assertRaisesRegex = (# camelcase-required
unittest.TestCase.assertRaisesRegexp) unittest.TestCase.assertRaisesRegexp)
class ifileindextests(basetestcase): class ifileindextests(basetestcase):
"""Generic tests for the ifileindex interface. """Generic tests for the ifileindex interface.
self.assertTrue(f.cmp(node0, stored0)) self.assertTrue(f.cmp(node0, stored0))
self.assertFalse(f.cmp(node1, fulltext1)) self.assertFalse(f.cmp(node1, fulltext1))
self.assertTrue(f.cmp(node1, stored0)) self.assertTrue(f.cmp(node1, stored0))
def testcensored(self): def testcensored(self):
f = self._makefilefn() f = self._makefilefn()
stored1 = revlog.packmeta({ stored1 = storageutil.packmeta({
b'censored': b'tombstone', b'censored': b'tombstone',
}, b'') }, b'')
# TODO tests are incomplete because we need the node to be # TODO tests are incomplete because we need the node to be
# different due to presence of censor metadata. But we can't # different due to presence of censor metadata. But we can't
# do this with addrevision(). # do this with addrevision().
with self._maketransactionfn() as tr: with self._maketransactionfn() as tr:
node0 = f.add(b'foo', None, tr, 0, nullid, nullid) node0 = f.add(b'foo', None, tr, 0, nullid, nullid)

mercurial/utils/storageutil.py

# storageutil.py - Storage functionality agnostic of backend implementation. # storageutil.py - Storage functionality agnostic of backend implementation.
# #
# Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com> # Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com>
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import hashlib import hashlib
import re
from ..node import ( from ..node import (
nullid, nullid,
) )
_nullhash = hashlib.sha1(nullid) _nullhash = hashlib.sha1(nullid)
def hashrevisionsha1(text, p1, p2): def hashrevisionsha1(text, p1, p2):
b = p2 b = p2
else: else:
a = p2 a = p2
b = p1 b = p1
s = hashlib.sha1(a) s = hashlib.sha1(a)
s.update(b) s.update(b)
s.update(text) s.update(text)
return s.digest() return s.digest()
METADATA_RE = re.compile(b'\x01\n')
def parsemeta(text):
"""Parse metadata header from revision data.
Returns a 2-tuple of (metadata, offset), where both can be None if there
is no metadata.
"""
# text can be buffer, so we can't use .startswith or .index
if text[:2] != b'\x01\n':
return None, None
s = METADATA_RE.search(text, 2).start()
mtext = text[2:s]
meta = {}
for l in mtext.splitlines():
k, v = l.split(b': ', 1)
meta[k] = v
return meta, s + 2
def packmeta(meta, text):
"""Add metadata to fulltext to produce revision text."""
keys = sorted(meta)
metatext = b''.join(b'%s: %s\n' % (k, meta[k]) for k in keys)
return b'\x01\n%s\x01\n%s' % (metatext, text)

tests/simplestorerepo.py

def renamed(self, node): def renamed(self, node):
validatenode(node) validatenode(node)
if self.parents(node)[0] != nullid: if self.parents(node)[0] != nullid:
return False return False
fulltext = self.revision(node) fulltext = self.revision(node)
m = revlog.parsemeta(fulltext)[0] m = storageutil.parsemeta(fulltext)[0]
if m and 'copy' in m: if m and 'copy' in m:
return m['copy'], bin(m['copyrev']) return m['copy'], bin(m['copyrev'])
return False return False
def cmp(self, node, text): def cmp(self, node, text):
validatenode(node) validatenode(node)
# Strip out undo.backup.* files created as part of transaction # Strip out undo.backup.* files created as part of transaction
# recording. # recording.
entries = [f for f in entries if not f.startswith('undo.backup.')] entries = [f for f in entries if not f.startswith('undo.backup.')]
return [b'/'.join((self._storepath, f)) for f in entries] return [b'/'.join((self._storepath, f)) for f in entries]
def add(self, text, meta, transaction, linkrev, p1, p2): def add(self, text, meta, transaction, linkrev, p1, p2):
if meta or text.startswith(b'\1\n'): if meta or text.startswith(b'\1\n'):
text = revlog.packmeta(meta, text) text = storageutil.packmeta(meta, text)
return self.addrevision(text, transaction, linkrev, p1, p2) return self.addrevision(text, transaction, linkrev, p1, p2)
def addrevision(self, text, transaction, linkrev, p1, p2, node=None, def addrevision(self, text, transaction, linkrev, p1, p2, node=None,
flags=revlog.REVIDX_DEFAULT_FLAGS, cachedelta=None): flags=revlog.REVIDX_DEFAULT_FLAGS, cachedelta=None):
validatenode(p1) validatenode(p1)
validatenode(p2) validatenode(p2)