This is an archive of the discontinued Mercurial Phabricator instance.

Differential D3150

revlog: move parsemeta() and packmeta() from filelog (API)
ClosedPublic

Authored by indygreg on Apr 5 2018, 8:51 PM.

Details

Reviewers
durin42
Group Reviewers
hg-reviewers
Commits
rHG0596d27457c6: revlog: move parsemeta() and packmeta() from filelog (API)
Summary

filelog.parsemeta() and filelog.packmeta() are used to decode
and encode metadata for file copies and censor.

An upcoming commit will move the core logic for censoring revlogs
into revlog.py. This would create a cycle between revlog.py and
filelog.py. So we move these metadata functions to revlog.py.

.. api::

filelog.parsemeta() and filelog.packmeta() have been moved to
the revlog module.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

indygreg created this revision.Apr 5 2018, 8:51 PM
indygreg updated this revision to Diff 7769.Apr 5 2018, 9:22 PM
durin42 accepted this revision.Apr 6 2018, 8:56 PM
This revision is now accepted and ready to land.Apr 6 2018, 8:56 PM
Closed by commit rHG0596d27457c6: revlog: move parsemeta() and packmeta() from filelog (API) (authored by indygreg). · Explain WhyApr 6 2018, 9:22 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mhgext/censor.py (3 lines)
Mhgext/lfs/wrapper.py (7 lines)
Mmercurial/filelog.py (26 lines)
Mmercurial/revlog.py (20 lines)
Mtests/simplestorerepo.py (5 lines)

Diff 7769

hgext/censor.py

from __future__ import absolute_import from __future__ import absolute_import
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial.node import short from mercurial.node import short
from mercurial import ( from mercurial import (
error, error,
filelog,
lock as lockmod, lock as lockmod,
registrar, registrar,
revlog, revlog,
scmutil, scmutil,
util, util,
) )
cmdtable = {} cmdtable = {}
raise error.Abort(_('cannot censor working directory'), raise error.Abort(_('cannot censor working directory'),
hint=_('clean/delete/update first')) hint=_('clean/delete/update first'))
flogv = flog.version & 0xFFFF flogv = flog.version & 0xFFFF
if flogv != revlog.REVLOGV1: if flogv != revlog.REVLOGV1:
raise error.Abort( raise error.Abort(
_('censor does not support revlog version %d') % (flogv,)) _('censor does not support revlog version %d') % (flogv,))
tombstone = filelog.packmeta({"censored": tombstone}, "") tombstone = revlog.packmeta({"censored": tombstone}, "")
crev = fctx.filerev() crev = fctx.filerev()
if len(tombstone) > flog.rawsize(crev): if len(tombstone) > flog.rawsize(crev):
raise error.Abort(_( raise error.Abort(_(
'censor tombstone must be no longer than censored data')) 'censor tombstone must be no longer than censored data'))
# Using two files instead of one makes it easy to rewrite entry-by-entry # Using two files instead of one makes it easy to rewrite entry-by-entry

hgext/lfs/wrapper.py

# wrapper.py - methods wrapping core mercurial logic # wrapper.py - methods wrapping core mercurial logic
# #
# Copyright 2017 Facebook, Inc. # Copyright 2017 Facebook, Inc.
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import hashlib import hashlib
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial.node import bin, hex, nullid, short from mercurial.node import bin, hex, nullid, short
from mercurial import ( from mercurial import (
error, error,
filelog,
revlog, revlog,
util, util,
) )
from mercurial.utils import ( from mercurial.utils import (
stringutil, stringutil,
) )
# pack hg filelog metadata # pack hg filelog metadata
hgmeta = {} hgmeta = {}
for k in p.keys(): for k in p.keys():
if k.startswith('x-hg-'): if k.startswith('x-hg-'):
name = k[len('x-hg-'):] name = k[len('x-hg-'):]
hgmeta[name] = p[k] hgmeta[name] = p[k]
if hgmeta or text.startswith('\1\n'): if hgmeta or text.startswith('\1\n'):
text = filelog.packmeta(hgmeta, text) text = revlog.packmeta(hgmeta, text)
return (text, True) return (text, True)
def writetostore(self, text): def writetostore(self, text):
# hg filelog metadata (includes rename, etc) # hg filelog metadata (includes rename, etc)
hgmeta, offset = filelog.parsemeta(text) hgmeta, offset = revlog.parsemeta(text)
if offset and offset > 0: if offset and offset > 0:
# lfs blob does not contain hg filelog metadata # lfs blob does not contain hg filelog metadata
text = text[offset:] text = text[offset:]
# git-lfs only supports sha256 # git-lfs only supports sha256
oid = hex(hashlib.sha256(text).digest()) oid = hex(hashlib.sha256(text).digest())
self.opener.lfslocalblobstore.write(oid, text) self.opener.lfslocalblobstore.write(oid, text)
flags = rlog.flags(rev) flags = rlog.flags(rev)
return bool(flags & revlog.REVIDX_EXTSTORED) return bool(flags & revlog.REVIDX_EXTSTORED)
def filelogaddrevision(orig, self, text, transaction, link, p1, p2, def filelogaddrevision(orig, self, text, transaction, link, p1, p2,
cachedelta=None, node=None, cachedelta=None, node=None,
flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds): flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds):
textlen = len(text) textlen = len(text)
# exclude hg rename meta from file size # exclude hg rename meta from file size
meta, offset = filelog.parsemeta(text) meta, offset = revlog.parsemeta(text)
if offset: if offset:
textlen -= offset textlen -= offset
lfstrack = self.opener.options['lfstrack'] lfstrack = self.opener.options['lfstrack']
if lfstrack(self.filename, textlen): if lfstrack(self.filename, textlen):
flags |= revlog.REVIDX_EXTSTORED flags |= revlog.REVIDX_EXTSTORED

mercurial/filelog.py

# filelog.py - file history class for mercurial # filelog.py - file history class for mercurial
# #
# Copyright 2005-2007 Matt Mackall <mpm@selenic.com> # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import re
import struct import struct
from .thirdparty.zope import ( from .thirdparty.zope import (
interface as zi, interface as zi,
) )
from . import ( from . import (
error, error,
mdiff, mdiff,
repository, repository,
revlog, revlog,
) )
_mdre = re.compile('\1\n')
def parsemeta(text):
"""return (metadatadict, metadatasize)"""
# text can be buffer, so we can't use .startswith or .index
if text[:2] != '\1\n':
return None, None
s = _mdre.search(text, 2).start()
mtext = text[2:s]
meta = {}
for l in mtext.splitlines():
k, v = l.split(": ", 1)
meta[k] = v
return meta, (s + 2)
def packmeta(meta, text):
keys = sorted(meta)
metatext = "".join("%s: %s\n" % (k, meta[k]) for k in keys)
return "\1\n%s\1\n%s" % (metatext, text)
def _censoredtext(text): def _censoredtext(text):
m, offs = parsemeta(text) m, offs = revlog.parsemeta(text)
return m and "censored" in m return m and "censored" in m
@zi.implementer(repository.ifilestorage) @zi.implementer(repository.ifilestorage)
class filelog(revlog.revlog): class filelog(revlog.revlog):
def __init__(self, opener, path): def __init__(self, opener, path):
super(filelog, self).__init__(opener, super(filelog, self).__init__(opener,
"/".join(("data", path + ".i"))) "/".join(("data", path + ".i")))
# full name of the user visible file, relative to the repository root # full name of the user visible file, relative to the repository root
self.filename = path self.filename = path
def read(self, node): def read(self, node):
t = self.revision(node) t = self.revision(node)
if not t.startswith('\1\n'): if not t.startswith('\1\n'):
return t return t
s = t.index('\1\n', 2) s = t.index('\1\n', 2)
return t[s + 2:] return t[s + 2:]
def add(self, text, meta, transaction, link, p1=None, p2=None): def add(self, text, meta, transaction, link, p1=None, p2=None):
if meta or text.startswith('\1\n'): if meta or text.startswith('\1\n'):
text = packmeta(meta, text) text = revlog.packmeta(meta, text)
return self.addrevision(text, transaction, link, p1, p2) return self.addrevision(text, transaction, link, p1, p2)
def renamed(self, node): def renamed(self, node):
if self.parents(node)[0] != revlog.nullid: if self.parents(node)[0] != revlog.nullid:
return False return False
t = self.revision(node) t = self.revision(node)
m = parsemeta(t)[0] m = revlog.parsemeta(t)[0]
if m and "copy" in m: if m and "copy" in m:
return (m["copy"], revlog.bin(m["copyrev"])) return (m["copy"], revlog.bin(m["copyrev"]))
return False return False
def size(self, rev): def size(self, rev):
"""return the size of a given revision""" """return the size of a given revision"""
# for revisions with renames, we have to go the slow way # for revisions with renames, we have to go the slow way

mercurial/revlog.py

from __future__ import absolute_import from __future__ import absolute_import
import collections import collections
import contextlib import contextlib
import errno import errno
import hashlib import hashlib
import heapq import heapq
import os import os
import re
import struct import struct
import zlib import zlib
# import stuff from node for others to import from revlog # import stuff from node for others to import from revlog
from .node import ( from .node import (
bin, bin,
hex, hex,
nullid, nullid,
CensoredNodeError = error.CensoredNodeError CensoredNodeError = error.CensoredNodeError
ProgrammingError = error.ProgrammingError ProgrammingError = error.ProgrammingError
# Store flag processors (cf. 'addflagprocessor()' to register) # Store flag processors (cf. 'addflagprocessor()' to register)
_flagprocessors = { _flagprocessors = {
REVIDX_ISCENSORED: None, REVIDX_ISCENSORED: None,
} }
_mdre = re.compile('\1\n')
def parsemeta(text):
"""return (metadatadict, metadatasize)"""
# text can be buffer, so we can't use .startswith or .index
if text[:2] != '\1\n':
return None, None
s = _mdre.search(text, 2).start()
mtext = text[2:s]
meta = {}
for l in mtext.splitlines():
k, v = l.split(": ", 1)
meta[k] = v
return meta, (s + 2)
def packmeta(meta, text):
keys = sorted(meta)
metatext = "".join("%s: %s\n" % (k, meta[k]) for k in keys)
return "\1\n%s\1\n%s" % (metatext, text)
def addflagprocessor(flag, processor): def addflagprocessor(flag, processor):
"""Register a flag processor on a revision data flag. """Register a flag processor on a revision data flag.
Invariant: Invariant:
- Flags need to be defined in REVIDX_KNOWN_FLAGS and REVIDX_FLAGS_ORDER, - Flags need to be defined in REVIDX_KNOWN_FLAGS and REVIDX_FLAGS_ORDER,
and REVIDX_RAWTEXT_CHANGING_FLAGS if they can alter rawtext. and REVIDX_RAWTEXT_CHANGING_FLAGS if they can alter rawtext.
- Only one flag processor can be registered on a specific flag. - Only one flag processor can be registered on a specific flag.
- flagprocessors must be 3-tuples of functions (read, write, raw) with the - flagprocessors must be 3-tuples of functions (read, write, raw) with the

tests/simplestorerepo.py

from mercurial.thirdparty.zope import ( from mercurial.thirdparty.zope import (
interface as zi, interface as zi,
) )
from mercurial import ( from mercurial import (
ancestor, ancestor,
bundlerepo, bundlerepo,
error, error,
extensions, extensions,
filelog,
localrepo, localrepo,
mdiff, mdiff,
pycompat, pycompat,
repository, repository,
revlog, revlog,
store, store,
verify, verify,
) )
def renamed(self, node): def renamed(self, node):
validatenode(node) validatenode(node)
if self.parents(node)[0] != nullid: if self.parents(node)[0] != nullid:
return False return False
fulltext = self.revision(node) fulltext = self.revision(node)
m = filelog.parsemeta(fulltext)[0] m = revlog.parsemeta(fulltext)[0]
if m and 'copy' in m: if m and 'copy' in m:
return m['copy'], bin(m['copyrev']) return m['copy'], bin(m['copyrev'])
return False return False
def cmp(self, node, text): def cmp(self, node, text):
validatenode(node) validatenode(node)
return [b'/'.join((self._storepath, f)) for f in entries] return [b'/'.join((self._storepath, f)) for f in entries]
# Required by verify. # Required by verify.
def checksize(self): def checksize(self):
return 0, 0 return 0, 0
def add(self, text, meta, transaction, linkrev, p1, p2): def add(self, text, meta, transaction, linkrev, p1, p2):
if meta or text.startswith(b'\1\n'): if meta or text.startswith(b'\1\n'):
text = filelog.packmeta(meta, text) text = revlog.packmeta(meta, text)
return self.addrevision(text, transaction, linkrev, p1, p2) return self.addrevision(text, transaction, linkrev, p1, p2)
def addrevision(self, text, transaction, linkrev, p1, p2, node=None, def addrevision(self, text, transaction, linkrev, p1, p2, node=None,
flags=revlog.REVIDX_DEFAULT_FLAGS, cachedelta=None): flags=revlog.REVIDX_DEFAULT_FLAGS, cachedelta=None):
validatenode(p1) validatenode(p1)
validatenode(p2) validatenode(p2)