This is an archive of the discontinued Mercurial Phabricator instance.

Differential D3151

revlog: move censor logic into main revlog class
ClosedPublic

Authored by indygreg on Apr 5 2018, 8:51 PM.

Details

Reviewers
durin42
Group Reviewers
hg-reviewers
Commits
rHG65250a66b55c: revlog: move censor logic into main revlog class
Summary

Previously, the revlog class implemented dummy methods for
various censor-related functionality. Revision censoring was
(and will continue to be) only possible on filelog instances.
So filelog implemented these methods to perform something
reasonable.

A problem with implementing censoring on filelog is that
it assumes filelog is a revlog. Upcoming work to formalize
the filelog interface will make this not true.

Furthermore, the censoring logic is security-sensitive. I
think action-at-a-distance with custom implementation of core
revlog APIs in derived classes is a bit dangerous. I think at
a minimum the censor logic should live in revlog.py.

I was tempted to created a "censored revlog" class that
basically pulled these methods out of filelog. But, I wasn't
a huge fan of overriding core methods in child classes. A
reason to do that would be performance. However, the censoring
code only comes into play when:

  • hash verification fails
  • delta generation
  • applying deltas from changegroups

The new code is conditional on an instance attribute. So the
overhead for running the censored code when the revlog isn't
censorable is an attribute lookup. All of these operations are
at least a magnitude slower than a Python attribute lookup. So
there shouldn't be a performance concern.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

indygreg created this revision.Apr 5 2018, 8:51 PM
durin42 accepted this revision.Apr 6 2018, 8:58 PM
This revision is now accepted and ready to land.Apr 6 2018, 8:58 PM
Closed by commit rHG65250a66b55c: revlog: move censor logic into main revlog class (authored by indygreg). · Explain WhyApr 6 2018, 9:23 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mmercurial/filelog.py (43 lines)
Mmercurial/revlog.py (56 lines)

Diff 7765

mercurial/filelog.py

# filelog.py - file history class for mercurial # filelog.py - file history class for mercurial
# #
# Copyright 2005-2007 Matt Mackall <mpm@selenic.com> # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import struct
from .thirdparty.zope import ( from .thirdparty.zope import (
interface as zi, interface as zi,
) )
from . import ( from . import (
error,
mdiff,
repository, repository,
revlog, revlog,
) )
def _censoredtext(text):
m, offs = revlog.parsemeta(text)
return m and "censored" in m
@zi.implementer(repository.ifilestorage) @zi.implementer(repository.ifilestorage)
class filelog(revlog.revlog): class filelog(revlog.revlog):
def __init__(self, opener, path): def __init__(self, opener, path):
super(filelog, self).__init__(opener, super(filelog, self).__init__(opener,
"/".join(("data", path + ".i"))) "/".join(("data", path + ".i")),
censorable=True)
# full name of the user visible file, relative to the repository root # full name of the user visible file, relative to the repository root
self.filename = path self.filename = path
def read(self, node): def read(self, node):
t = self.revision(node) t = self.revision(node)
if not t.startswith('\1\n'): if not t.startswith('\1\n'):
return t return t
s = t.index('\1\n', 2) s = t.index('\1\n', 2)
# renaming a file produces a different hash, even if the data # renaming a file produces a different hash, even if the data
# remains unchanged. Check if it's the case (slow): # remains unchanged. Check if it's the case (slow):
if self.renamed(node): if self.renamed(node):
t2 = self.read(node) t2 = self.read(node)
return t2 != text return t2 != text
return True return True
def checkhash(self, text, node, p1=None, p2=None, rev=None):
try:
super(filelog, self).checkhash(text, node, p1=p1, p2=p2, rev=rev)
except error.RevlogError:
if _censoredtext(text):
raise error.CensoredNodeError(self.indexfile, node, text)
raise
def iscensored(self, rev):
"""Check if a file revision is censored."""
return self.flags(rev) & revlog.REVIDX_ISCENSORED
def _peek_iscensored(self, baserev, delta, flush):
"""Quickly check if a delta produces a censored revision."""
# Fragile heuristic: unless new file meta keys are added alphabetically
# preceding "censored", all censored revisions are prefixed by
# "\1\ncensored:". A delta producing such a censored revision must be a
# full-replacement delta, so we inspect the first and only patch in the
# delta for this prefix.
hlen = struct.calcsize(">lll")
if len(delta) <= hlen:
return False
oldlen = self.rawsize(baserev)
newlen = len(delta) - hlen
if delta[:hlen] != mdiff.replacediffheader(oldlen, newlen):
return False
add = "\1\ncensored:"
addlen = len(add)
return newlen >= addlen and delta[hlen:hlen + addlen] == add

mercurial/revlog.py

meta[k] = v meta[k] = v
return meta, (s + 2) return meta, (s + 2)
def packmeta(meta, text): def packmeta(meta, text):
keys = sorted(meta) keys = sorted(meta)
metatext = "".join("%s: %s\n" % (k, meta[k]) for k in keys) metatext = "".join("%s: %s\n" % (k, meta[k]) for k in keys)
return "\1\n%s\1\n%s" % (metatext, text) return "\1\n%s\1\n%s" % (metatext, text)
def _censoredtext(text):
m, offs = parsemeta(text)
return m and "censored" in m
def addflagprocessor(flag, processor): def addflagprocessor(flag, processor):
"""Register a flag processor on a revision data flag. """Register a flag processor on a revision data flag.
Invariant: Invariant:
- Flags need to be defined in REVIDX_KNOWN_FLAGS and REVIDX_FLAGS_ORDER, - Flags need to be defined in REVIDX_KNOWN_FLAGS and REVIDX_FLAGS_ORDER,
and REVIDX_RAWTEXT_CHANGING_FLAGS if they can alter rawtext. and REVIDX_RAWTEXT_CHANGING_FLAGS if they can alter rawtext.
- Only one flag processor can be registered on a specific flag. - Only one flag processor can be registered on a specific flag.
- flagprocessors must be 3-tuples of functions (read, write, raw) with the - flagprocessors must be 3-tuples of functions (read, write, raw) with the
for locking while reading. for locking while reading.
If checkambig, indexfile is opened with checkambig=True at If checkambig, indexfile is opened with checkambig=True at
writing, to avoid file stat ambiguity. writing, to avoid file stat ambiguity.
If mmaplargeindex is True, and an mmapindexthreshold is set, the If mmaplargeindex is True, and an mmapindexthreshold is set, the
index will be mmapped rather than read if it is larger than the index will be mmapped rather than read if it is larger than the
configured threshold. configured threshold.
If censorable is True, the revlog can have censored revisions.
""" """
def __init__(self, opener, indexfile, datafile=None, checkambig=False, def __init__(self, opener, indexfile, datafile=None, checkambig=False,
mmaplargeindex=False): mmaplargeindex=False, censorable=False):
""" """
create a revlog object create a revlog object
opener is a function that abstracts the file opening operation opener is a function that abstracts the file opening operation
and can be used to implement COW semantics or the like. and can be used to implement COW semantics or the like.
""" """
self.indexfile = indexfile self.indexfile = indexfile
self.datafile = datafile or (indexfile[:-2] + ".d") self.datafile = datafile or (indexfile[:-2] + ".d")
self.opener = opener self.opener = opener
# When True, indexfile is opened with checkambig=True at writing, to # When True, indexfile is opened with checkambig=True at writing, to
# avoid file stat ambiguity. # avoid file stat ambiguity.
self._checkambig = checkambig self._checkambig = checkambig
self._censorable = censorable
# 3-tuple of (node, rev, text) for a raw revision. # 3-tuple of (node, rev, text) for a raw revision.
self._cache = None self._cache = None
# Maps rev to chain base rev. # Maps rev to chain base rev.
self._chainbasecache = util.lrucachedict(100) self._chainbasecache = util.lrucachedict(100)
# 2-tuple of (offset, data) of raw data from the revlog at an offset. # 2-tuple of (offset, data) of raw data from the revlog at an offset.
self._chunkcache = (0, '') self._chunkcache = (0, '')
# How much data to read and cache into the raw revlog data cache. # How much data to read and cache into the raw revlog data cache.
self._chunkcachesize = 65536 self._chunkcachesize = 65536
return text, validatehash return text, validatehash
def checkhash(self, text, node, p1=None, p2=None, rev=None): def checkhash(self, text, node, p1=None, p2=None, rev=None):
"""Check node hash integrity. """Check node hash integrity.
Available as a function so that subclasses can extend hash mismatch Available as a function so that subclasses can extend hash mismatch
behaviors as needed. behaviors as needed.
""" """
try:
if p1 is None and p2 is None: if p1 is None and p2 is None:
p1, p2 = self.parents(node) p1, p2 = self.parents(node)
if node != self.hash(text, p1, p2): if node != self.hash(text, p1, p2):
revornode = rev revornode = rev
if revornode is None: if revornode is None:
revornode = templatefilters.short(hex(node)) revornode = templatefilters.short(hex(node))
raise RevlogError(_("integrity check failed on %s:%s") raise RevlogError(_("integrity check failed on %s:%s")
% (self.indexfile, pycompat.bytestr(revornode))) % (self.indexfile, pycompat.bytestr(revornode)))
except RevlogError:
if self._censorable and _censoredtext(text):
raise error.CensoredNodeError(self.indexfile, node, text)
raise
def _enforceinlinesize(self, tr, fp=None): def _enforceinlinesize(self, tr, fp=None):
"""Check if the revlog is too big for inline and convert if so. """Check if the revlog is too big for inline and convert if so.
This should be called after revisions are added to the revlog. If the This should be called after revisions are added to the revlog. If the
revlog has grown too large to be an inline revlog, it will convert it revlog has grown too large to be an inline revlog, it will convert it
to use multiple index and data files. to use multiple index and data files.
""" """
if dfh: if dfh:
dfh.close() dfh.close()
ifh.close() ifh.close()
return nodes return nodes
def iscensored(self, rev): def iscensored(self, rev):
"""Check if a file revision is censored.""" """Check if a file revision is censored."""
if not self._censorable:
return False return False
return self.flags(rev) & REVIDX_ISCENSORED
def _peek_iscensored(self, baserev, delta, flush): def _peek_iscensored(self, baserev, delta, flush):
"""Quickly check if a delta produces a censored revision.""" """Quickly check if a delta produces a censored revision."""
if not self._censorable:
return False return False
# Fragile heuristic: unless new file meta keys are added alphabetically
# preceding "censored", all censored revisions are prefixed by
# "\1\ncensored:". A delta producing such a censored revision must be a
# full-replacement delta, so we inspect the first and only patch in the
# delta for this prefix.
hlen = struct.calcsize(">lll")
if len(delta) <= hlen:
return False
oldlen = self.rawsize(baserev)
newlen = len(delta) - hlen
if delta[:hlen] != mdiff.replacediffheader(oldlen, newlen):
return False
add = "\1\ncensored:"
addlen = len(add)
return newlen >= addlen and delta[hlen:hlen + addlen] == add
def getstrippoint(self, minlink): def getstrippoint(self, minlink):
"""find the minimum rev that must be stripped to strip the linkrev """find the minimum rev that must be stripped to strip the linkrev
Returns a tuple containing the minimum rev and a set of all revs that Returns a tuple containing the minimum rev and a set of all revs that
have linkrevs that will be broken by this strip. have linkrevs that will be broken by this strip.
""" """
brokenrevs = set() brokenrevs = set()
strippoint = len(self) strippoint = len(self)