This is an archive of the discontinued Mercurial Phabricator instance.

Differential D4656

revlog: move censor logic out of censor extension
ClosedPublic

Authored by indygreg on Sep 18 2018, 10:34 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHGa6b3c4c1019f: revlog: move censor logic out of censor extension
Summary

The censor extension is doing very low-level things with revlogs.
It is fundamentally impossible for this logic to remain in the censor
extension while support multiple storage backends: we need each
storage backend to implement censor in its own storage-specific
way.

This commit effectively moves the revlog-specific censoring code to
be a method of revlogs themselves.

We've defined a new API on the file storage interface for censoring
an individual node. Even though the current censoring code doesn't
use it, the API requires a transaction instance because it logically
makes sense for storage backends to require an active transaction
(which implies a held write lock) in order to rewrite storage.

After this commit, the censor extension has been reduced to
boilerplate precondition checking before invoking the generic
storage API.

I tried to keep the code as similar as possible. But some minor
changes were made:

  • We use self._io instead of instantiating a new revlogio instance.
  • We compare self.version against REVLOGV0 instead of != REVLOGV1 because presumably all future revlog versions will support censoring.
  • We use self.opener instead of going through repo.svfs (we don't have a handle on the repo instance from a revlog).
  • "revlog" dropped
  • Replace "flog" with "self".

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

PathPackages
Mhgext/censor.py (92 lines)
Mmercurial/filelog.py (3 lines)
Mmercurial/repository.py (17 lines)
Mmercurial/revlog.py (89 lines)

Diff 11282

hgext/censor.py

from __future__ import absolute_import from __future__ import absolute_import
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial.node import short from mercurial.node import short
from mercurial import ( from mercurial import (
error, error,
pycompat,
registrar, registrar,
revlog,
scmutil, scmutil,
util,
) )
cmdtable = {} cmdtable = {}
command = registrar.command(cmdtable) command = registrar.command(cmdtable)
# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
# be specifying the version(s) of Mercurial they are tested with, or # be specifying the version(s) of Mercurial they are tested with, or
# leave the attribute unspecified. # leave the attribute unspecified.
raise error.Abort(_('cannot censor file in heads (%s)') % headlist, raise error.Abort(_('cannot censor file in heads (%s)') % headlist,
hint=_('clean/delete and commit first')) hint=_('clean/delete and commit first'))
wp = wctx.parents() wp = wctx.parents()
if ctx.node() in [p.node() for p in wp]: if ctx.node() in [p.node() for p in wp]:
raise error.Abort(_('cannot censor working directory'), raise error.Abort(_('cannot censor working directory'),
hint=_('clean/delete/update first')) hint=_('clean/delete/update first'))
flogv = flog.version & 0xFFFF with repo.transaction(b'censor') as tr:
if flogv != revlog.REVLOGV1: flog.censorrevision(tr, fnode, tombstone=tombstone)
raise error.Abort(
_('censor does not support revlog version %d') % (flogv,))
tombstone = revlog.packmeta({"censored": tombstone}, "")
crev = fctx.filerev()
if len(tombstone) > flog.rawsize(crev):
raise error.Abort(_(
'censor tombstone must be no longer than censored data'))
# Using two files instead of one makes it easy to rewrite entry-by-entry
idxread = repo.svfs(flog.indexfile, 'r')
idxwrite = repo.svfs(flog.indexfile, 'wb', atomictemp=True)
if flog.version & revlog.FLAG_INLINE_DATA:
dataread, datawrite = idxread, idxwrite
else:
dataread = repo.svfs(flog.datafile, 'r')
datawrite = repo.svfs(flog.datafile, 'wb', atomictemp=True)
# Copy all revlog data up to the entry to be censored.
rio = revlog.revlogio()
offset = flog.start(crev)
for chunk in util.filechunkiter(idxread, limit=crev * rio.size):
idxwrite.write(chunk)
for chunk in util.filechunkiter(dataread, limit=offset):
datawrite.write(chunk)
def rewriteindex(r, newoffs, newdata=None):
"""Rewrite the index entry with a new data offset and optional new data.
The newdata argument, if given, is a tuple of three positive integers:
(new compressed, new uncompressed, added flag bits).
"""
offlags, comp, uncomp, base, link, p1, p2, nodeid = flog.index[r]
flags = revlog.gettype(offlags)
if newdata:
comp, uncomp, nflags = newdata
flags |= nflags
offlags = revlog.offset_type(newoffs, flags)
e = (offlags, comp, uncomp, r, link, p1, p2, nodeid)
idxwrite.write(rio.packentry(e, None, flog.version, r))
idxread.seek(rio.size, 1)
def rewrite(r, offs, data, nflags=revlog.REVIDX_DEFAULT_FLAGS):
"""Write the given full text to the filelog with the given data offset.
Returns:
The integer number of data bytes written, for tracking data offsets.
"""
flag, compdata = flog.compress(data)
newcomp = len(flag) + len(compdata)
rewriteindex(r, offs, (newcomp, len(data), nflags))
datawrite.write(flag)
datawrite.write(compdata)
dataread.seek(flog.length(r), 1)
return newcomp
# Rewrite censored revlog entry with (padded) tombstone data.
pad = ' ' * (flog.rawsize(crev) - len(tombstone))
offset += rewrite(crev, offset, tombstone + pad, revlog.REVIDX_ISCENSORED)
# Rewrite all following filelog revisions fixing up offsets and deltas.
for srev in pycompat.xrange(crev + 1, len(flog)):
if crev in flog.parentrevs(srev):
# Immediate children of censored node must be re-added as fulltext.
try:
revdata = flog.revision(srev)
except error.CensoredNodeError as e:
revdata = e.tombstone
dlen = rewrite(srev, offset, revdata)
else:
# Copy any other revision data verbatim after fixing up the offset.
rewriteindex(srev, offset)
dlen = flog.length(srev)
for chunk in util.filechunkiter(dataread, limit=dlen):
datawrite.write(chunk)
offset += dlen
idxread.close()
idxwrite.close()
if dataread is not idxread:
dataread.close()
datawrite.close()

mercurial/filelog.py

addrevisioncb=addrevisioncb) addrevisioncb=addrevisioncb)
def getstrippoint(self, minlink): def getstrippoint(self, minlink):
return self._revlog.getstrippoint(minlink) return self._revlog.getstrippoint(minlink)
def strip(self, minlink, transaction): def strip(self, minlink, transaction):
return self._revlog.strip(minlink, transaction) return self._revlog.strip(minlink, transaction)
def censorrevision(self, tr, node, tombstone=b''):
return self._revlog.censorrevision(node, tombstone=tombstone)
def files(self): def files(self):
return self._revlog.files() return self._revlog.files()
def checksize(self): def checksize(self):
return self._revlog.checksize() return self._revlog.checksize()
def read(self, node): def read(self, node):
t = self.revision(node) t = self.revision(node)

mercurial/repository.py

and the delta used as the fulltext. and the delta used as the fulltext.
``addrevisioncb`` should be called for each node as it is committed. ``addrevisioncb`` should be called for each node as it is committed.
Returns a list of nodes that were processed. A node will be in the list Returns a list of nodes that were processed. A node will be in the list
even if it existed in the store previously. even if it existed in the store previously.
""" """
def censorrevision(tr, node, tombstone=b''):
"""Remove the content of a single revision.
The specified ``node`` will have its content purged from storage.
Future attempts to access the revision data for this node will
result in failure.
A ``tombstone`` message can optionally be stored. This message may be
displayed to users when they attempt to access the missing revision
data.
Storage backends may have stored deltas against the previous content
in this revision. As part of censoring a revision, these storage
backends are expected to rewrite any internally stored deltas such
that they no longer reference the deleted content.
"""
def getstrippoint(minlink): def getstrippoint(minlink):
"""Find the minimum revision that must be stripped to strip a linkrev. """Find the minimum revision that must be stripped to strip a linkrev.
Returns a 2-tuple containing the minimum revision number and a set Returns a 2-tuple containing the minimum revision number and a set
of all revisions numbers that would be broken by this strip. of all revisions numbers that would be broken by this strip.
TODO this is highly revlog centric and should be abstracted into TODO this is highly revlog centric and should be abstracted into
a higher-level deletion API. ``repair.strip()`` relies on this. a higher-level deletion API. ``repair.strip()`` relies on this.

mercurial/revlog.py

dfh.close() dfh.close()
ifh.close() ifh.close()
if addrevisioncb: if addrevisioncb:
addrevisioncb(self, rev, node) addrevisioncb(self, rev, node)
finally: finally:
destrevlog._lazydeltabase = oldlazydeltabase destrevlog._lazydeltabase = oldlazydeltabase
destrevlog._deltabothparents = oldamd destrevlog._deltabothparents = oldamd
def censorrevision(self, node, tombstone=b''):
if (self.version & 0xFFFF) == REVLOGV0:
raise error.RevlogError(_('cannot censor with version %d revlogs') %
self.version)
rev = self.rev(node)
tombstone = packmeta({b'censored': tombstone}, b'')
if len(tombstone) > self.rawsize(rev):
raise error.Abort(_('censor tombstone must be no longer than '
'censored data'))
# Using two files instead of one makes it easy to rewrite entry-by-entry
idxread = self.opener(self.indexfile, 'r')
idxwrite = self.opener(self.indexfile, 'wb', atomictemp=True)
if self.version & FLAG_INLINE_DATA:
dataread, datawrite = idxread, idxwrite
else:
dataread = self.opener(self.datafile, 'r')
datawrite = self.opener(self.datafile, 'wb', atomictemp=True)
# Copy all revlog data up to the entry to be censored.
offset = self.start(rev)
for chunk in util.filechunkiter(idxread, limit=rev * self._io.size):
idxwrite.write(chunk)
for chunk in util.filechunkiter(dataread, limit=offset):
datawrite.write(chunk)
def rewriteindex(r, newoffs, newdata=None):
"""Rewrite the index entry with a new data offset and new data.
The newdata argument, if given, is a tuple of three positive
integers: (new compressed, new uncompressed, added flag bits).
"""
offlags, comp, uncomp, base, link, p1, p2, nodeid = self.index[r]
flags = gettype(offlags)
if newdata:
comp, uncomp, nflags = newdata
flags |= nflags
offlags = offset_type(newoffs, flags)
e = (offlags, comp, uncomp, r, link, p1, p2, nodeid)
idxwrite.write(self._io.packentry(e, None, self.version, r))
idxread.seek(self._io.size, 1)
def rewrite(r, offs, data, nflags=REVIDX_DEFAULT_FLAGS):
"""Write the given fulltext with the given data offset.
Returns:
The integer number of data bytes written, for tracking data
offsets.
"""
flag, compdata = self.compress(data)
newcomp = len(flag) + len(compdata)
rewriteindex(r, offs, (newcomp, len(data), nflags))
datawrite.write(flag)
datawrite.write(compdata)
dataread.seek(self.length(r), 1)
return newcomp
# Rewrite censored entry with (padded) tombstone data.
pad = ' ' * (self.rawsize(rev) - len(tombstone))
offset += rewrite(rev, offset, tombstone + pad, REVIDX_ISCENSORED)
# Rewrite all following filelog revisions fixing up offsets and deltas.
for srev in pycompat.xrange(rev + 1, len(self)):
if rev in self.parentrevs(srev):
# Immediate children of censored node must be re-added as
# fulltext.
try:
revdata = self.revision(srev)
except error.CensoredNodeError as e:
revdata = e.tombstone
dlen = rewrite(srev, offset, revdata)
else:
# Copy any other revision data verbatim after fixing up the
# offset.
rewriteindex(srev, offset)
dlen = self.length(srev)
for chunk in util.filechunkiter(dataread, limit=dlen):
datawrite.write(chunk)
offset += dlen
idxread.close()
idxwrite.close()
if dataread is not idxread:
dataread.close()
datawrite.close()