This is an archive of the discontinued Mercurial Phabricator instance.

Differential D6816

flagprocessors: writetransform function take side data as parameter (API)
ClosedPublic

Authored by marmoute on Sep 7 2019, 5:28 AM.

Details

Reviewers
yuja
durin42
indygreg
Group Reviewers
hg-reviewers
Commits
rHGf4caf910669e: flagprocessors: writetransform function take side data as parameter (API)
Summary

If we want some flag processors to be able to store sidedata it needs to be
actually fed that data.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Revision Contents
Changeset List

Diff 16482

hgext/lfs/wrapper.py

if k.startswith('x-hg-'): if k.startswith('x-hg-'):
name = k[len('x-hg-'):] name = k[len('x-hg-'):]
hgmeta[name] = p[k] hgmeta[name] = p[k]
if hgmeta or text.startswith('\1\n'): if hgmeta or text.startswith('\1\n'):
text = storageutil.packmeta(hgmeta, text) text = storageutil.packmeta(hgmeta, text)
return (text, True, {}) return (text, True, {})
def writetostore(self, text): def writetostore(self, text, sidedata):
# hg filelog metadata (includes rename, etc) # hg filelog metadata (includes rename, etc)
hgmeta, offset = storageutil.parsemeta(text) hgmeta, offset = storageutil.parsemeta(text)
if offset and offset > 0: if offset and offset > 0:
# lfs blob does not contain hg filelog metadata # lfs blob does not contain hg filelog metadata
text = text[offset:] text = text[offset:]
# git-lfs only supports sha256 # git-lfs only supports sha256
oid = hex(hashlib.sha256(text).digest()) oid = hex(hashlib.sha256(text).digest())

mercurial/revlog.py

# max size of revlog with inline data # max size of revlog with inline data
_maxinline = 131072 _maxinline = 131072
_chunksize = 1048576 _chunksize = 1048576
# Flag processors for REVIDX_ELLIPSIS. # Flag processors for REVIDX_ELLIPSIS.
def ellipsisreadprocessor(rl, text): def ellipsisreadprocessor(rl, text):
return text, False, {} return text, False, {}
def ellipsiswriteprocessor(rl, text): def ellipsiswriteprocessor(rl, text, sidedata):
return text, False return text, False
def ellipsisrawprocessor(rl, text): def ellipsisrawprocessor(rl, text):
return False return False
ellipsisprocessor = ( ellipsisprocessor = (
ellipsisreadprocessor, ellipsisreadprocessor,
ellipsiswriteprocessor, ellipsiswriteprocessor,

mercurial/revlogutils/flagutil.py

``operation`` is 'write') defined by REVIDX_FLAGS_ORDER, applying the ``operation`` is 'write') defined by REVIDX_FLAGS_ORDER, applying the
flag processors registered for present flags. The order of flags defined flag processors registered for present flags. The order of flags defined
in REVIDX_FLAGS_ORDER needs to be stable to allow non-commutativity. in REVIDX_FLAGS_ORDER needs to be stable to allow non-commutativity.
Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the
processed text and ``validatehash`` is a bool indicating whether the processed text and ``validatehash`` is a bool indicating whether the
returned text should be checked for hash integrity. returned text should be checked for hash integrity.
""" """
assert not sidedata # XXX until it is actually processed return self._processflagsfunc(text, flags, 'write',
return self._processflagsfunc(text, flags, 'write')[:2] sidedata=sidedata)[:2]
def _processflagsraw(self, text, flags): def _processflagsraw(self, text, flags):
"""Inspect revision data flags to check is the content hash should be """Inspect revision data flags to check is the content hash should be
validated. validated.
``text`` - the revision data to process ``text`` - the revision data to process
``flags`` - the revision flags ``flags`` - the revision flags
This method processes the flags in the order (or reverse order if This method processes the flags in the order (or reverse order if
``operation`` is 'write') defined by REVIDX_FLAGS_ORDER, applying the ``operation`` is 'write') defined by REVIDX_FLAGS_ORDER, applying the
flag processors registered for present flags. The order of flags defined flag processors registered for present flags. The order of flags defined
in REVIDX_FLAGS_ORDER needs to be stable to allow non-commutativity. in REVIDX_FLAGS_ORDER needs to be stable to allow non-commutativity.
Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the
processed text and ``validatehash`` is a bool indicating whether the processed text and ``validatehash`` is a bool indicating whether the
returned text should be checked for hash integrity. returned text should be checked for hash integrity.
""" """
return self._processflagsfunc(text, flags, 'raw')[1] return self._processflagsfunc(text, flags, 'raw')[1]
def _processflagsfunc(self, text, flags, operation): def _processflagsfunc(self, text, flags, operation, sidedata=None):
# fast path: no flag processors will run # fast path: no flag processors will run
if flags == 0: if flags == 0:
return text, True, {} return text, True, {}
if operation not in ('read', 'write', 'raw'): if operation not in ('read', 'write', 'raw'):
raise error.ProgrammingError(_("invalid '%s' operation") % raise error.ProgrammingError(_("invalid '%s' operation") %
operation) operation)
# Check all flags are known. # Check all flags are known.
if flags & ~REVIDX_KNOWN_FLAGS: if flags & ~REVIDX_KNOWN_FLAGS:
readtransform, writetransform, rawtransform = processor readtransform, writetransform, rawtransform = processor
if operation == 'raw': if operation == 'raw':
vhash = rawtransform(self, text) vhash = rawtransform(self, text)
elif operation == 'read': elif operation == 'read':
text, vhash, s = readtransform(self, text) text, vhash, s = readtransform(self, text)
outsidedata.update(s) outsidedata.update(s)
else: # write operation else: # write operation
text, vhash = writetransform(self, text) text, vhash = writetransform(self, text, sidedata)
validatehash = validatehash and vhash validatehash = validatehash and vhash
return text, validatehash, outsidedata return text, validatehash, outsidedata

tests/flagprocessorext.py

REVIDX_FAIL = 1 REVIDX_FAIL = 1
def validatehash(self, text): def validatehash(self, text):
return True return True
def bypass(self, text): def bypass(self, text):
return False return False
def noopdonothing(self, text): def noopdonothing(self, text, sidedata):
return (text, True) return (text, True)
def noopdonothingread(self, text): def noopdonothingread(self, text):
return (text, True, {}) return (text, True, {})
def b64encode(self, text): def b64encode(self, text, sidedata):
return (base64.b64encode(text), False) return (base64.b64encode(text), False)
def b64decode(self, text): def b64decode(self, text):
return (base64.b64decode(text), True, {}) return (base64.b64decode(text), True, {})
def gzipcompress(self, text): def gzipcompress(self, text, sidedata):
return (zlib.compress(text), False) return (zlib.compress(text), False)
def gzipdecompress(self, text): def gzipdecompress(self, text):
return (zlib.decompress(text), True, {}) return (zlib.decompress(text), True, {})
def supportedoutgoingversions(orig, repo): def supportedoutgoingversions(orig, repo):
versions = orig(repo) versions = orig(repo)
versions.discard(b'01') versions.discard(b'01')

tests/test-revlog-raw.py

# deltas. # deltas.
_extheader = b'E\n' _extheader = b'E\n'
def readprocessor(self, rawtext): def readprocessor(self, rawtext):
# True: the returned text could be used to verify hash # True: the returned text could be used to verify hash
text = rawtext[len(_extheader):].replace(b'i', b'1') text = rawtext[len(_extheader):].replace(b'i', b'1')
return text, True, {} return text, True, {}
def writeprocessor(self, text): def writeprocessor(self, text, sidedata):
# False: the returned rawtext shouldn't be used to verify hash # False: the returned rawtext shouldn't be used to verify hash
rawtext = _extheader + text.replace(b'1', b'i') rawtext = _extheader + text.replace(b'1', b'i')
return rawtext, False return rawtext, False
def rawprocessor(self, rawtext): def rawprocessor(self, rawtext):
# False: do not verify hash. Only the content returned by "readprocessor" # False: do not verify hash. Only the content returned by "readprocessor"
# can be used to verify hash. # can be used to verify hash.
return False return False
if isempty: if isempty:
text = b'' text = b''
else: else:
text = gentext(i) text = gentext(i)
rev = appendrev(rlog, text, tr, isext=isext, isdelta=isdelta) rev = appendrev(rlog, text, tr, isext=isext, isdelta=isdelta)
# Verify text, rawtext, and rawsize # Verify text, rawtext, and rawsize
if isext: if isext:
rawtext = writeprocessor(None, text)[0] rawtext = writeprocessor(None, text, {})[0]
else: else:
rawtext = text rawtext = text
if rlog.rawsize(rev) != len(rawtext): if rlog.rawsize(rev) != len(rawtext):
abort('rev %d: wrong rawsize' % rev) abort('rev %d: wrong rawsize' % rev)
if rlog.revision(rev, raw=False) != text: if rlog.revision(rev, raw=False) != text:
abort('rev %d: wrong text' % rev) abort('rev %d: wrong text' % rev)
if rlog.rawdata(rev) != rawtext: if rlog.rawdata(rev) != rawtext:
abort('rev %d: wrong rawtext' % rev) abort('rev %d: wrong rawtext' % rev)