This is an archive of the discontinued Mercurial Phabricator instance.

Differential D3116

simplestore: correctly implement flag processors
ClosedPublic

Authored by indygreg on Apr 5 2018, 12:38 AM.

Details

Reviewers
durin42
Group Reviewers
hg-reviewers
Commits
rHG9d4f09bfe3ec: simplestore: correctly implement flag processors
Summary

There were a couple of bugs around the implementation of
flags processing with the simple store. After these changes,
test-flagprocessor.t now passes!

test-flagprocessor.t was also updated to include explicit test
coverage that pushed data is as expected on the server.

The test extension used by test-flagprocessor.t has been updated
so it monkeypatches the object returned from repo.file() instead
of monkeypatching filelog.filelog. This allows it to work with
extensions that return custom types from repo.file().

The monkeypatching is rather hacky and probably is performance
prohibitive for real repos. We should probably come up with a
better mechanism for registering flag processors so monkeypatching
isn't needed.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

indygreg created this revision.Apr 5 2018, 12:38 AM
durin42 accepted this revision.Apr 6 2018, 8:44 PM
This revision is now accepted and ready to land.Apr 6 2018, 8:44 PM
Closed by commit rHG9d4f09bfe3ec: simplestore: correctly implement flag processors (authored by indygreg). · Explain WhyApr 6 2018, 9:22 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 7698

tests/flagprocessorext.py

# coding=UTF-8 # coding=UTF-8
from __future__ import absolute_import from __future__ import absolute_import
import base64 import base64
import zlib import zlib
from mercurial import ( from mercurial import (
changegroup, changegroup,
exchange, exchange,
extensions, extensions,
filelog,
revlog, revlog,
util, util,
) )
# Test only: These flags are defined here only in the context of testing the # Test only: These flags are defined here only in the context of testing the
# behavior of the flag processor. The canonical way to add flags is to get in # behavior of the flag processor. The canonical way to add flags is to get in
# touch with the community and make them known in revlog. # touch with the community and make them known in revlog.
REVIDX_NOOP = (1 << 3) REVIDX_NOOP = (1 << 3)
versions.add(b'03') versions.add(b'03')
return versions return versions
def allsupportedversions(orig, ui): def allsupportedversions(orig, ui):
versions = orig(ui) versions = orig(ui)
versions.add(b'03') versions.add(b'03')
return versions return versions
def noopaddrevision(orig, self, text, transaction, link, p1, p2, def makewrappedfile(obj):
class wrappedfile(obj.__class__):
def addrevision(self, text, transaction, link, p1, p2,
cachedelta=None, node=None, cachedelta=None, node=None,
flags=revlog.REVIDX_DEFAULT_FLAGS): flags=revlog.REVIDX_DEFAULT_FLAGS):
if b'[NOOP]' in text: if b'[NOOP]' in text:
flags |= REVIDX_NOOP flags |= REVIDX_NOOP
return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
node=node, flags=flags)
def b64addrevision(orig, self, text, transaction, link, p1, p2,
cachedelta=None, node=None,
flags=revlog.REVIDX_DEFAULT_FLAGS):
if b'[BASE64]' in text: if b'[BASE64]' in text:
flags |= REVIDX_BASE64 flags |= REVIDX_BASE64
return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
node=node, flags=flags)
def gzipaddrevision(orig, self, text, transaction, link, p1, p2,
cachedelta=None, node=None,
flags=revlog.REVIDX_DEFAULT_FLAGS):
if b'[GZIP]' in text: if b'[GZIP]' in text:
flags |= REVIDX_GZIP flags |= REVIDX_GZIP
return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
node=node, flags=flags)
def failaddrevision(orig, self, text, transaction, link, p1, p2,
cachedelta=None, node=None,
flags=revlog.REVIDX_DEFAULT_FLAGS):
# This addrevision wrapper is meant to add a flag we will not have # This addrevision wrapper is meant to add a flag we will not have
# transforms registered for, ensuring we handle this error case. # transforms registered for, ensuring we handle this error case.
if b'[FAIL]' in text: if b'[FAIL]' in text:
flags |= REVIDX_FAIL flags |= REVIDX_FAIL
return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
node=node, flags=flags) return super(wrappedfile, self).addrevision(text, transaction, link,
p1, p2,
cachedelta=cachedelta,
node=node,
flags=flags)
obj.__class__ = wrappedfile
def reposetup(ui, repo):
class wrappingflagprocessorrepo(repo.__class__):
def file(self, f):
orig = super(wrappingflagprocessorrepo, self).file(f)
makewrappedfile(orig)
return orig
repo.__class__ = wrappingflagprocessorrepo
def extsetup(ui): def extsetup(ui):
# Enable changegroup3 for flags to be sent over the wire # Enable changegroup3 for flags to be sent over the wire
wrapfunction = extensions.wrapfunction wrapfunction = extensions.wrapfunction
wrapfunction(changegroup, wrapfunction(changegroup,
'supportedoutgoingversions', 'supportedoutgoingversions',
supportedoutgoingversions) supportedoutgoingversions)
wrapfunction(changegroup, wrapfunction(changegroup,
'allsupportedversions', 'allsupportedversions',
allsupportedversions) allsupportedversions)
# Teach revlog about our test flags # Teach revlog about our test flags
flags = [REVIDX_NOOP, REVIDX_BASE64, REVIDX_GZIP, REVIDX_FAIL] flags = [REVIDX_NOOP, REVIDX_BASE64, REVIDX_GZIP, REVIDX_FAIL]
revlog.REVIDX_KNOWN_FLAGS |= util.bitsfrom(flags) revlog.REVIDX_KNOWN_FLAGS |= util.bitsfrom(flags)
revlog.REVIDX_FLAGS_ORDER.extend(flags) revlog.REVIDX_FLAGS_ORDER.extend(flags)
# Teach exchange to use changegroup 3 # Teach exchange to use changegroup 3
for k in exchange._bundlespeccontentopts.keys(): for k in exchange._bundlespeccontentopts.keys():
exchange._bundlespeccontentopts[k]["cg.version"] = "03" exchange._bundlespeccontentopts[k]["cg.version"] = "03"
# Add wrappers for addrevision, responsible to set flags depending on the
# revision data contents.
wrapfunction(filelog.filelog, 'addrevision', noopaddrevision)
wrapfunction(filelog.filelog, 'addrevision', b64addrevision)
wrapfunction(filelog.filelog, 'addrevision', gzipaddrevision)
wrapfunction(filelog.filelog, 'addrevision', failaddrevision)
# Register flag processors for each extension # Register flag processors for each extension
revlog.addflagprocessor( revlog.addflagprocessor(
REVIDX_NOOP, REVIDX_NOOP,
( (
noopdonothing, noopdonothing,
noopdonothing, noopdonothing,
validatehash, validatehash,
) )

tests/simplestorerepo.py

validaterev(rev) validaterev(rev)
node = self.node(rev) node = self.node(rev)
return len(self.revision(node, raw=True)) return len(self.revision(node, raw=True))
def _processflags(self, text, flags, operation, raw=False): def _processflags(self, text, flags, operation, raw=False):
if flags == 0: if flags == 0:
return text, True return text, True
if flags & ~revlog.REVIDX_KNOWN_FLAGS:
raise error.RevlogError(_("incompatible revision flag '%#x'") %
(flags & ~revlog.REVIDX_KNOWN_FLAGS))
validatehash = True validatehash = True
# Depending on the operation (read or write), the order might be # Depending on the operation (read or write), the order might be
# reversed due to non-commutative transforms. # reversed due to non-commutative transforms.
orderedflags = revlog.REVIDX_FLAGS_ORDER orderedflags = revlog.REVIDX_FLAGS_ORDER
if operation == 'write': if operation == 'write':
orderedflags = reversed(orderedflags) orderedflags = reversed(orderedflags)
for flag in orderedflags: for flag in orderedflags:
return [b'/'.join((self._storepath, f)) for f in entries] return [b'/'.join((self._storepath, f)) for f in entries]
# Required by verify. # Required by verify.
def checksize(self): def checksize(self):
return 0, 0 return 0, 0
def add(self, text, meta, transaction, linkrev, p1, p2): def add(self, text, meta, transaction, linkrev, p1, p2):
transaction.addbackup(self._indexpath)
if meta or text.startswith(b'\1\n'): if meta or text.startswith(b'\1\n'):
text = filelog.packmeta(meta, text) text = filelog.packmeta(meta, text)
return self.addrevision(text, transaction, linkrev, p1, p2) return self.addrevision(text, transaction, linkrev, p1, p2)
def addrevision(self, text, transaction, linkrev, p1, p2, node=None, def addrevision(self, text, transaction, linkrev, p1, p2, node=None,
flags=0): flags=revlog.REVIDX_DEFAULT_FLAGS, cachedelta=None):
validatenode(p1) validatenode(p1)
validatenode(p2) validatenode(p2)
if flags: if flags:
node = node or revlog.hash(text, p1, p2) node = node or revlog.hash(text, p1, p2)
rawtext, validatehash = self._processflags(text, flags, 'write') rawtext, validatehash = self._processflags(text, flags, 'write')
node = node or revlog.hash(text, p1, p2) node = node or revlog.hash(text, p1, p2)
if node in self._indexbynode: if node in self._indexbynode:
return node return node
if validatehash: if validatehash:
self.checkhash(rawtext, node, p1=p1, p2=p2) self.checkhash(rawtext, node, p1=p1, p2=p2)
return self._addrawrevision(node, rawtext, transaction, linkrev, p1, p2,
flags)
def _addrawrevision(self, node, rawtext, transaction, link, p1, p2, flags):
transaction.addbackup(self._indexpath)
path = b'/'.join([self._storepath, hex(node)]) path = b'/'.join([self._storepath, hex(node)])
self._svfs.write(path, text) self._svfs.write(path, rawtext)
self._indexdata.append({ self._indexdata.append({
b'node': node, b'node': node,
b'p1': p1, b'p1': p1,
b'p2': p2, b'p2': p2,
b'linkrev': linkrev, b'linkrev': link,
b'flags': flags, b'flags': flags,
}) })
self._reflectindexupdate() self._reflectindexupdate()
return node return node
def _reflectindexupdate(self): def _reflectindexupdate(self):
self._refreshindex() self._refreshindex()
self._svfs.write(self._indexpath, cbor.dumps(self._indexdata)) self._svfs.write(self._indexpath, cbor.dumps(self._indexdata))
def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None): def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
nodes = [] nodes = []
transaction.addbackup(self._indexpath) transaction.addbackup(self._indexpath)
for node, p1, p2, linknode, deltabase, delta, flags in deltas: for node, p1, p2, linknode, deltabase, delta, flags in deltas:
linkrev = linkmapper(linknode) linkrev = linkmapper(linknode)
flags = flags or revlog.REVIDX_DEFAULT_FLAGS
nodes.append(node) nodes.append(node)
if node in self._indexbynode: if node in self._indexbynode:
continue continue
# Need to resolve the fulltext from the delta base. # Need to resolve the fulltext from the delta base.
if deltabase == nullid: if deltabase == nullid:
text = mdiff.patch(b'', delta) text = mdiff.patch(b'', delta)
else: else:
text = mdiff.patch(self.revision(deltabase), delta) text = mdiff.patch(self.revision(deltabase), delta)
self.addrevision(text, transaction, linkrev, p1, p2, flags) self._addrawrevision(node, text, transaction, linkrev, p1, p2,
flags)
if addrevisioncb: if addrevisioncb:
addrevisioncb(self, node) addrevisioncb(self, node)
return nodes return nodes
def revdiff(self, rev1, rev2): def revdiff(self, rev1, rev2):
validaterev(rev1) validaterev(rev1)

tests/test-flagprocessor.t

$ hg push $ hg push
pushing to $TESTTMP/server pushing to $TESTTMP/server
searching for changes searching for changes
adding changesets adding changesets
adding manifests adding manifests
adding file changes adding file changes
added 7 changesets with 7 changes to 7 files added 7 changesets with 7 changes to 7 files
Ensure the data got to the server OK
$ cd ../server
$ hg cat -r 6e48f4215d24 noop
[NOOP]
$ hg debugdata noop 0
[NOOP]
$ hg cat -r 6e48f4215d24 base64
[BASE64]
$ hg debugdata base64 0
W0JBU0U2NF0K (no-eol)
$ hg cat -r 6e48f4215d24 gzip
[GZIP]
$ hg debugdata gzip 0
x\x9c\x8bv\x8f\xf2\x0c\x88\xe5\x02\x00\x08\xc8\x01\xfd (no-eol) (esc)
$ hg cat -r 6e48f4215d24 noop-base64
[NOOP][BASE64]
$ hg debugdata noop-base64 0
W05PT1BdW0JBU0U2NF0K (no-eol)
$ hg cat -r 6e48f4215d24 noop-gzip
[NOOP][GZIP]
$ hg debugdata noop-gzip 0
x\x9c\x8b\xf6\xf3\xf7\x0f\x88\x8dv\x8f\xf2\x0c\x88\xe5\x02\x00\x1dH\x03\xf1 (no-eol) (esc)
$ hg cat -r 6e48f4215d24 base64-gzip
[BASE64][GZIP]
$ hg debugdata base64-gzip 0
eJyLdnIMdjUziY12j/IMiOUCACLBBDo= (no-eol)
$ hg cat -r 6e48f4215d24 base64-gzip-noop
[BASE64][GZIP][NOOP]
$ hg debugdata base64-gzip-noop 0
eJyLdnIMdjUziY12j/IMiI328/cPiOUCAESjBi4= (no-eol)
# Initialize new client (not cloning) and setup extension # Initialize new client (not cloning) and setup extension
$ cd .. $ cd ..
$ hg init client2 $ hg init client2
$ cd client2 $ cd client2
$ cat >> .hg/hgrc << EOF $ cat >> .hg/hgrc << EOF
> [paths] > [paths]
> default = $TESTTMP/server > default = $TESTTMP/server
> [extensions] > [extensions]
> echo '[BASE64]a-bit-longer-'$i > base64 > echo '[BASE64]a-bit-longer-'$i > base64
> hg commit -m base64-$i -A base64 > hg commit -m base64-$i -A base64
> done > done
$ hg update 2 -q $ hg update 2 -q
$ echo '[BASE64]a-bit-longer-branching' > base64 $ echo '[BASE64]a-bit-longer-branching' > base64
$ hg commit -q -m branching $ hg commit -q -m branching
#if repobundlerepo
$ hg bundle --base 1 bundle.hg $ hg bundle --base 1 bundle.hg
4 changesets found 4 changesets found
$ hg --config extensions.strip= strip -r 2 --no-backup --force -q $ hg --config extensions.strip= strip -r 2 --no-backup --force -q
$ hg -R bundle.hg log --stat -T '{rev} {desc}\n' base64 $ hg -R bundle.hg log --stat -T '{rev} {desc}\n' base64
5 branching 5 branching
base64 | 2 +- base64 | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-) 1 files changed, 1 insertions(+), 1 deletions(-)
base64 | 2 +- base64 | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-) 1 files changed, 1 insertions(+), 1 deletions(-)
0 base64-0 0 base64-0
base64 | 1 + base64 | 1 +
1 files changed, 1 insertions(+), 0 deletions(-) 1 files changed, 1 insertions(+), 0 deletions(-)
$ rm bundle.hg bundle-again.hg $ rm bundle.hg bundle-again.hg
#endif
# TEST: hg status # TEST: hg status
$ hg status $ hg status
$ hg diff $ hg diff