This is an archive of the discontinued Mercurial Phabricator instance.

Differential D9883

revlog: change addgroup callbacks to take revision numbers
ClosedPublic

Authored by joerg.sonnenberger on Jan 26 2021, 9:10 PM.

Details

Reviewers
indygreg
pulkit
Group Reviewers
hg-reviewers
Commits
rHG7a93b7b3dc2d: revlog: change addgroup callbacks to take revision numbers
rHG8ce2d7d993a5: revlog: change addgroup callbacks to take revision numbers

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

pulkit accepted this revision.Jan 31 2021, 2:56 PM
pulkit added a subscriber: pulkit.
pulkit added inline comments.
hgext/sqlitestore.py
640

This should probably be a part of one of the previous patch?

This revision is now accepted and ready to land.Jan 31 2021, 2:56 PM
joerg.sonnenberger marked an inline comment as done.Feb 1 2021, 11:26 AM

Revision Contents
Changeset List

Diff 25290

hgext/sqlitestore.py

# End of ifiledata interface. # End of ifiledata interface.
# Start of ifilemutation interface. # Start of ifilemutation interface.
def add(self, filedata, meta, transaction, linkrev, p1, p2): def add(self, filedata, meta, transaction, linkrev, p1, p2):
if meta or filedata.startswith(b'\x01\n'): if meta or filedata.startswith(b'\x01\n'):
filedata = storageutil.packmeta(meta, filedata) filedata = storageutil.packmeta(meta, filedata)
return self.addrevision(filedata, transaction, linkrev, p1, p2) rev = self.addrevision(filedata, transaction, linkrev, p1, p2)
return self.node(rev)
pulkitUnsubmitted
Done

This should probably be a part of one of the previous patch?

pulkit: This should probably be a part of one of the previous patch?
def addrevision( def addrevision(
self, self,
revisiondata, revisiondata,
transaction, transaction,
linkrev, linkrev,
p1, p1,
p2, p2,
node=None, node=None,
flags=0, flags=0,
cachedelta=None, cachedelta=None,
): ):
if flags: if flags:
raise SQLiteStoreError(_(b'flags not supported on revisions')) raise SQLiteStoreError(_(b'flags not supported on revisions'))
validatehash = node is not None validatehash = node is not None
node = node or storageutil.hashrevisionsha1(revisiondata, p1, p2) node = node or storageutil.hashrevisionsha1(revisiondata, p1, p2)
if validatehash: if validatehash:
self._checkhash(revisiondata, node, p1, p2) self._checkhash(revisiondata, node, p1, p2)
if node in self._nodetorev: rev = self._nodetorev.get(node)
return node if rev is not None:
return rev
node = self._addrawrevision( rev = self._addrawrevision(
node, revisiondata, transaction, linkrev, p1, p2 node, revisiondata, transaction, linkrev, p1, p2
) )
self._revisioncache[node] = revisiondata self._revisioncache[node] = revisiondata
return node return rev
def addgroup( def addgroup(
self, self,
deltas, deltas,
linkmapper, linkmapper,
transaction, transaction,
addrevisioncb=None, addrevisioncb=None,
duplicaterevisioncb=None, duplicaterevisioncb=None,
entry.flags &= ~FLAG_MISSING_P2 entry.flags &= ~FLAG_MISSING_P2
self._db.execute( self._db.execute(
'UPDATE fileindex SET p2rev=?, flags=? WHERE id=?', 'UPDATE fileindex SET p2rev=?, flags=? WHERE id=?',
(self._nodetorev[p1], entry.flags, entry.rid), (self._nodetorev[p1], entry.flags, entry.rid),
) )
if duplicaterevisioncb: if duplicaterevisioncb:
duplicaterevisioncb(self, node) duplicaterevisioncb(self, self.rev(node))
empty = False empty = False
continue continue
if deltabase == nullid: if deltabase == nullid:
text = mdiff.patch(b'', delta) text = mdiff.patch(b'', delta)
storedelta = None storedelta = None
else: else:
text = None text = None
storedelta = (deltabase, delta) storedelta = (deltabase, delta)
self._addrawrevision( rev = self._addrawrevision(
node, node,
text, text,
transaction, transaction,
linkrev, linkrev,
p1, p1,
p2, p2,
storedelta=storedelta, storedelta=storedelta,
flags=storeflags, flags=storeflags,
) )
if addrevisioncb: if addrevisioncb:
addrevisioncb(self, node) addrevisioncb(self, rev)
empty = False empty = False
return not empty return not empty
def censorrevision(self, tr, censornode, tombstone=b''): def censorrevision(self, tr, censornode, tombstone=b''):
tombstone = storageutil.packmeta({b'censored': tombstone}, b'') tombstone = storageutil.packmeta({b'censored': tombstone}, b'')
# This restriction is cargo culted from revlogs and makes no sense for # This restriction is cargo culted from revlogs and makes no sense for
linkrev=linkrev, linkrev=linkrev,
flags=flags, flags=flags,
) )
self._nodetorev[node] = rev self._nodetorev[node] = rev
self._revtonode[rev] = node self._revtonode[rev] = node
self._revisions[node] = entry self._revisions[node] = entry
return node return rev
class sqliterepository(localrepo.localrepository): class sqliterepository(localrepo.localrepository):
def cancopy(self): def cancopy(self):
return False return False
def transaction(self, *args, **kwargs): def transaction(self, *args, **kwargs):
current = self.currenttransaction() current = self.currenttransaction()

mercurial/changegroup.py

progress = repo.ui.makeprogress( progress = repo.ui.makeprogress(
_(b'changesets'), unit=_(b'chunks'), total=expectedtotal _(b'changesets'), unit=_(b'chunks'), total=expectedtotal
) )
self.callback = progress.increment self.callback = progress.increment
efilesset = set() efilesset = set()
cgnodes = [] cgnodes = []
def ondupchangelog(cl, node): def ondupchangelog(cl, rev):
if cl.rev(node) < clstart: if rev < clstart:
cgnodes.append(node) cgnodes.append(cl.node(rev))
def onchangelog(cl, node): def onchangelog(cl, rev):
rev = cl.rev(node)
ctx = cl.changelogrevision(rev) ctx = cl.changelogrevision(rev)
efilesset.update(ctx.files) efilesset.update(ctx.files)
repo.register_changeset(rev, ctx) repo.register_changeset(rev, ctx)
self.changelogheader() self.changelogheader()
deltas = self.deltaiter() deltas = self.deltaiter()
if not cl.addgroup( if not cl.addgroup(
deltas, deltas,

mercurial/changelog.py

def branchinfo(self, rev): def branchinfo(self, rev):
"""return the branch name and open/close state of a revision """return the branch name and open/close state of a revision
This function exists because creating a changectx object This function exists because creating a changectx object
just to access this is costly.""" just to access this is costly."""
return self.changelogrevision(rev).branchinfo return self.changelogrevision(rev).branchinfo
def _nodeduplicatecallback(self, transaction, node): def _nodeduplicatecallback(self, transaction, rev):
# keep track of revisions that got "re-added", eg: unbunde of know rev. # keep track of revisions that got "re-added", eg: unbunde of know rev.
# #
# We track them in a list to preserve their order from the source bundle # We track them in a list to preserve their order from the source bundle
duplicates = transaction.changes.setdefault(b'revduplicates', []) duplicates = transaction.changes.setdefault(b'revduplicates', [])
duplicates.append(self.rev(node)) duplicates.append(rev)

mercurial/exchangev2.py

manifestnodes = {} manifestnodes = {}
added = [] added = []
def linkrev(node): def linkrev(node):
repo.ui.debug(b'add changeset %s\n' % short(node)) repo.ui.debug(b'add changeset %s\n' % short(node))
# Linkrev for changelog is always self. # Linkrev for changelog is always self.
return len(cl) return len(cl)
def ondupchangeset(cl, node): def ondupchangeset(cl, rev):
added.append(node) added.append(cl.node(rev))
def onchangeset(cl, node): def onchangeset(cl, rev):
progress.increment() progress.increment()
rev = cl.rev(node)
revision = cl.changelogrevision(rev) revision = cl.changelogrevision(rev)
added.append(node) added.append(cl.node(rev))
# We need to preserve the mapping of changelog revision to node # We need to preserve the mapping of changelog revision to node
# so we can set the linkrev accordingly when manifests are added. # so we can set the linkrev accordingly when manifests are added.
manifestnodes[rev] = revision.manifest manifestnodes[rev] = revision.manifest
repo.register_changeset(rev, revision) repo.register_changeset(rev, revision)
nodesbyphase = {phase: set() for phase in phases.phasenames.values()} nodesbyphase = {phase: set() for phase in phases.phasenames.values()}
b'fields': {b'parents', b'revision'}, b'fields': {b'parents', b'revision'},
b'haveparents': True, b'haveparents': True,
}, },
).result() ).result()
# Chomp off header object. # Chomp off header object.
next(objs) next(objs)
def onchangeset(cl, node): def onchangeset(cl, rev):
added.append(node) added.append(cl.node(rev))
rootmanifest.addgroup( rootmanifest.addgroup(
iterrevisions(objs, progress), iterrevisions(objs, progress),
linkrevs.__getitem__, linkrevs.__getitem__,
weakref.proxy(tr), weakref.proxy(tr),
addrevisioncb=onchangeset, addrevisioncb=onchangeset,
duplicaterevisioncb=onchangeset, duplicaterevisioncb=onchangeset,
) )

mercurial/interfaces/repository.py

``nullid``, in which case the header from the delta can be ignored ``nullid``, in which case the header from the delta can be ignored
and the delta used as the fulltext. and the delta used as the fulltext.
``alwayscache`` instructs the lower layers to cache the content of the ``alwayscache`` instructs the lower layers to cache the content of the
newly added revision, even if it needs to be explicitly computed. newly added revision, even if it needs to be explicitly computed.
This used to be the default when ``addrevisioncb`` was provided up to This used to be the default when ``addrevisioncb`` was provided up to
Mercurial 5.8. Mercurial 5.8.
``addrevisioncb`` should be called for each node as it is committed. ``addrevisioncb`` should be called for each new rev as it is committed.
``duplicaterevisioncb`` should be called for each pre-existing node. ``duplicaterevisioncb`` should be called for all revs with a
pre-existing node.
``maybemissingparents`` is a bool indicating whether the incoming ``maybemissingparents`` is a bool indicating whether the incoming
data may reference parents/ancestor revisions that aren't present. data may reference parents/ancestor revisions that aren't present.
This flag is set when receiving data into a "shallow" store that This flag is set when receiving data into a "shallow" store that
doesn't hold all history. doesn't hold all history.
Returns a list of nodes that were processed. A node will be in the list Returns a list of nodes that were processed. A node will be in the list
even if it existed in the store previously. even if it existed in the store previously.

mercurial/revlog.py

try: try:
deltacomputer = deltautil.deltacomputer(self) deltacomputer = deltautil.deltacomputer(self)
# loop through our set of deltas # loop through our set of deltas
for data in deltas: for data in deltas:
node, p1, p2, linknode, deltabase, delta, flags = data node, p1, p2, linknode, deltabase, delta, flags = data
link = linkmapper(linknode) link = linkmapper(linknode)
flags = flags or REVIDX_DEFAULT_FLAGS flags = flags or REVIDX_DEFAULT_FLAGS
if self.index.has_node(node): rev = self.index.get_rev(node)
if rev is not None:
# this can happen if two branches make the same change # this can happen if two branches make the same change
self._nodeduplicatecallback(transaction, node) self._nodeduplicatecallback(transaction, rev)
if duplicaterevisioncb: if duplicaterevisioncb:
duplicaterevisioncb(self, node) duplicaterevisioncb(self, rev)
empty = False empty = False
continue continue
for p in (p1, p2): for p in (p1, p2):
if not self.index.has_node(p): if not self.index.has_node(p):
raise error.LookupError( raise error.LookupError(
p, self.indexfile, _(b'unknown parent') p, self.indexfile, _(b'unknown parent')
) )
# We assume consumers of addrevisioncb will want to retrieve # We assume consumers of addrevisioncb will want to retrieve
# the added revision, which will require a call to # the added revision, which will require a call to
# revision(). revision() will fast path if there is a cache # revision(). revision() will fast path if there is a cache
# hit. So, we tell _addrevision() to always cache in this case. # hit. So, we tell _addrevision() to always cache in this case.
# We're only using addgroup() in the context of changegroup # We're only using addgroup() in the context of changegroup
# generation so the revision data can always be handled as raw # generation so the revision data can always be handled as raw
# by the flagprocessor. # by the flagprocessor.
self._addrevision( rev = self._addrevision(
node, node,
None, None,
transaction, transaction,
link, link,
p1, p1,
p2, p2,
flags, flags,
(baserev, delta), (baserev, delta),
ifh, ifh,
dfh, dfh,
alwayscache=alwayscache, alwayscache=alwayscache,
deltacomputer=deltacomputer, deltacomputer=deltacomputer,
) )
if addrevisioncb: if addrevisioncb:
addrevisioncb(self, node) addrevisioncb(self, rev)
empty = False empty = False
if not dfh and not self._inline: if not dfh and not self._inline:
# addrevision switched from inline to conventional # addrevision switched from inline to conventional
# reopen the index # reopen the index
ifh.close() ifh.close()
dfh = self._datafp(b"a+") dfh = self._datafp(b"a+")
ifh = self._indexfp(b"a+") ifh = self._indexfp(b"a+")

mercurial/testing/storage.py

callbackargs.append((args, kwargs)) callbackargs.append((args, kwargs))
def linkmapper(node): def linkmapper(node):
return 0 return 0
with self._maketransactionfn() as tr: with self._maketransactionfn() as tr:
nodes = [] nodes = []
def onchangeset(cl, node): def onchangeset(cl, rev):
node = cl.node(rev)
nodes.append(node) nodes.append(node)
cb(cl, node) cb(cl, node)
def ondupchangeset(cl, node): def ondupchangeset(cl, rev):
nodes.append(node) nodes.append(cl.node(rev))
f.addgroup( f.addgroup(
[], [],
None, None,
tr, tr,
addrevisioncb=onchangeset, addrevisioncb=onchangeset,
duplicaterevisioncb=ondupchangeset, duplicaterevisioncb=ondupchangeset,
) )
deltas = [ deltas = [
(node0, nullid, nullid, nullid, nullid, delta0, 0), (node0, nullid, nullid, nullid, nullid, delta0, 0),
] ]
with self._maketransactionfn() as tr: with self._maketransactionfn() as tr:
nodes = [] nodes = []
def onchangeset(cl, node): def onchangeset(cl, rev):
node = cl.node(rev)
nodes.append(node) nodes.append(node)
cb(cl, node) cb(cl, node)
def ondupchangeset(cl, node): def ondupchangeset(cl, rev):
nodes.append(node) nodes.append(cl.node(rev))
f.addgroup( f.addgroup(
deltas, deltas,
linkmapper, linkmapper,
tr, tr,
addrevisioncb=onchangeset, addrevisioncb=onchangeset,
duplicaterevisioncb=ondupchangeset, duplicaterevisioncb=ondupchangeset,
) )
for i, fulltext in enumerate(fulltexts): for i, fulltext in enumerate(fulltexts):
delta = mdiff.trivialdiffheader(len(fulltext)) + fulltext delta = mdiff.trivialdiffheader(len(fulltext)) + fulltext
deltas.append((nodes[i], nullid, nullid, nullid, nullid, delta, 0)) deltas.append((nodes[i], nullid, nullid, nullid, nullid, delta, 0))
with self._maketransactionfn() as tr: with self._maketransactionfn() as tr:
newnodes = [] newnodes = []
def onchangeset(cl, node): def onchangeset(cl, rev):
newnodes.append(node) newnodes.append(cl.node(rev))
f.addgroup( f.addgroup(
deltas, deltas,
lambda x: 0, lambda x: 0,
tr, tr,
addrevisioncb=onchangeset, addrevisioncb=onchangeset,
duplicaterevisioncb=onchangeset, duplicaterevisioncb=onchangeset,
) )

tests/simplestorerepo.py

transaction.addbackup(self._indexpath) transaction.addbackup(self._indexpath)
for node, p1, p2, linknode, deltabase, delta, flags in deltas: for node, p1, p2, linknode, deltabase, delta, flags in deltas:
linkrev = linkmapper(linknode) linkrev = linkmapper(linknode)
flags = flags or revlog.REVIDX_DEFAULT_FLAGS flags = flags or revlog.REVIDX_DEFAULT_FLAGS
if node in self._indexbynode: if node in self._indexbynode:
if duplicaterevisioncb: if duplicaterevisioncb:
duplicaterevisioncb(self, node) duplicaterevisioncb(self, self.rev(node))
empty = False empty = False
continue continue
# Need to resolve the fulltext from the delta base. # Need to resolve the fulltext from the delta base.
if deltabase == nullid: if deltabase == nullid:
text = mdiff.patch(b'', delta) text = mdiff.patch(b'', delta)
else: else:
text = mdiff.patch(self.revision(deltabase), delta) text = mdiff.patch(self.revision(deltabase), delta)
self._addrawrevision( rev = self._addrawrevision(
node, text, transaction, linkrev, p1, p2, flags node, text, transaction, linkrev, p1, p2, flags
) )
if addrevisioncb: if addrevisioncb:
addrevisioncb(self, node) addrevisioncb(self, rev)
empty = False empty = False
return not empty return not empty
def _headrevs(self): def _headrevs(self):
# Assume all revisions are heads by default. # Assume all revisions are heads by default.
revishead = {rev: True for rev in self._indexbyrev} revishead = {rev: True for rev in self._indexbyrev}
for rev, entry in self._indexbyrev.items(): for rev, entry in self._indexbyrev.items():