This is an archive of the discontinued Mercurial Phabricator instance.

Differential D1017

infinitepush: send treepack parts with the client push
ClosedPublic

Authored by durham on Oct 11 2017, 7:59 PM.
Tags
None
Subscribers
None

Details

Reviewers
phillco
Group Reviewers
Restricted Project
Commits
rFBHGXd3cd9d8a1443: infinitepush: send treepack parts with the client push
Summary

This makes the client side pushes also include tree packs in the push.

Also adds a test to see that the tree parts round trip between two tree-only
clients.

This was reverted in D921 because clients couldn't push old non-tree commits to
infinitepush anymore. I've fixed the issue and added a test to this commit.

Diff Detail

Repository
rFBHGX Facebook Mercurial Extensions
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

durham created this revision.Oct 11 2017, 7:59 PM
Herald added a reviewer: Restricted Project. · View Herald TranscriptOct 11 2017, 7:59 PM
durham planned changes to this revision.Oct 11 2017, 8:06 PM

I've found a bug.

durham updated this revision to Diff 3162.Oct 30 2017, 7:25 PM
durham added a comment.Nov 1 2017, 7:03 PM

The bug is fixed. This is ready for review

phillco accepted this revision.Nov 1 2017, 7:07 PM
This revision is now accepted and ready to land.Nov 1 2017, 7:07 PM
Closed by commit rFBHGXd3cd9d8a1443: infinitepush: send treepack parts with the client push (authored by durham). · Explain WhyNov 2 2017, 3:05 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 3222

infinitepush/__init__.py

import socket import socket
import struct import struct
import subprocess import subprocess
import sys import sys
import tempfile import tempfile
import time import time
from .bundleparts import ( from .bundleparts import (
getscratchbranchpart, getscratchbranchparts,
scratchbookmarksparttype, scratchbookmarksparttype,
scratchbranchparttype, scratchbranchparttype,
) )
from . import ( from . import (
backupcommands, backupcommands,
infinitepushcommands, infinitepushcommands,
common, common,
) )
cgpart.addparam('version', version) cgpart.addparam('version', version)
parts.append(cgpart) parts.append(cgpart)
try: try:
treemod = extensions.find('treemanifest') treemod = extensions.find('treemanifest')
except KeyError: except KeyError:
pass pass
else: else:
mfnodes = []
for node in outgoing.missing:
mfnodes.append(('', bundlerepo[node].manifestnode()))
# Only include the tree parts if they all exist
if not bundlerepo.manifestlog.datastore.getmissing(mfnodes):
treepart = treemod.createtreepackpart(bundlerepo, outgoing, treepart = treemod.createtreepackpart(bundlerepo, outgoing,
treemod.TREEGROUP_PARTTYPE2) treemod.TREEGROUP_PARTTYPE2)
parts.append(treepart) parts.append(treepart)
return parts return parts
def _getbundleroots(oldrepo, bundlerepo, bundlerevs): def _getbundleroots(oldrepo, bundlerepo, bundlerevs):
cl = bundlerepo.changelog cl = bundlerepo.changelog
bundleroots = [] bundleroots = []
for rev in bundlerevs: for rev in bundlerevs:
node = cl.node(rev) node = cl.node(rev)
scratchpush = pushop.ui.configbool(experimental, configscratchpush) scratchpush = pushop.ui.configbool(experimental, configscratchpush)
if 'changesets' in pushop.stepsdone or not scratchpush: if 'changesets' in pushop.stepsdone or not scratchpush:
return return
if scratchbranchparttype not in bundle2.bundle2caps(pushop.remote): if scratchbranchparttype not in bundle2.bundle2caps(pushop.remote):
return return
pushop.stepsdone.add('changesets') pushop.stepsdone.add('changesets')
pushop.stepsdone.add('treepack')
if not pushop.outgoing.missing: if not pushop.outgoing.missing:
pushop.ui.status(_('no changes found\n')) pushop.ui.status(_('no changes found\n'))
pushop.cgresult = 0 pushop.cgresult = 0
return return
# This parameter tells the server that the following bundle is an # This parameter tells the server that the following bundle is an
# infinitepush. This let's it switch the part processing to our infinitepush # infinitepush. This let's it switch the part processing to our infinitepush
# code path. # code path.
bundler.addparam("infinitepush", "True") bundler.addparam("infinitepush", "True")
nonforwardmove = pushop.force or pushop.ui.configbool(experimental, nonforwardmove = pushop.force or pushop.ui.configbool(experimental,
confignonforwardmove) confignonforwardmove)
scratchpart = getscratchbranchpart(pushop.repo, scratchparts = getscratchbranchparts(pushop.repo,
pushop.remote, pushop.remote,
pushop.outgoing, pushop.outgoing,
nonforwardmove, nonforwardmove,
pushop.ui, pushop.ui,
bookmark, bookmark,
create) create)
for scratchpart in scratchparts:
bundler.addpart(scratchpart) bundler.addpart(scratchpart)
def handlereply(op): def handlereply(op):
# server either succeeds or aborts; no code to read # server either succeeds or aborts; no code to read
pushop.cgresult = 1 pushop.cgresult = 1
return handlereply return handlereply
bundle2.capabilities[scratchbranchparttype] = () bundle2.capabilities[scratchbranchparttype] = ()

infinitepush/backupcommands.py

import re import re
import socket import socket
import stat import stat
import subprocess import subprocess
import time import time
from .bundleparts import ( from .bundleparts import (
getscratchbookmarkspart, getscratchbookmarkspart,
getscratchbranchpart, getscratchbranchparts,
) )
from mercurial import ( from mercurial import (
bundle2, bundle2,
changegroup, changegroup,
commands, commands,
discovery, discovery,
dispatch, dispatch,
encoding, encoding,
# See _deltaparent comments for details # See _deltaparent comments for details
wrapfunction(changegroup.cg2packer, 'deltaparent', _deltaparent) wrapfunction(changegroup.cg2packer, 'deltaparent', _deltaparent)
try: try:
bundler = _createbundler(ui, repo, other) bundler = _createbundler(ui, repo, other)
bundler.addparam("infinitepush", "True") bundler.addparam("infinitepush", "True")
backup = False backup = False
if outgoing and outgoing.missing: if outgoing and outgoing.missing:
backup = True backup = True
bundler.addpart(getscratchbranchpart(repo, other, outgoing, parts = getscratchbranchparts(repo, other, outgoing,
confignonforwardmove=False, confignonforwardmove=False,
ui=ui, bookmark=None, ui=ui, bookmark=None,
create=False)) create=False)
for part in parts:
bundler.addpart(part)
if bookmarkstobackup: if bookmarkstobackup:
backup = True backup = True
bundler.addpart(getscratchbookmarkspart(other, bookmarkstobackup)) bundler.addpart(getscratchbookmarkspart(other, bookmarkstobackup))
if backup: if backup:
_sendbundle(bundler, other) _sendbundle(bundler, other)
_writelocalbackupstate(repo.vfs, afterbackupheads, _writelocalbackupstate(repo.vfs, afterbackupheads,

infinitepush/bundleparts.py

extensions, extensions,
revsetlang, revsetlang,
) )
from mercurial.i18n import _ from mercurial.i18n import _
scratchbranchparttype = 'b2x:infinitepush' scratchbranchparttype = 'b2x:infinitepush'
scratchbookmarksparttype = 'b2x:infinitepushscratchbookmarks' scratchbookmarksparttype = 'b2x:infinitepushscratchbookmarks'
def getscratchbranchpart(repo, peer, outgoing, confignonforwardmove, def getscratchbranchparts(repo, peer, outgoing, confignonforwardmove,
ui, bookmark, create): ui, bookmark, create):
if not outgoing.missing: if not outgoing.missing:
raise error.Abort(_('no commits to push')) raise error.Abort(_('no commits to push'))
if scratchbranchparttype not in bundle2.bundle2caps(peer): if scratchbranchparttype not in bundle2.bundle2caps(peer):
raise error.Abort(_('no server support for %r') % scratchbranchparttype) raise error.Abort(_('no server support for %r') % scratchbranchparttype)
_validaterevset(repo, revsetlang.formatspec('%ln', outgoing.missing), _validaterevset(repo, revsetlang.formatspec('%ln', outgoing.missing),
if confignonforwardmove: if confignonforwardmove:
params['force'] = '1' params['force'] = '1'
# Do not send pushback bundle2 part with bookmarks if remotenames extension # Do not send pushback bundle2 part with bookmarks if remotenames extension
# is enabled. It will be handled manually in `_push()` # is enabled. It will be handled manually in `_push()`
if not isremotebooksenabled(ui): if not isremotebooksenabled(ui):
params['pushbackbookmarks'] = '1' params['pushbackbookmarks'] = '1'
parts = []
# .upper() marks this as a mandatory part: server will abort if there's no # .upper() marks this as a mandatory part: server will abort if there's no
# handler # handler
return bundle2.bundlepart( parts.append(bundle2.bundlepart(
scratchbranchparttype.upper(), scratchbranchparttype.upper(),
advisoryparams=params.iteritems(), advisoryparams=params.iteritems(),
data=cg) data=cg))
try:
treemod = extensions.find('treemanifest')
mfnodes = []
for node in outgoing.missing:
mfnodes.append(('', repo[node].manifestnode()))
# Only include the tree parts if they all exist
if not repo.manifestlog.datastore.getmissing(mfnodes):
parts.append(treemod.createtreepackpart(
repo, outgoing, treemod.TREEGROUP_PARTTYPE2))
except KeyError:
pass
return parts
def getscratchbookmarkspart(peer, bookmarks): def getscratchbookmarkspart(peer, bookmarks):
if scratchbookmarksparttype not in bundle2.bundle2caps(peer): if scratchbookmarksparttype not in bundle2.bundle2caps(peer):
raise error.Abort( raise error.Abort(
_('no server support for %r') % scratchbookmarksparttype) _('no server support for %r') % scratchbookmarksparttype)
return bundle2.bundlepart( return bundle2.bundlepart(
scratchbookmarksparttype.upper(), scratchbookmarksparttype.upper(),

remotefilelog/shallowbundle.py

yield c yield c
yield self.close() yield self.close()
def _cansendflat(repo, mfnodes): def _cansendflat(repo, mfnodes):
if not util.safehasattr(repo.manifestlog, '_revlog'): if not util.safehasattr(repo.manifestlog, '_revlog'):
return False return False
if repo.ui.configbool('treemanifest', 'treeonly'):
return False
revlog = repo.manifestlog._revlog revlog = repo.manifestlog._revlog
for mfnode in mfnodes: for mfnode in mfnodes:
if mfnode not in revlog.nodemap: if mfnode not in revlog.nodemap:
return False return False
return True return True
@shallowutil.interposeclass(changegroup, 'cg1packer') @shallowutil.interposeclass(changegroup, 'cg1packer')

tests/test-treemanifest-infinitepush.t

  • This file was added.
$ . "$TESTDIR/library.sh"
$ . "$TESTDIR/library-infinitepush.sh"
$ PYTHONPATH=$TESTDIR/..:$PYTHONPATH
$ export PYTHONPATH
$ setupcommon
$ hginit master
$ cd master
$ setupserver
$ cat >> .hg/hgrc <<EOF
> [extensions]
> pushrebase=$TESTDIR/../hgext3rd/pushrebase.py
> treemanifest=$TESTDIR/../treemanifest
> [remotefilelog]
> server=True
> [treemanifest]
> server=True
> EOF
$ echo x > x
$ hg commit -qAm 'add x'
$ cd ..
Push a non-tree scratch branch from one client
$ hgcloneshallow ssh://user@dummy/master normal-client -q
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ cd normal-client
$ mkdir bar
$ echo >> bar/car
$ hg commit -qAm 'add bar/car'
$ cat >> .hg/hgrc <<EOF
> [extensions]
> treemanifest=$TESTDIR/../treemanifest
> fastmanifest=$TESTDIR/../fastmanifest
>
> [remotefilelog]
> usefastdatapack=True
>
> [fastmanifest]
> usecache=False
> usetree=True
> EOF
$ hg push --to scratch/nontree --create
pushing to ssh://user@dummy/master
searching for changes
1 trees fetched over * (glob)
remote: pushing 1 commit:
remote: 42ec76eb772a add bar/car
$ clearcache
$ cd ..
Push a tree-only scratch branch from another client
$ hgcloneshallow ssh://user@dummy/master client1 -q --config extensions.treemanifest=$TESTDIR/../treemanifest --config treemanifest.treeonly=True
1 trees fetched over * (glob)
1 trees fetched over * (glob)
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ cd client1
$ cat >> .hg/hgrc <<EOF
> [extensions]
> treemanifest=$TESTDIR/../treemanifest
>
> [remotefilelog]
> usefastdatapack=True
>
> [treemanifest]
> treeonly=True
> EOF
$ mkdir subdir
$ echo "my change" >> subdir/a
$ hg commit -qAm 'add subdir/a'
$ hg push --to scratch/foo --create
pushing to ssh://user@dummy/master
searching for changes
remote: pushing 1 commit:
remote: 02c12aef64ff add subdir/a
$ cd ..
Pull a non-tree scratch branch into a normal client
$ hgcloneshallow ssh://user@dummy/master normal-client2 -q
$ cd normal-client2
$ cat >> .hg/hgrc <<EOF
> [extensions]
> treemanifest=$TESTDIR/../treemanifest
> fastmanifest=$TESTDIR/../fastmanifest
>
> [remotefilelog]
> usefastdatapack=True
>
> [fastmanifest]
> usecache=False
> usetree=True
> EOF
$ hg pull -r scratch/nontree
pulling from ssh://user@dummy/master
searching for changes
adding changesets
adding manifests
adding file changes
added 1 changesets with 1 changes to 1 files
new changesets 42ec76eb772a
(run 'hg update' to get a working copy)
$ hg log -r tip -vp
changeset: 1:42ec76eb772a
tag: tip
user: test
date: Thu Jan 01 00:00:00 1970 +0000
files: bar/car
description:
add bar/car
diff -r 085784c01c08 -r 42ec76eb772a bar/car
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/bar/car Thu Jan 01 00:00:00 1970 +0000
@@ -0,0 +1,1 @@
+
Pull a treeonly scratch branch into a normal client
$ hg debugindex -m
rev offset length delta linkrev nodeid p1 p2
0 0 44 -1 0 bc0c2c938b92 000000000000 000000000000
1 44 59 0 1 bf0601d5cb94 bc0c2c938b92 000000000000
$ hg pull -r scratch/foo
pulling from ssh://user@dummy/master
searching for changes
adding changesets
adding manifests
adding file changes
added 1 changesets with 1 changes to 1 files (+1 heads)
new changesets 02c12aef64ff
(run 'hg heads' to see heads, 'hg merge' to merge)
- Verify no new manifest revlog entry was written
$ hg debugindex -m
rev offset length delta linkrev nodeid p1 p2
0 0 44 -1 0 bc0c2c938b92 000000000000 000000000000
1 44 59 0 1 bf0601d5cb94 bc0c2c938b92 000000000000
- ...but we can still read the manifest
$ hg log -r 02c12aef64ff --stat -T '{rev}\n'
2
subdir/a | 1 +
1 files changed, 1 insertions(+), 0 deletions(-)
$ cd ..
Pull a treeonly scratch branch into a treeonly client
$ hgcloneshallow ssh://user@dummy/master client2 -q --config extensions.treemanifest=$TESTDIR/../treemanifest --config treemanifest.treeonly=True
$ cd client2
$ cat >> .hg/hgrc <<EOF
> [extensions]
> treemanifest=$TESTDIR/../treemanifest
>
> [remotefilelog]
> usefastdatapack=True
>
> [treemanifest]
> treeonly=True
> EOF
$ hg pull -r scratch/foo
pulling from ssh://user@dummy/master
searching for changes
adding changesets
adding manifests
adding file changes
added 1 changesets with 1 changes to 1 files
new changesets 02c12aef64ff
(run 'hg update' to get a working copy)
$ hg log -G
o changeset: 1:02c12aef64ff
| tag: tip
| user: test
| date: Thu Jan 01 00:00:00 1970 +0000
| summary: add subdir/a
|
@ changeset: 0:085784c01c08
user: test
date: Thu Jan 01 00:00:00 1970 +0000
summary: add x
$ hg cat -r tip subdir/a
my change
$ ls_l .hg/store
-rw-r--r-- 257 00changelog.i
-rw-r--r-- 108 00manifesttree.i
drwxr-xr-x data
drwxrwxr-x packs
-rw-r--r-- 43 phaseroots
-rw-r--r-- 18 undo
-rw-r--r-- 17 undo.backupfiles
-rw-r--r-- 0 undo.phaseroots
Pull a normal scratch branch into a treeonly client
$ cd ..
Verify its not on the server
$ cd master
$ hg log -G
@ changeset: 0:085784c01c08
tag: tip
user: test
date: Thu Jan 01 00:00:00 1970 +0000
summary: add x