This is an archive of the discontinued Mercurial Phabricator instance.

Differential D721

infinitepush: send treepack parts with the client push
ClosedPublic

Authored by durham on Sep 14 2017, 7:50 PM.
Tags
None
Subscribers
None

Details

Reviewers
stash
mitrandir
Group Reviewers
Restricted Project
Commits
rFBHGX879ee9e28464: infinitepush: send treepack parts with the client push
Summary

This makes the client side pushes also include tree packs in the push.

Also adds a test to see that the tree parts round trip between two tree-only
clients.

Diff Detail

Repository
rFBHGX Facebook Mercurial Extensions
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

durham created this revision.Sep 14 2017, 7:50 PM
Herald added a reviewer: Restricted Project. · View Herald TranscriptSep 14 2017, 7:50 PM
mitrandir accepted this revision.Sep 18 2017, 11:31 AM
This revision is now accepted and ready to land.Sep 18 2017, 11:31 AM
stash added inline comments.Sep 18 2017, 12:20 PM
remotefilelog/shallowbundle.py
75

Is it related?

durham added inline comments.Sep 20 2017, 12:57 PM
remotefilelog/shallowbundle.py
75

This was needed to make the tests pass. Basically, if you're in treeonly mode you should also not be sending flat manifests. I think eventually I'll just delete the sendflat config.

Closed by commit rFBHGX879ee9e28464: infinitepush: send treepack parts with the client push (authored by durham). · Explain WhySep 22 2017, 1:33 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 2037

infinitepush/__init__.py

import socket import socket
import struct import struct
import subprocess import subprocess
import sys import sys
import tempfile import tempfile
import time import time
from .bundleparts import ( from .bundleparts import (
getscratchbranchpart, getscratchbranchparts,
scratchbookmarksparttype, scratchbookmarksparttype,
scratchbranchparttype, scratchbranchparttype,
) )
from . import ( from . import (
backupcommands, backupcommands,
infinitepushcommands, infinitepushcommands,
common, common,
) )
# This parameter tells the server that the following bundle is an # This parameter tells the server that the following bundle is an
# infinitepush. This let's it switch the part processing to our infinitepush # infinitepush. This let's it switch the part processing to our infinitepush
# code path. # code path.
bundler.addparam("infinitepush", "True") bundler.addparam("infinitepush", "True")
nonforwardmove = pushop.force or pushop.ui.configbool(experimental, nonforwardmove = pushop.force or pushop.ui.configbool(experimental,
confignonforwardmove) confignonforwardmove)
scratchpart = getscratchbranchpart(pushop.repo, scratchparts = getscratchbranchparts(pushop.repo,
pushop.remote, pushop.remote,
pushop.outgoing, pushop.outgoing,
nonforwardmove, nonforwardmove,
pushop.ui, pushop.ui,
bookmark, bookmark,
create) create)
for scratchpart in scratchparts:
bundler.addpart(scratchpart) bundler.addpart(scratchpart)
def handlereply(op): def handlereply(op):
# server either succeeds or aborts; no code to read # server either succeeds or aborts; no code to read
pushop.cgresult = 1 pushop.cgresult = 1
return handlereply return handlereply
bundle2.capabilities[scratchbranchparttype] = () bundle2.capabilities[scratchbranchparttype] = ()

infinitepush/backupcommands.py

import re import re
import socket import socket
import stat import stat
import subprocess import subprocess
import time import time
from .bundleparts import ( from .bundleparts import (
getscratchbookmarkspart, getscratchbookmarkspart,
getscratchbranchpart, getscratchbranchparts,
) )
from mercurial import ( from mercurial import (
bundle2, bundle2,
changegroup, changegroup,
commands, commands,
discovery, discovery,
dispatch, dispatch,
encoding, encoding,
# See _deltaparent comments for details # See _deltaparent comments for details
wrapfunction(changegroup.cg2packer, 'deltaparent', _deltaparent) wrapfunction(changegroup.cg2packer, 'deltaparent', _deltaparent)
try: try:
bundler = _createbundler(ui, repo, other) bundler = _createbundler(ui, repo, other)
bundler.addparam("infinitepush", "True") bundler.addparam("infinitepush", "True")
backup = False backup = False
if outgoing and outgoing.missing: if outgoing and outgoing.missing:
backup = True backup = True
bundler.addpart(getscratchbranchpart(repo, other, outgoing, parts = getscratchbranchparts(repo, other, outgoing,
confignonforwardmove=False, confignonforwardmove=False,
ui=ui, bookmark=None, ui=ui, bookmark=None,
create=False)) create=False)
for part in parts:
bundler.addpart(part)
if bookmarkstobackup: if bookmarkstobackup:
backup = True backup = True
bundler.addpart(getscratchbookmarkspart(other, bookmarkstobackup)) bundler.addpart(getscratchbookmarkspart(other, bookmarkstobackup))
if backup: if backup:
_sendbundle(bundler, other) _sendbundle(bundler, other)
_writelocalbackupstate(repo.vfs, afterbackupheads, _writelocalbackupstate(repo.vfs, afterbackupheads,

infinitepush/bundleparts.py

extensions, extensions,
revsetlang, revsetlang,
) )
from mercurial.i18n import _ from mercurial.i18n import _
scratchbranchparttype = 'b2x:infinitepush' scratchbranchparttype = 'b2x:infinitepush'
scratchbookmarksparttype = 'b2x:infinitepushscratchbookmarks' scratchbookmarksparttype = 'b2x:infinitepushscratchbookmarks'
def getscratchbranchpart(repo, peer, outgoing, confignonforwardmove, def getscratchbranchparts(repo, peer, outgoing, confignonforwardmove,
ui, bookmark, create): ui, bookmark, create):
if not outgoing.missing: if not outgoing.missing:
raise error.Abort(_('no commits to push')) raise error.Abort(_('no commits to push'))
if scratchbranchparttype not in bundle2.bundle2caps(peer): if scratchbranchparttype not in bundle2.bundle2caps(peer):
raise error.Abort(_('no server support for %r') % scratchbranchparttype) raise error.Abort(_('no server support for %r') % scratchbranchparttype)
_validaterevset(repo, revsetlang.formatspec('%ln', outgoing.missing), _validaterevset(repo, revsetlang.formatspec('%ln', outgoing.missing),
if confignonforwardmove: if confignonforwardmove:
params['force'] = '1' params['force'] = '1'
# Do not send pushback bundle2 part with bookmarks if remotenames extension # Do not send pushback bundle2 part with bookmarks if remotenames extension
# is enabled. It will be handled manually in `_push()` # is enabled. It will be handled manually in `_push()`
if not isremotebooksenabled(ui): if not isremotebooksenabled(ui):
params['pushbackbookmarks'] = '1' params['pushbackbookmarks'] = '1'
parts = []
# .upper() marks this as a mandatory part: server will abort if there's no # .upper() marks this as a mandatory part: server will abort if there's no
# handler # handler
return bundle2.bundlepart( parts.append(bundle2.bundlepart(
scratchbranchparttype.upper(), scratchbranchparttype.upper(),
advisoryparams=params.iteritems(), advisoryparams=params.iteritems(),
data=cg) data=cg))
try:
treemod = extensions.find('treemanifest')
parts.append(treemod.createtreepackpart(repo, outgoing,
treemod.TREEGROUP_PARTTYPE2))
except KeyError:
pass
return parts
def getscratchbookmarkspart(peer, bookmarks): def getscratchbookmarkspart(peer, bookmarks):
if scratchbookmarksparttype not in bundle2.bundle2caps(peer): if scratchbookmarksparttype not in bundle2.bundle2caps(peer):
raise error.Abort( raise error.Abort(
_('no server support for %r') % scratchbookmarksparttype) _('no server support for %r') % scratchbookmarksparttype)
return bundle2.bundlepart( return bundle2.bundlepart(
scratchbookmarksparttype.upper(), scratchbookmarksparttype.upper(),

remotefilelog/shallowbundle.py

with maps to their linknodes with maps to their linknodes
{ manifest root node -> link node } { manifest root node -> link node }
- `fnodes` is a mapping of { filepath -> { node -> clnode } } - `fnodes` is a mapping of { filepath -> { node -> clnode } }
If fastpathlinkrev is false, we are responsible for populating If fastpathlinkrev is false, we are responsible for populating
fnodes. fnodes.
""" """
sendflat = self._repo.ui.configbool('treemanifest', 'sendflat', sendflat = self._repo.ui.configbool('treemanifest', 'sendflat',
True) True)
sendflat &= not self._repo.ui.configbool('treemanifest', 'treeonly')
stashUnsubmitted
Not Done

Is it related?

stash: Is it related?
durhamAuthorUnsubmitted
Not Done

This was needed to make the tests pass. Basically, if you're in treeonly mode you should also not be sending flat manifests. I think eventually I'll just delete the sendflat config.

durham: This was needed to make the tests pass. Basically, if you're in treeonly mode you should also…
if sendflat: if sendflat:
# In this code path, generating the manifests populates fnodes for # In this code path, generating the manifests populates fnodes for
# us. # us.
chunks = super(shallowcg1packer, self).generatemanifests( chunks = super(shallowcg1packer, self).generatemanifests(
commonrevs, commonrevs,
clrevorder, clrevorder,
fastpathlinkrev, fastpathlinkrev,
mfs, mfs,

tests/test-treemanifest-infinitepush.t

  • This file was added.
$ . "$TESTDIR/library.sh"
$ . "$TESTDIR/library-infinitepush.sh"
$ PYTHONPATH=$TESTDIR/..:$PYTHONPATH
$ export PYTHONPATH
$ setupcommon
$ hginit master
$ cd master
$ setupserver
$ cat >> .hg/hgrc <<EOF
> [extensions]
> pushrebase=$TESTDIR/../hgext3rd/pushrebase.py
> treemanifest=$TESTDIR/../treemanifest
> [remotefilelog]
> server=True
> [treemanifest]
> server=True
> EOF
$ echo x > x
$ hg commit -qAm 'add x'
$ cd ..
Push a scratch branch from one client
$ hgcloneshallow ssh://user@dummy/master client1 -q --config extensions.treemanifest=$TESTDIR/../treemanifest --config treemanifest.treeonly=True
1 trees fetched over * (glob)
1 trees fetched over * (glob)
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ cd client1
$ cat >> .hg/hgrc <<EOF
> [extensions]
> treemanifest=$TESTDIR/../treemanifest
>
> [remotefilelog]
> usefastdatapack=True
>
> [treemanifest]
> treeonly=True
> EOF
$ mkdir subdir
$ echo "my change" >> subdir/a
$ hg commit -qAm 'add subdir/a'
$ hg push --to scratch/foo --create
pushing to ssh://user@dummy/master
searching for changes
remote: pushing 1 commit:
remote: 02c12aef64ff add subdir/a
$ cd ..
Pull a scratch branch from another client
$ hgcloneshallow ssh://user@dummy/master client2 -q --config extensions.treemanifest=$TESTDIR/../treemanifest --config treemanifest.treeonly=True
$ cd client2
$ cat >> .hg/hgrc <<EOF
> [extensions]
> treemanifest=$TESTDIR/../treemanifest
>
> [remotefilelog]
> usefastdatapack=True
>
> [treemanifest]
> treeonly=True
> EOF
$ hg pull -r scratch/foo
pulling from ssh://user@dummy/master
searching for changes
adding changesets
adding manifests
adding file changes
added 1 changesets with 1 changes to 1 files
(run 'hg update' to get a working copy)
$ hg log -G
o changeset: 1:02c12aef64ff
| tag: tip
| user: test
| date: Thu Jan 01 00:00:00 1970 +0000
| summary: add subdir/a
|
@ changeset: 0:085784c01c08
user: test
date: Thu Jan 01 00:00:00 1970 +0000
summary: add x
$ hg cat -r tip subdir/a
my change
$ ls_l .hg/store
-rw-r--r-- 257 00changelog.i
-rw-r--r-- 108 00manifesttree.i
drwxr-xr-x data
drwxrwxr-x packs
-rw-r--r-- 43 phaseroots
-rw-r--r-- 18 undo
-rw-r--r-- 17 undo.backupfiles
-rw-r--r-- 0 undo.phaseroots
$ cd ..
Verify its not on the server
$ cd master
$ hg log -G
@ changeset: 0:085784c01c08
tag: tip
user: test
date: Thu Jan 01 00:00:00 1970 +0000
summary: add x