( )⚙ D5129 remotefilelog: rename wireproto methods and remaining capabilities

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D5129

remotefilelog: rename wireproto methods and remaining capabilities
ClosedPublic

Authored by durin42 on Oct 16 2018, 5:45 PM.

Details

Reviewers
indygreg
Group Reviewers
hg-reviewers
Commits
rHG466dd4d70bff: remotefilelog: rename wireproto methods and remaining capabilities
Summary

This is a mess, in part because there should be more constants
throughout. I know we typically do exp- instead of the x_ business in
this change, but I also had to use this in some function names, so I
figured until I can break that coupling I'd go with this. If it's too
unpleasant during review, let me know and I can probably clean it up
some more.

  1. no-check-commit due to new foo_bar naming - too hard to avoid right now :(

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

durin42 created this revision.Oct 16 2018, 5:45 PM
indygreg accepted this revision.Oct 17 2018, 2:56 AM
indygreg added a subscriber: indygreg.

Yuck. But the names don't matter much for an experimental feature. And the wireproto v1 peer batching code dictates certain things with regards to naming. So our hands are a bit tied here :/

This revision is now accepted and ready to land.Oct 17 2018, 2:56 AM
Closed by commit rHG466dd4d70bff: remotefilelog: rename wireproto methods and remaining capabilities (authored by durin42). · Explain WhyNov 5 2018, 2:56 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 12188

hgext/remotefilelog/fileserverclient.py

def getlocalkey(file, id): def getlocalkey(file, id):
pathhash = hashlib.sha1(file).hexdigest() pathhash = hashlib.sha1(file).hexdigest()
return os.path.join(pathhash, id) return os.path.join(pathhash, id)
def peersetup(ui, peer): def peersetup(ui, peer):
class remotefilepeer(peer.__class__): class remotefilepeer(peer.__class__):
@wireprotov1peer.batchable @wireprotov1peer.batchable
def getfile(self, file, node): def x_rfl_getfile(self, file, node):
if not self.capable('getfile'): if not self.capable('x_rfl_getfile'):
raise error.Abort( raise error.Abort(
'configured remotefile server does not support getfile') 'configured remotefile server does not support getfile')
f = wireprotov1peer.future() f = wireprotov1peer.future()
yield {'file': file, 'node': node}, f yield {'file': file, 'node': node}, f
code, data = f.value.split('\0', 1) code, data = f.value.split('\0', 1)
if int(code): if int(code):
raise error.LookupError(file, node, data) raise error.LookupError(file, node, data)
yield data yield data
@wireprotov1peer.batchable @wireprotov1peer.batchable
def getflogheads(self, path): def x_rfl_getflogheads(self, path):
if not self.capable('getflogheads'): if not self.capable('x_rfl_getflogheads'):
raise error.Abort('configured remotefile server does not ' raise error.Abort('configured remotefile server does not '
'support getflogheads') 'support getflogheads')
f = wireprotov1peer.future() f = wireprotov1peer.future()
yield {'path': path}, f yield {'path': path}, f
heads = f.value.split('\n') if f.value else [] heads = f.value.split('\n') if f.value else []
yield heads yield heads
def _updatecallstreamopts(self, command, opts): def _updatecallstreamopts(self, command, opts):
# #
# Over ssh, iterbatch isn't streamy because batch() wasn't # Over ssh, iterbatch isn't streamy because batch() wasn't
# explicitly designed as a streaming method. In the future we # explicitly designed as a streaming method. In the future we
# should probably introduce a streambatch() method upstream and # should probably introduce a streambatch() method upstream and
# use that for this. # use that for this.
with remote.commandexecutor() as e: with remote.commandexecutor() as e:
futures = [] futures = []
for m in missed: for m in missed:
futures.append(e.callcommand('getfile', { futures.append(e.callcommand('x_rfl_getfile', {
'file': idmap[m], 'file': idmap[m],
'node': m[-40:] 'node': m[-40:]
})) }))
for i, m in enumerate(missed): for i, m in enumerate(missed):
r = futures[i].result() r = futures[i].result()
futures[i] = None # release memory futures[i] = None # release memory
file_ = idmap[m] file_ = idmap[m]
node = m[-40:] node = m[-40:]
receivemissing(io.BytesIO('%d\n%s' % (len(r), r)), file_, node) receivemissing(io.BytesIO('%d\n%s' % (len(r), r)), file_, node)
progresstick() progresstick()
def _getfiles_optimistic( def _getfiles_optimistic(
remote, receivemissing, progresstick, missed, idmap, step): remote, receivemissing, progresstick, missed, idmap, step):
remote._callstream("getfiles") remote._callstream("x_rfl_getfiles")
i = 0 i = 0
pipeo = remote._pipeo pipeo = remote._pipeo
pipei = remote._pipei pipei = remote._pipei
while i < len(missed): while i < len(missed):
# issue a batch of requests # issue a batch of requests
start = i start = i
end = min(len(missed), start + step) end = min(len(missed), start + step)
i = end i = end
getfilestype = self.ui.config('remotefilelog', getfilestype = self.ui.config('remotefilelog',
'getfilestype') 'getfilestype')
if getfilestype == 'threaded': if getfilestype == 'threaded':
_getfiles = _getfiles_threaded _getfiles = _getfiles_threaded
else: else:
_getfiles = _getfiles_optimistic _getfiles = _getfiles_optimistic
_getfiles(remote, self.receivemissing, progresstick, _getfiles(remote, self.receivemissing, progresstick,
missed, idmap, step) missed, idmap, step)
elif remote.capable("getfile"): elif remote.capable("x_rfl_getfile"):
if remote.capable('batch'): if remote.capable('batch'):
batchdefault = 100 batchdefault = 100
else: else:
batchdefault = 10 batchdefault = 10
batchsize = self.ui.configint( batchsize = self.ui.configint(
'remotefilelog', 'batchsize', batchdefault) 'remotefilelog', 'batchsize', batchdefault)
_getfilesbatch( _getfilesbatch(
remote, self.receivemissing, progresstick, remote, self.receivemissing, progresstick,

hgext/remotefilelog/remotefilelogserver.py

""" """
global onetime global onetime
if onetime: if onetime:
return return
onetime = True onetime = True
# support file content requests # support file content requests
wireprotov1server.wireprotocommand( wireprotov1server.wireprotocommand(
'getflogheads', 'path', permission='pull')(getflogheads) 'x_rfl_getflogheads', 'path', permission='pull')(getflogheads)
wireprotov1server.wireprotocommand( wireprotov1server.wireprotocommand(
'getfiles', '', permission='pull')(getfiles) 'x_rfl_getfiles', '', permission='pull')(getfiles)
wireprotov1server.wireprotocommand( wireprotov1server.wireprotocommand(
'getfile', 'file node', permission='pull')(getfile) 'x_rfl_getfile', 'file node', permission='pull')(getfile)
class streamstate(object): class streamstate(object):
match = None match = None
shallowremote = False shallowremote = False
noflatmf = False noflatmf = False
state = streamstate() state = streamstate()
def stream_out_shallow(repo, proto, other): def stream_out_shallow(repo, proto, other):
# expose remotefilelog capabilities # expose remotefilelog capabilities
def _capabilities(orig, repo, proto): def _capabilities(orig, repo, proto):
caps = orig(repo, proto) caps = orig(repo, proto)
if ((constants.SHALLOWREPO_REQUIREMENT in repo.requirements or if ((constants.SHALLOWREPO_REQUIREMENT in repo.requirements or
ui.configbool('remotefilelog', 'server'))): ui.configbool('remotefilelog', 'server'))):
if isinstance(proto, _sshv1server): if isinstance(proto, _sshv1server):
# legacy getfiles method which only works over ssh # legacy getfiles method which only works over ssh
caps.append(constants.NETWORK_CAP_LEGACY_SSH_GETFILES) caps.append(constants.NETWORK_CAP_LEGACY_SSH_GETFILES)
caps.append('getflogheads') caps.append('x_rfl_getflogheads')
caps.append('getfile') caps.append('x_rfl_getfile')
return caps return caps
extensions.wrapfunction(wireprotov1server, '_capabilities', _capabilities) extensions.wrapfunction(wireprotov1server, '_capabilities', _capabilities)
def _adjustlinkrev(orig, self, *args, **kwargs): def _adjustlinkrev(orig, self, *args, **kwargs):
# When generating file blobs, taking the real path is too slow on large # When generating file blobs, taking the real path is too slow on large
# repos, so force it to just return the linkrev directly. # repos, so force it to just return the linkrev directly.
repo = self._repo repo = self._repo
if util.safehasattr(repo, 'forcelinkrev') and repo.forcelinkrev: if util.safehasattr(repo, 'forcelinkrev') and repo.forcelinkrev:
return self._filelog.linkrev(self._filelog.rev(self._filenode)) return self._filelog.linkrev(self._filelog.rev(self._filenode))
return orig(self, *args, **kwargs) return orig(self, *args, **kwargs)
extensions.wrapfunction( extensions.wrapfunction(
context.basefilectx, '_adjustlinkrev', _adjustlinkrev) context.basefilectx, '_adjustlinkrev', _adjustlinkrev)
def _iscmd(orig, cmd): def _iscmd(orig, cmd):
if cmd == 'getfiles': if cmd == 'x_rfl_getfiles':
return False return False
return orig(cmd) return orig(cmd)
extensions.wrapfunction(wireprotoserver, 'iscmd', _iscmd) extensions.wrapfunction(wireprotoserver, 'iscmd', _iscmd)
def _loadfileblob(repo, cachepath, path, node): def _loadfileblob(repo, cachepath, path, node):
filecachepath = os.path.join(cachepath, path, hex(node)) filecachepath = os.path.join(cachepath, path, hex(node))
if not os.path.exists(filecachepath) or os.path.getsize(filecachepath) == 0: if not os.path.exists(filecachepath) or os.path.getsize(filecachepath) == 0:

tests/remotefilelog-getflogheads.py

Extension printing a remotefilelog's heads Extension printing a remotefilelog's heads
Used for testing purpose Used for testing purpose
""" """
dest = repo.ui.expandpath('default') dest = repo.ui.expandpath('default')
peer = hg.peer(repo, {}, dest) peer = hg.peer(repo, {}, dest)
flogheads = peer.getflogheads(path) flogheads = peer.x_rfl_getflogheads(path)
if flogheads: if flogheads:
for head in flogheads: for head in flogheads:
ui.write(head + '\n') ui.write(head + '\n')
else: else:
ui.write(_('EMPTY\n')) ui.write(_('EMPTY\n'))

tests/test-remotefilelog-http.t

> EOF > EOF
$ echo x > x $ echo x > x
$ echo y > y $ echo y > y
$ hg commit -qAm x $ hg commit -qAm x
$ hg serve -p $HGPORT -d --pid-file=../hg1.pid -E ../error.log -A ../access.log $ hg serve -p $HGPORT -d --pid-file=../hg1.pid -E ../error.log -A ../access.log
Build a query string for later use: Build a query string for later use:
$ GET=`hg debugdata -m 0 | $PYTHON -c \ $ GET=`hg debugdata -m 0 | $PYTHON -c \
> 'import sys ; print [("?cmd=getfile&file=%s&node=%s" % tuple(s.split("\0"))) for s in sys.stdin.read().splitlines()][0]'` > 'import sys ; print [("?cmd=x_rfl_getfile&file=%s&node=%s" % tuple(s.split("\0"))) for s in sys.stdin.read().splitlines()][0]'`
$ cd .. $ cd ..
$ cat hg1.pid >> $DAEMON_PIDS $ cat hg1.pid >> $DAEMON_PIDS
$ hgcloneshallow http://localhost:$HGPORT/ shallow -q $ hgcloneshallow http://localhost:$HGPORT/ shallow -q
2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob) 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
$ grep getfile access.log $ grep getfile access.log
* "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=getfile+*node%3D1406e74118627694268417491f018a4a883152f0* (glob) * "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=x_rfl_getfile+*node%3D1406e74118627694268417491f018a4a883152f0* (glob)
Clear filenode cache so we can test fetching with a modified batch size Clear filenode cache so we can test fetching with a modified batch size
$ rm -r $TESTTMP/hgcache $ rm -r $TESTTMP/hgcache
Now do a fetch with a large batch size so we're sure it works Now do a fetch with a large batch size so we're sure it works
$ hgcloneshallow http://localhost:$HGPORT/ shallow-large-batch \ $ hgcloneshallow http://localhost:$HGPORT/ shallow-large-batch \
> --config remotefilelog.batchsize=1000 -q > --config remotefilelog.batchsize=1000 -q
2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob) 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
The 'remotefilelog' capability should *not* be exported over http(s), The 'remotefilelog' capability should *not* be exported over http(s),
as the getfile method it offers doesn't work with http. as the getfile method it offers doesn't work with http.
$ get-with-headers.py localhost:$HGPORT '?cmd=capabilities' | grep lookup | identifyrflcaps $ get-with-headers.py localhost:$HGPORT '?cmd=capabilities' | grep lookup | identifyrflcaps
getfile x_rfl_getfile
getflogheads x_rfl_getflogheads
$ get-with-headers.py localhost:$HGPORT '?cmd=hello' | grep lookup | identifyrflcaps $ get-with-headers.py localhost:$HGPORT '?cmd=hello' | grep lookup | identifyrflcaps
getfile x_rfl_getfile
getflogheads x_rfl_getflogheads
$ get-with-headers.py localhost:$HGPORT '?cmd=this-command-does-not-exist' | head -n 1 $ get-with-headers.py localhost:$HGPORT '?cmd=this-command-does-not-exist' | head -n 1
400 no such method: this-command-does-not-exist 400 no such method: this-command-does-not-exist
$ get-with-headers.py localhost:$HGPORT '?cmd=getfiles' | head -n 1 $ get-with-headers.py localhost:$HGPORT '?cmd=x_rfl_getfiles' | head -n 1
400 no such method: getfiles 400 no such method: x_rfl_getfiles
Verify serving from a shallow clone doesn't allow for remotefile Verify serving from a shallow clone doesn't allow for remotefile
fetches. This also serves to test the error handling for our batchable fetches. This also serves to test the error handling for our batchable
getfile RPC. getfile RPC.
$ cd shallow $ cd shallow
$ hg serve -p $HGPORT1 -d --pid-file=../hg2.pid -E ../error2.log $ hg serve -p $HGPORT1 -d --pid-file=../hg2.pid -E ../error2.log
$ cd .. $ cd ..

tests/test-remotefilelog-pull-noshallow.t

$ hg clone ssh://user@dummy/master child -q $ hg clone ssh://user@dummy/master child -q
We should see the remotefilelog capability here, which advertises that We should see the remotefilelog capability here, which advertises that
the server supports our custom getfiles method. the server supports our custom getfiles method.
$ cd master $ cd master
$ echo 'hello' | hg -R . serve --stdio | grep capa | identifyrflcaps $ echo 'hello' | hg -R . serve --stdio | grep capa | identifyrflcaps
exp-remotefilelog-ssh-getfiles-1 exp-remotefilelog-ssh-getfiles-1
getfile x_rfl_getfile
getflogheads x_rfl_getflogheads
$ echo 'capabilities' | hg -R . serve --stdio | identifyrflcaps ; echo $ echo 'capabilities' | hg -R . serve --stdio | identifyrflcaps ; echo
exp-remotefilelog-ssh-getfiles-1 exp-remotefilelog-ssh-getfiles-1
getfile x_rfl_getfile
getflogheads x_rfl_getflogheads
Pull to the child repository. Use our custom setupremotefilelog extension Pull to the child repository. Use our custom setupremotefilelog extension
to ensure that remotefilelog.onetimeclientsetup() gets triggered. (Without to ensure that remotefilelog.onetimeclientsetup() gets triggered. (Without
using chg it normally would not be run in this case since the local repository using chg it normally would not be run in this case since the local repository
is not shallow.) is not shallow.)
$ echo y > y $ echo y > y

tests/test-remotefilelog-push-pull.t

$ hgcloneshallow ssh://user@dummy/master shallow2 -q $ hgcloneshallow ssh://user@dummy/master shallow2 -q
We should see the remotefilelog capability here, which advertises that We should see the remotefilelog capability here, which advertises that
the server supports our custom getfiles method. the server supports our custom getfiles method.
$ cd master $ cd master
$ echo 'hello' | hg -R . serve --stdio | grep capa | identifyrflcaps $ echo 'hello' | hg -R . serve --stdio | grep capa | identifyrflcaps
exp-remotefilelog-ssh-getfiles-1 exp-remotefilelog-ssh-getfiles-1
getfile x_rfl_getfile
getflogheads x_rfl_getflogheads
$ echo 'capabilities' | hg -R . serve --stdio | identifyrflcaps ; echo $ echo 'capabilities' | hg -R . serve --stdio | identifyrflcaps ; echo
exp-remotefilelog-ssh-getfiles-1 exp-remotefilelog-ssh-getfiles-1
getfile x_rfl_getfile
getflogheads x_rfl_getflogheads
# pull to shallow from full # pull to shallow from full
$ echo y > y $ echo y > y
$ hg commit -qAm y $ hg commit -qAm y
$ cd ../shallow $ cd ../shallow
$ hg pull $ hg pull