This is an archive of the discontinued Mercurial Phabricator instance.

Differential D5125

WIP: rename requirements and wireproto methods
AbandonedPublic

Authored by durin42 on Oct 16 2018, 3:18 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Summary

This is a mess, in part because there should be more constants
throughout.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

durin42 created this revision.Oct 16 2018, 3:18 PM
durin42 abandoned this revision.Oct 16 2018, 5:50 PM

Replaced by a more principled collection of refactors that has the benefit of working.

Revision Contents
Changeset List

Diff 12183

hgext/remotefilelog/__init__.py

def exchangepull(orig, repo, remote, *args, **kwargs): def exchangepull(orig, repo, remote, *args, **kwargs):
# Hook into the callstream/getbundle to insert bundle capabilities # Hook into the callstream/getbundle to insert bundle capabilities
# during a pull. # during a pull.
def localgetbundle(orig, source, heads=None, common=None, bundlecaps=None, def localgetbundle(orig, source, heads=None, common=None, bundlecaps=None,
**kwargs): **kwargs):
if not bundlecaps: if not bundlecaps:
bundlecaps = set() bundlecaps = set()
bundlecaps.add('remotefilelog') bundlecaps.add('x_rfl_remotefilelog')
return orig(source, heads=heads, common=common, bundlecaps=bundlecaps, return orig(source, heads=heads, common=common, bundlecaps=bundlecaps,
**kwargs) **kwargs)
if util.safehasattr(remote, '_callstream'): if util.safehasattr(remote, '_callstream'):
remote._localrepo = repo remote._localrepo = repo
elif util.safehasattr(remote, 'getbundle'): elif util.safehasattr(remote, 'getbundle'):
extensions.wrapfunction(remote, 'getbundle', localgetbundle) extensions.wrapfunction(remote, 'getbundle', localgetbundle)

hgext/remotefilelog/fileserverclient.py

def getlocalkey(file, id): def getlocalkey(file, id):
pathhash = hashlib.sha1(file).hexdigest() pathhash = hashlib.sha1(file).hexdigest()
return os.path.join(pathhash, id) return os.path.join(pathhash, id)
def peersetup(ui, peer): def peersetup(ui, peer):
class remotefilepeer(peer.__class__): class remotefilepeer(peer.__class__):
@wireprotov1peer.batchable @wireprotov1peer.batchable
def getfile(self, file, node): def x_rfl_getfile(self, file, node):
if not self.capable('getfile'): if not self.capable('x_rfl_getfile'):
raise error.Abort( raise error.Abort(
'configured remotefile server does not support getfile') 'configured remotefile server does not support getfile')
f = wireprotov1peer.future() f = wireprotov1peer.future()
yield {'file': file, 'node': node}, f yield {'file': file, 'node': node}, f
code, data = f.value.split('\0', 1) code, data = f.value.split('\0', 1)
if int(code): if int(code):
raise error.LookupError(file, node, data) raise error.LookupError(file, node, data)
yield data yield data
@wireprotov1peer.batchable @wireprotov1peer.batchable
def getflogheads(self, path): def x_rfl_getflogheads(self, path):
if not self.capable('getflogheads'): if not self.capable('x_rfl_getflogheads'):
raise error.Abort('configured remotefile server does not ' raise error.Abort('configured remotefile server does not '
'support getflogheads') 'support getflogheads')
f = wireprotov1peer.future() f = wireprotov1peer.future()
yield {'path': path}, f yield {'path': path}, f
heads = f.value.split('\n') if f.value else [] heads = f.value.split('\n') if f.value else []
yield heads yield heads
def _updatecallstreamopts(self, command, opts): def _updatecallstreamopts(self, command, opts):
# #
# Over ssh, iterbatch isn't streamy because batch() wasn't # Over ssh, iterbatch isn't streamy because batch() wasn't
# explicitly designed as a streaming method. In the future we # explicitly designed as a streaming method. In the future we
# should probably introduce a streambatch() method upstream and # should probably introduce a streambatch() method upstream and
# use that for this. # use that for this.
with remote.commandexecutor() as e: with remote.commandexecutor() as e:
futures = [] futures = []
for m in missed: for m in missed:
futures.append(e.callcommand('getfile', { futures.append(e.callcommand('x_rfl_getfile', {
'file': idmap[m], 'file': idmap[m],
'node': m[-40:] 'node': m[-40:]
})) }))
for i, m in enumerate(missed): for i, m in enumerate(missed):
r = futures[i].result() r = futures[i].result()
futures[i] = None # release memory futures[i] = None # release memory
file_ = idmap[m] file_ = idmap[m]
node = m[-40:] node = m[-40:]
receivemissing(io.BytesIO('%d\n%s' % (len(r), r)), file_, node) receivemissing(io.BytesIO('%d\n%s' % (len(r), r)), file_, node)
progresstick() progresstick()
def _getfiles_optimistic( def _getfiles_optimistic(
remote, receivemissing, progresstick, missed, idmap, step): remote, receivemissing, progresstick, missed, idmap, step):
remote._callstream("getfiles") remote._callstream("x_rfl_getfiles")
i = 0 i = 0
pipeo = remote._pipeo pipeo = remote._pipeo
pipei = remote._pipei pipei = remote._pipei
while i < len(missed): while i < len(missed):
# issue a batch of requests # issue a batch of requests
start = i start = i
end = min(len(missed), start + step) end = min(len(missed), start + step)
i = end i = end
# outputs # outputs
verbose = self.ui.verbose verbose = self.ui.verbose
self.ui.verbose = False self.ui.verbose = False
try: try:
with self._connect() as conn: with self._connect() as conn:
remote = conn.peer remote = conn.peer
# TODO: deduplicate this with the constant in # TODO: deduplicate this with the constant in
# shallowrepo # shallowrepo
if remote.capable("remotefilelog"): if remote.capable("x_rfl_remotefilelog"):
if not isinstance(remote, _sshv1peer): if not isinstance(remote, _sshv1peer):
raise error.Abort('remotefilelog requires ssh ' raise error.Abort('remotefilelog requires ssh '
'servers') 'servers')
step = self.ui.configint('remotefilelog', step = self.ui.configint('remotefilelog',
'getfilesstep') 'getfilesstep')
getfilestype = self.ui.config('remotefilelog', getfilestype = self.ui.config('remotefilelog',
'getfilestype') 'getfilestype')
if getfilestype == 'threaded': if getfilestype == 'threaded':
_getfiles = _getfiles_threaded _getfiles = _getfiles_threaded
else: else:
_getfiles = _getfiles_optimistic _getfiles = _getfiles_optimistic
_getfiles(remote, self.receivemissing, progresstick, _getfiles(remote, self.receivemissing, progresstick,
missed, idmap, step) missed, idmap, step)
elif remote.capable("getfile"): elif remote.capable("x_rfl_getfile"):
if remote.capable('batch'): if remote.capable('batch'):
batchdefault = 100 batchdefault = 100
else: else:
batchdefault = 10 batchdefault = 10
batchsize = self.ui.configint( batchsize = self.ui.configint(
'remotefilelog', 'batchsize', batchdefault) 'remotefilelog', 'batchsize', batchdefault)
_getfilesbatch( _getfilesbatch(
remote, self.receivemissing, progresstick, remote, self.receivemissing, progresstick,

hgext/remotefilelog/remotefilelogserver.py

""" """
global onetime global onetime
if onetime: if onetime:
return return
onetime = True onetime = True
# support file content requests # support file content requests
wireprotov1server.wireprotocommand( wireprotov1server.wireprotocommand(
'getflogheads', 'path', permission='pull')(getflogheads) 'x_rfl_getflogheads', 'path', permission='pull')(getflogheads)
wireprotov1server.wireprotocommand( wireprotov1server.wireprotocommand(
'getfiles', '', permission='pull')(getfiles) 'x_rfl_getfiles', '', permission='pull')(getfiles)
wireprotov1server.wireprotocommand( wireprotov1server.wireprotocommand(
'getfile', 'file node', permission='pull')(getfile) 'x_rfl_getfile', 'file node', permission='pull')(getfile)
class streamstate(object): class streamstate(object):
match = None match = None
shallowremote = False shallowremote = False
noflatmf = False noflatmf = False
state = streamstate() state = streamstate()
def stream_out_shallow(repo, proto, other): def stream_out_shallow(repo, proto, other):
# expose remotefilelog capabilities # expose remotefilelog capabilities
def _capabilities(orig, repo, proto): def _capabilities(orig, repo, proto):
caps = orig(repo, proto) caps = orig(repo, proto)
if ((shallowrepo.requirement in repo.requirements or if ((shallowrepo.requirement in repo.requirements or
ui.configbool('remotefilelog', 'server'))): ui.configbool('remotefilelog', 'server'))):
if isinstance(proto, _sshv1server): if isinstance(proto, _sshv1server):
# legacy getfiles method which only works over ssh # legacy getfiles method which only works over ssh
caps.append(shallowrepo.requirement) caps.append(shallowrepo.requirement)
caps.append('getflogheads') caps.append('x_rfl_getflogheads')
caps.append('getfile') caps.append('x_rfl_getfile')
return caps return caps
extensions.wrapfunction(wireprotov1server, '_capabilities', _capabilities) extensions.wrapfunction(wireprotov1server, '_capabilities', _capabilities)
def _adjustlinkrev(orig, self, *args, **kwargs): def _adjustlinkrev(orig, self, *args, **kwargs):
# When generating file blobs, taking the real path is too slow on large # When generating file blobs, taking the real path is too slow on large
# repos, so force it to just return the linkrev directly. # repos, so force it to just return the linkrev directly.
repo = self._repo repo = self._repo
if util.safehasattr(repo, 'forcelinkrev') and repo.forcelinkrev: if util.safehasattr(repo, 'forcelinkrev') and repo.forcelinkrev:
return self._filelog.linkrev(self._filelog.rev(self._filenode)) return self._filelog.linkrev(self._filelog.rev(self._filenode))
return orig(self, *args, **kwargs) return orig(self, *args, **kwargs)
extensions.wrapfunction( extensions.wrapfunction(
context.basefilectx, '_adjustlinkrev', _adjustlinkrev) context.basefilectx, '_adjustlinkrev', _adjustlinkrev)
def _iscmd(orig, cmd): def _iscmd(orig, cmd):
if cmd == 'getfiles': if cmd == 'x_rfl_getfiles':
return False return False
return orig(cmd) return orig(cmd)
extensions.wrapfunction(wireprotoserver, 'iscmd', _iscmd) extensions.wrapfunction(wireprotoserver, 'iscmd', _iscmd)
def _loadfileblob(repo, cachepath, path, node): def _loadfileblob(repo, cachepath, path, node):
filecachepath = os.path.join(cachepath, path, hex(node)) filecachepath = os.path.join(cachepath, path, hex(node))
if not os.path.exists(filecachepath) or os.path.getsize(filecachepath) == 0: if not os.path.exists(filecachepath) or os.path.getsize(filecachepath) == 0:

hgext/remotefilelog/shallowbundle.py

remotefilelog, remotefilelog,
shallowutil, shallowutil,
) )
NoFiles = 0 NoFiles = 0
LocalFiles = 1 LocalFiles = 1
AllFiles = 2 AllFiles = 2
requirement = "remotefilelog" requirement = "x_rfl_remotefilelog"
def shallowgroup(cls, self, nodelist, rlog, lookup, units=None, reorder=None): def shallowgroup(cls, self, nodelist, rlog, lookup, units=None, reorder=None):
if not isinstance(rlog, remotefilelog.remotefilelog): if not isinstance(rlog, remotefilelog.remotefilelog):
for c in super(cls, self).group(nodelist, rlog, lookup, for c in super(cls, self).group(nodelist, rlog, lookup,
units=units): units=units):
yield c yield c
return return

hgext/remotefilelog/shallowrepo.py

if util.safehasattr(util, '_hgexecutable'): if util.safehasattr(util, '_hgexecutable'):
# Before 5be286db # Before 5be286db
_hgexecutable = util.hgexecutable _hgexecutable = util.hgexecutable
else: else:
from mercurial.utils import procutil from mercurial.utils import procutil
_hgexecutable = procutil.hgexecutable _hgexecutable = procutil.hgexecutable
requirement = "remotefilelog" requirement = "x_rfl_remotefilelog"
_prefetching = _('prefetching') _prefetching = _('prefetching')
# These make*stores functions are global so that other extensions can replace # These make*stores functions are global so that other extensions can replace
# them. # them.
def makelocalstores(repo): def makelocalstores(repo):
"""In-repo stores, like .hg/store/data; can not be discarded.""" """In-repo stores, like .hg/store/data; can not be discarded."""
localpath = os.path.join(repo.svfs.vfs.base, 'data') localpath = os.path.join(repo.svfs.vfs.base, 'data')
if not os.path.exists(localpath): if not os.path.exists(localpath):

tests/test-remotefilelog-http.t

> EOF > EOF
$ echo x > x $ echo x > x
$ echo y > y $ echo y > y
$ hg commit -qAm x $ hg commit -qAm x
$ hg serve -p $HGPORT -d --pid-file=../hg1.pid -E ../error.log -A ../access.log $ hg serve -p $HGPORT -d --pid-file=../hg1.pid -E ../error.log -A ../access.log
Build a query string for later use: Build a query string for later use:
$ GET=`hg debugdata -m 0 | $PYTHON -c \ $ GET=`hg debugdata -m 0 | $PYTHON -c \
> 'import sys ; print [("?cmd=getfile&file=%s&node=%s" % tuple(s.split("\0"))) for s in sys.stdin.read().splitlines()][0]'` > 'import sys ; print [("?cmd=x_rfl_getfile&file=%s&node=%s" % tuple(s.split("\0"))) for s in sys.stdin.read().splitlines()][0]'`
$ cd .. $ cd ..
$ cat hg1.pid >> $DAEMON_PIDS $ cat hg1.pid >> $DAEMON_PIDS
$ hgcloneshallow http://localhost:$HGPORT/ shallow -q $ hgcloneshallow http://localhost:$HGPORT/ shallow -q
2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob) 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
$ grep getfile access.log $ grep getfile access.log
* "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=getfile+*node%3D1406e74118627694268417491f018a4a883152f0* (glob) * "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=x_rfl_getfile+*node%3D1406e74118627694268417491f018a4a883152f0* (glob)
Clear filenode cache so we can test fetching with a modified batch size Clear filenode cache so we can test fetching with a modified batch size
$ rm -r $TESTTMP/hgcache $ rm -r $TESTTMP/hgcache
Now do a fetch with a large batch size so we're sure it works Now do a fetch with a large batch size so we're sure it works
$ hgcloneshallow http://localhost:$HGPORT/ shallow-large-batch \ $ hgcloneshallow http://localhost:$HGPORT/ shallow-large-batch \
> --config remotefilelog.batchsize=1000 -q > --config remotefilelog.batchsize=1000 -q
2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob) 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
The 'remotefilelog' capability should *not* be exported over http(s), The 'remotefilelog' capability should *not* be exported over http(s),
as the getfile method it offers doesn't work with http. as the getfile method it offers doesn't work with http.
$ get-with-headers.py localhost:$HGPORT '?cmd=capabilities' | grep lookup | identifyrflcaps $ get-with-headers.py localhost:$HGPORT '?cmd=capabilities' | grep lookup | identifyrflcaps
getfile x_rfl_getfile
getflogheads x_rfl_getflogheads
$ get-with-headers.py localhost:$HGPORT '?cmd=hello' | grep lookup | identifyrflcaps $ get-with-headers.py localhost:$HGPORT '?cmd=hello' | grep lookup | identifyrflcaps
getfile x_rfl_getfile
getflogheads x_rfl_getflogheads
$ get-with-headers.py localhost:$HGPORT '?cmd=this-command-does-not-exist' | head -n 1 $ get-with-headers.py localhost:$HGPORT '?cmd=this-command-does-not-exist' | head -n 1
400 no such method: this-command-does-not-exist 400 no such method: this-command-does-not-exist
$ get-with-headers.py localhost:$HGPORT '?cmd=getfiles' | head -n 1 $ get-with-headers.py localhost:$HGPORT '?cmd=x_rfl_getfiles' | head -n 1
400 no such method: getfiles 400 no such method: x_rfl_getfiles
Verify serving from a shallow clone doesn't allow for remotefile Verify serving from a shallow clone doesn't allow for remotefile
fetches. This also serves to test the error handling for our batchable fetches. This also serves to test the error handling for our batchable
getfile RPC. getfile RPC.
$ cd shallow $ cd shallow
$ hg serve -p $HGPORT1 -d --pid-file=../hg2.pid -E ../error2.log $ hg serve -p $HGPORT1 -d --pid-file=../hg2.pid -E ../error2.log
$ cd .. $ cd ..