This is an archive of the discontinued Mercurial Phabricator instance.

Differential D1455

treemanifest: use connection pool for hg pull
ClosedPublic

Authored by durham on Nov 20 2017, 10:23 AM.
Tags
None
Subscribers
quark

Details

Reviewers
quark
Group Reviewers
Restricted Project
Commits
rFBHGXe64038f89490: treemanifest: use connection pool for hg pull
Summary

Previously hg pull created it's own ssh connection, then the tree prefetch
created one as well. Let's change hg pull to use a connection from the pool when
possible, so it can be reused by treemanifest later.

Diff Detail

Repository
rFBHGX Facebook Mercurial Extensions
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

durham created this revision.Nov 20 2017, 10:23 AM
Herald added a reviewer: Restricted Project. · View Herald TranscriptNov 20 2017, 10:23 AM
quark accepted this revision.Nov 21 2017, 4:48 AM
quark added a subscriber: quark.
quark added inline comments.
treemanifest/__init__.py
1246

Maybe connection pool could take an argument group connections by their addresses.

This revision is now accepted and ready to land.Nov 21 2017, 4:48 AM
durham added inline comments.Nov 21 2017, 9:52 AM
treemanifest/__init__.py
1246

It already does for the remote path/uri, but it doesn't group by ssh or remotecmd.

Closed by commit rFBHGXe64038f89490: treemanifest: use connection pool for hg pull (authored by durham). · Explain WhyNov 21 2017, 5:12 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mtreemanifest/__init__.py (40 lines)

Diff 3751

treemanifest/__init__.py

from mercurial import ( from mercurial import (
bundle2, bundle2,
bundlerepo, bundlerepo,
changegroup, changegroup,
commands, commands,
error, error,
exchange, exchange,
extensions, extensions,
hg,
localrepo, localrepo,
manifest, manifest,
mdiff, mdiff,
phases, phases,
policy, policy,
registrar, registrar,
repair, repair,
revlog, revlog,
base = opts.get('base') base = opts.get('base')
if base is not None: if base is not None:
basemfnode.add(repo[base].manifestnode()) basemfnode.add(repo[base].manifestnode())
_prefetchtrees(repo, '', mfnodes, basemfnode, []) _prefetchtrees(repo, '', mfnodes, basemfnode, [])
def _prefetchtrees(repo, rootdir, mfnodes, basemfnodes, directories): def _prefetchtrees(repo, rootdir, mfnodes, basemfnodes, directories):
# If possible, use remotefilelog's more expressive fallbackpath # If possible, use remotefilelog's more expressive fallbackpath
if util.safehasattr(repo, 'fallbackpath'): fallbackpath = getfallbackpath(repo)
fallbackpath = repo.fallbackpath
else:
fallbackpath = repo.ui.config('paths', 'default')
start = time.time() start = time.time()
with repo.connectionpool.get(fallbackpath) as conn: with repo.connectionpool.get(fallbackpath) as conn:
remote = conn.peer remote = conn.peer
_gettrees(repo, remote, rootdir, mfnodes, basemfnodes, directories, _gettrees(repo, remote, rootdir, mfnodes, basemfnodes, directories,
start) start)
def _gettrees(repo, remote, rootdir, mfnodes, basemfnodes, directories, start): def _gettrees(repo, remote, rootdir, mfnodes, basemfnodes, directories, start):
partname, partname,
data = packstream) data = packstream)
part.addparam('version', '1') part.addparam('version', '1')
part.addparam('cache', 'False') part.addparam('cache', 'False')
part.addparam('category', PACK_CATEGORY) part.addparam('category', PACK_CATEGORY)
return part return part
def getfallbackpath(repo):
if util.safehasattr(repo, 'fallbackpath'):
return repo.fallbackpath
else:
return repo.ui.config('paths', 'default')
def pull(orig, ui, repo, *pats, **opts): def pull(orig, ui, repo, *pats, **opts):
# If special arguments were passed, we can't reuse the pull connection
if opts.get('ssh') or opts.get('remotecmd'):
quarkUnsubmitted
Not Done

Maybe connection pool could take an argument group connections by their addresses.

quark: Maybe connection pool could take an argument group connections by their addresses.
durhamAuthorUnsubmitted
Not Done

It already does for the remote path/uri, but it doesn't group by ssh or remotecmd.

durham: It already does for the remote path/uri, but it doesn't group by ssh or remotecmd.
return orig(ui, repo, *pats, **opts)
# Use a connection from the connection pool for the pull, so it can be
# reused by future requests, like tree prefetching.
fallbackpath = getfallbackpath(repo)
with repo.connectionpool.get(fallbackpath) as conn:
peer = conn.peer
def wrappedpeer(orig, uiorrepo, opts, path, *args, **kwargs):
if path == fallbackpath:
return peer
else:
return orig(uiorrepo, opts, path, *args, **kwargs)
with extensions.wrappedfunction(hg, 'peer', wrappedpeer):
result = orig(ui, repo, *pats, **opts) result = orig(ui, repo, *pats, **opts)
if not treeenabled(repo.ui): if not treeenabled(repo.ui):
return result return result
_postpullprefetch(ui, repo)
return result
def _postpullprefetch(ui, repo):
repo = repo.unfiltered() repo = repo.unfiltered()
ctxs = [] ctxs = []
mfstore = repo.manifestlog.datastore mfstore = repo.manifestlog.datastore
# prefetch if it's configured # prefetch if it's configured
prefetchcount = ui.configint('treemanifest', 'pullprefetchcount', None) prefetchcount = ui.configint('treemanifest', 'pullprefetchcount', None)
if prefetchcount: if prefetchcount:
# If we have no base nodes, scan the change log looking for a # If we have no base nodes, scan the change log looking for a
# semi-recent manifest node to treat as the base. # semi-recent manifest node to treat as the base.
if not basemfnodes: if not basemfnodes:
basemfnodes = _findrecenttree(repo, len(repo.changelog) - 1) basemfnodes = _findrecenttree(repo, len(repo.changelog) - 1)
_prefetchtrees(repo, '', mfnodes, basemfnodes, []) _prefetchtrees(repo, '', mfnodes, basemfnodes, [])
return result
def _findrecenttree(repo, startrev): def _findrecenttree(repo, startrev):
cl = repo.changelog cl = repo.changelog
mfstore = repo.manifestlog.datastore mfstore = repo.manifestlog.datastore
phasecache = repo._phasecache phasecache = repo._phasecache
maxrev = min(len(cl) - 1, startrev + BASENODESEARCHMAX) maxrev = min(len(cl) - 1, startrev + BASENODESEARCHMAX)
minrev = max(0, startrev - BASENODESEARCHMAX) minrev = max(0, startrev - BASENODESEARCHMAX)
# Look up and down from the given rev # Look up and down from the given rev