This is an archive of the discontinued Mercurial Phabricator instance.

Differential D1455

treemanifest: use connection pool for hg pull
ClosedPublic

Authored by durham on Nov 20 2017, 10:23 AM.
Tags
None
Subscribers
quark

Details

Reviewers
quark
Group Reviewers
Restricted Project
Commits
rFBHGXe64038f89490: treemanifest: use connection pool for hg pull
Summary

Previously hg pull created it's own ssh connection, then the tree prefetch
created one as well. Let's change hg pull to use a connection from the pool when
possible, so it can be reused by treemanifest later.

Diff Detail

Repository
rFBHGX Facebook Mercurial Extensions
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

durham created this revision.Nov 20 2017, 10:23 AM
Herald added a reviewer: Restricted Project. · View Herald TranscriptNov 20 2017, 10:23 AM
quark accepted this revision.Nov 21 2017, 4:48 AM
quark added a subscriber: quark.
quark added inline comments.
treemanifest/__init__.py
1247

Maybe connection pool could take an argument group connections by their addresses.

This revision is now accepted and ready to land.Nov 21 2017, 4:48 AM
durham added inline comments.Nov 21 2017, 9:52 AM
treemanifest/__init__.py
1247

It already does for the remote path/uri, but it doesn't group by ssh or remotecmd.

Closed by commit rFBHGXe64038f89490: treemanifest: use connection pool for hg pull (authored by durham). · Explain WhyNov 21 2017, 5:12 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mtreemanifest/__init__.py (40 lines)

Diff 3645

treemanifest/__init__.py

from mercurial import ( from mercurial import (
bundle2, bundle2,
bundlerepo, bundlerepo,
changegroup, changegroup,
commands, commands,
error, error,
exchange, exchange,
extensions, extensions,
hg,
localrepo, localrepo,
manifest, manifest,
mdiff, mdiff,
phases, phases,
policy, policy,
registrar, registrar,
repair, repair,
revlog, revlog,
base = opts.get('base') base = opts.get('base')
if base is not None: if base is not None:
basemfnode.add(repo[base].manifestnode()) basemfnode.add(repo[base].manifestnode())
_prefetchtrees(repo, '', mfnodes, basemfnode, []) _prefetchtrees(repo, '', mfnodes, basemfnode, [])
def _prefetchtrees(repo, rootdir, mfnodes, basemfnodes, directories): def _prefetchtrees(repo, rootdir, mfnodes, basemfnodes, directories):
# If possible, use remotefilelog's more expressive fallbackpath # If possible, use remotefilelog's more expressive fallbackpath
if util.safehasattr(repo, 'fallbackpath'): fallbackpath = getfallbackpath(repo)
fallbackpath = repo.fallbackpath
else:
fallbackpath = repo.ui.config('paths', 'default')
start = time.time() start = time.time()
with repo.connectionpool.get(fallbackpath) as conn: with repo.connectionpool.get(fallbackpath) as conn:
remote = conn.peer remote = conn.peer
_gettrees(repo, remote, rootdir, mfnodes, basemfnodes, directories, _gettrees(repo, remote, rootdir, mfnodes, basemfnodes, directories,
start) start)
def _gettrees(repo, remote, rootdir, mfnodes, basemfnodes, directories, start): def _gettrees(repo, remote, rootdir, mfnodes, basemfnodes, directories, start):
partname, partname,
data = packstream) data = packstream)
part.addparam('version', '1') part.addparam('version', '1')
part.addparam('cache', 'False') part.addparam('cache', 'False')
part.addparam('category', PACK_CATEGORY) part.addparam('category', PACK_CATEGORY)
return part return part
def getfallbackpath(repo):
if util.safehasattr(repo, 'fallbackpath'):
return repo.fallbackpath
else:
return repo.ui.config('paths', 'default')
def pull(orig, ui, repo, *pats, **opts): def pull(orig, ui, repo, *pats, **opts):
# If special arguments were passed, we can't reuse the pull connection
if opts.get('ssh') or opts.get('remotecmd'):
quarkUnsubmitted
Not Done

Maybe connection pool could take an argument group connections by their addresses.

quark: Maybe connection pool could take an argument group connections by their addresses.
durhamAuthorUnsubmitted
Not Done

It already does for the remote path/uri, but it doesn't group by ssh or remotecmd.

durham: It already does for the remote path/uri, but it doesn't group by ssh or remotecmd.
return orig(ui, repo, *pats, **opts)
# Use a connection from the connection pool for the pull, so it can be
# reused by future requests, like tree prefetching.
fallbackpath = getfallbackpath(repo)
with repo.connectionpool.get(fallbackpath) as conn:
peer = conn.peer
def wrappedpeer(orig, uiorrepo, opts, path, *args, **kwargs):
if path == fallbackpath:
return peer
else:
return orig(uiorrepo, opts, path, *args, **kwargs)
with extensions.wrappedfunction(hg, 'peer', wrappedpeer):
result = orig(ui, repo, *pats, **opts) result = orig(ui, repo, *pats, **opts)
if not treeenabled(repo.ui): if not treeenabled(repo.ui):
return result return result
_postpullprefetch(ui, repo)
return result
def _postpullprefetch(ui, repo):
repo = repo.unfiltered() repo = repo.unfiltered()
ctxs = [] ctxs = []
mfstore = repo.manifestlog.datastore mfstore = repo.manifestlog.datastore
# prefetch if it's configured # prefetch if it's configured
prefetchcount = ui.configint('treemanifest', 'pullprefetchcount', None) prefetchcount = ui.configint('treemanifest', 'pullprefetchcount', None)
if prefetchcount: if prefetchcount:
# If we have no base nodes, scan the change log looking for a # If we have no base nodes, scan the change log looking for a
# semi-recent manifest node to treat as the base. # semi-recent manifest node to treat as the base.
if not basemfnodes: if not basemfnodes:
basemfnodes = _findrecenttree(repo, len(repo.changelog) - 1) basemfnodes = _findrecenttree(repo, len(repo.changelog) - 1)
_prefetchtrees(repo, '', mfnodes, basemfnodes, []) _prefetchtrees(repo, '', mfnodes, basemfnodes, [])
return result
def _findrecenttree(repo, startrev): def _findrecenttree(repo, startrev):
cl = repo.changelog cl = repo.changelog
mfstore = repo.manifestlog.datastore mfstore = repo.manifestlog.datastore
phasecache = repo._phasecache phasecache = repo._phasecache
maxrev = min(len(cl) - 1, startrev + BASENODESEARCHMAX) maxrev = min(len(cl) - 1, startrev + BASENODESEARCHMAX)
minrev = max(0, startrev - BASENODESEARCHMAX) minrev = max(0, startrev - BASENODESEARCHMAX)
# Look up and down from the given rev # Look up and down from the given rev