This is an archive of the discontinued Mercurial Phabricator instance.

Differential D5381

remotefilelog: use progress helper in fileserverclient
ClosedPublic

Authored by martinvonz on Dec 5 2018, 12:39 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHGe58cd7ede1c3: remotefilelog: use progress helper in fileserverclient
Summary

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Revision Contents
Changeset List

Diff 12720

hgext/remotefilelog/fileserverclient.py

# Statistics for debugging # Statistics for debugging
fetchcost = 0 fetchcost = 0
fetches = 0 fetches = 0
fetched = 0 fetched = 0
fetchmisses = 0 fetchmisses = 0
_lfsmod = None _lfsmod = None
_downloading = _('downloading')
def getcachekey(reponame, file, id): def getcachekey(reponame, file, id):
pathhash = node.hex(hashlib.sha1(file).digest()) pathhash = node.hex(hashlib.sha1(file).digest())
return os.path.join(reponame, pathhash[:2], pathhash[2:], id) return os.path.join(reponame, pathhash[:2], pathhash[2:], id)
def getlocalkey(file, id): def getlocalkey(file, id):
pathhash = node.hex(hashlib.sha1(file).digest()) pathhash = node.hex(hashlib.sha1(file).digest())
return os.path.join(pathhash, id) return os.path.join(pathhash, id)
if self.cacheprocesspasspath: if self.cacheprocesspasspath:
request += file + '\0' request += file + '\0'
request += fullid + "\n" request += fullid + "\n"
idmap[fullid] = file idmap[fullid] = file
cache.request(request) cache.request(request)
total = count total = count
self.ui.progress(_downloading, 0, total=count) progress = self.ui.makeprogress(_('downloading'), total=count)
progress.update(0)
missed = [] missed = []
count = 0 count = 0
while True: while True:
missingid = cache.receiveline() missingid = cache.receiveline()
if not missingid: if not missingid:
missedset = set(missed) missedset = set(missed)
for missingid in idmap: for missingid in idmap:
if not missingid in missedset: if not missingid in missedset:
missed.append(missingid) missed.append(missingid)
self.ui.warn(_("warning: cache connection closed early - " + self.ui.warn(_("warning: cache connection closed early - " +
"falling back to server\n")) "falling back to server\n"))
break break
if missingid == "0": if missingid == "0":
break break
if missingid.startswith("_hits_"): if missingid.startswith("_hits_"):
# receive progress reports # receive progress reports
parts = missingid.split("_") parts = missingid.split("_")
count += int(parts[2]) count += int(parts[2])
self.ui.progress(_downloading, count, total=total) progress.update(count)
continue continue
missed.append(missingid) missed.append(missingid)
global fetchmisses global fetchmisses
fetchmisses += len(missed) fetchmisses += len(missed)
count = [total - len(missed)] count = [total - len(missed)]
fromcache = count[0] fromcache = count[0]
self.ui.progress(_downloading, count[0], total=total) progress.update(count[0], total=total)
self.ui.log("remotefilelog", "remote cache hit rate is %r of %r\n", self.ui.log("remotefilelog", "remote cache hit rate is %r of %r\n",
count[0], total, hit=count[0], total=total) count[0], total, hit=count[0], total=total)
oldumask = os.umask(0o002) oldumask = os.umask(0o002)
try: try:
# receive cache misses from master # receive cache misses from master
if missed: if missed:
def progresstick(): def progresstick():
count[0] += 1 count[0] += 1
self.ui.progress(_downloading, count[0], total=total) progress.update(count[0])
# When verbose is true, sshpeer prints 'running ssh...' # When verbose is true, sshpeer prints 'running ssh...'
# to stdout, which can interfere with some command # to stdout, which can interfere with some command
# outputs # outputs
verbose = self.ui.verbose verbose = self.ui.verbose
self.ui.verbose = False self.ui.verbose = False
try: try:
with self._connect() as conn: with self._connect() as conn:
remote = conn.peer remote = conn.peer
raise raise
finally: finally:
self.ui.verbose = verbose self.ui.verbose = verbose
# send to memcache # send to memcache
count[0] = len(missed) count[0] = len(missed)
request = "set\n%d\n%s\n" % (count[0], "\n".join(missed)) request = "set\n%d\n%s\n" % (count[0], "\n".join(missed))
cache.request(request) cache.request(request)
self.ui.progress(_downloading, None) progress.complete()
# mark ourselves as a user of this cache # mark ourselves as a user of this cache
writedata.markrepo(self.repo.path) writedata.markrepo(self.repo.path)
finally: finally:
os.umask(oldumask) os.umask(oldumask)
def receivemissing(self, pipe, filename, node): def receivemissing(self, pipe, filename, node):
line = pipe.readline()[:-1] line = pipe.readline()[:-1]