This is an archive of the discontinued Mercurial Phabricator instance.

Differential D5376

remotefilelog: use progress helper in basestore
ClosedPublic

Authored by martinvonz on Dec 5 2018, 12:39 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHGad21fbcb1ba5: remotefilelog: use progress helper in basestore
Summary

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

martinvonz created this revision.Dec 5 2018, 12:39 PM
Closed by commit rHGad21fbcb1ba5: remotefilelog: use progress helper in basestore (authored by martinvonz). · Explain WhyDec 6 2018, 6:41 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mhgext/remotefilelog/basestore.py (24 lines)

Diff 12715

hgext/remotefilelog/basestore.py

for node in nodes: for node in nodes:
ledger.markdataentry(self, filename, node) ledger.markdataentry(self, filename, node)
ledger.markhistoryentry(self, filename, node) ledger.markhistoryentry(self, filename, node)
def cleanup(self, ledger): def cleanup(self, ledger):
ui = self.ui ui = self.ui
entries = ledger.sources.get(self, []) entries = ledger.sources.get(self, [])
count = 0 count = 0
progress = ui.makeprogress(_("cleaning up"), unit="files",
total=len(entries))
for entry in entries: for entry in entries:
if entry.gced or (entry.datarepacked and entry.historyrepacked): if entry.gced or (entry.datarepacked and entry.historyrepacked):
ui.progress(_("cleaning up"), count, unit="files", progress.update(count)
total=len(entries))
path = self._getfilepath(entry.filename, entry.node) path = self._getfilepath(entry.filename, entry.node)
util.tryunlink(path) util.tryunlink(path)
count += 1 count += 1
ui.progress(_("cleaning up"), None) progress.complete()
# Clean up the repo cache directory. # Clean up the repo cache directory.
self._cleanupdirectory(self._getrepocachepath()) self._cleanupdirectory(self._getrepocachepath())
# BELOW THIS ARE NON-STANDARD APIS # BELOW THIS ARE NON-STANDARD APIS
def _cleanupdirectory(self, rootdir): def _cleanupdirectory(self, rootdir):
"""Removes the empty directories and unnecessary files within the root """Removes the empty directories and unnecessary files within the root
except (ValueError, RuntimeError): except (ValueError, RuntimeError):
pass pass
return False return False
def gc(self, keepkeys): def gc(self, keepkeys):
ui = self.ui ui = self.ui
cachepath = self._path cachepath = self._path
_removing = _("removing unnecessary files")
_truncating = _("enforcing cache limit")
# prune cache # prune cache
import Queue import Queue
queue = Queue.PriorityQueue() queue = Queue.PriorityQueue()
originalsize = 0 originalsize = 0
size = 0 size = 0
count = 0 count = 0
removed = 0 removed = 0
# keep files newer than a day even if they aren't needed # keep files newer than a day even if they aren't needed
limit = time.time() - (60 * 60 * 24) limit = time.time() - (60 * 60 * 24)
ui.progress(_removing, count, unit="files") progress = ui.makeprogress(_("removing unnecessary files"),
unit="files")
progress.update(0)
for root, dirs, files in os.walk(cachepath): for root, dirs, files in os.walk(cachepath):
for file in files: for file in files:
if file == 'repos': if file == 'repos':
continue continue
# Don't delete pack files # Don't delete pack files
if '/packs/' in root: if '/packs/' in root:
continue continue
ui.progress(_removing, count, unit="files") progress.update(count)
path = os.path.join(root, file) path = os.path.join(root, file)
key = os.path.relpath(path, cachepath) key = os.path.relpath(path, cachepath)
count += 1 count += 1
try: try:
pathstat = os.stat(path) pathstat = os.stat(path)
except OSError as e: except OSError as e:
# errno.ENOENT = no such file or directory # errno.ENOENT = no such file or directory
if e.errno != errno.ENOENT: if e.errno != errno.ENOENT:
# errno.ENOENT = no such file or directory # errno.ENOENT = no such file or directory
if e.errno != errno.ENOENT: if e.errno != errno.ENOENT:
raise raise
msg = _("warning: file %s was removed by another " msg = _("warning: file %s was removed by another "
"process\n") "process\n")
ui.warn(msg % path) ui.warn(msg % path)
continue continue
removed += 1 removed += 1
ui.progress(_removing, None) progress.complete()
# remove oldest files until under limit # remove oldest files until under limit
limit = ui.configbytes("remotefilelog", "cachelimit") limit = ui.configbytes("remotefilelog", "cachelimit")
if size > limit: if size > limit:
excess = size - limit excess = size - limit
progress = ui.makeprogress(_("enforcing cache limit"), unit="bytes",
total=excess)
removedexcess = 0 removedexcess = 0
while queue and size > limit and size > 0: while queue and size > limit and size > 0:
ui.progress(_truncating, removedexcess, unit="bytes", progress.update(removedexcess)
total=excess)
atime, oldpath, oldpathstat = queue.get() atime, oldpath, oldpathstat = queue.get()
try: try:
shallowutil.unlinkfile(oldpath) shallowutil.unlinkfile(oldpath)
except OSError as e: except OSError as e:
# errno.ENOENT = no such file or directory # errno.ENOENT = no such file or directory
if e.errno != errno.ENOENT: if e.errno != errno.ENOENT:
raise raise
msg = _("warning: file %s was removed by another process\n") msg = _("warning: file %s was removed by another process\n")
ui.warn(msg % oldpath) ui.warn(msg % oldpath)
size -= oldpathstat.st_size size -= oldpathstat.st_size
removed += 1 removed += 1
removedexcess += oldpathstat.st_size removedexcess += oldpathstat.st_size
ui.progress(_truncating, None) progress.complete()
ui.status(_("finished: removed %s of %s files (%0.2f GB to %0.2f GB)\n") ui.status(_("finished: removed %s of %s files (%0.2f GB to %0.2f GB)\n")
% (removed, count, % (removed, count,
float(originalsize) / 1024.0 / 1024.0 / 1024.0, float(originalsize) / 1024.0 / 1024.0 / 1024.0,
float(size) / 1024.0 / 1024.0 / 1024.0)) float(size) / 1024.0 / 1024.0 / 1024.0))
class baseunionstore(object): class baseunionstore(object):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):