This is an archive of the discontinued Mercurial Phabricator instance.

Differential D5378

remotefilelog: use progress helper in repack
ClosedPublic

Authored by martinvonz on Dec 5 2018, 12:39 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHGb80915b52476: remotefilelog: use progress helper in repack
Summary

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

martinvonz created this revision.Dec 5 2018, 12:39 PM
Closed by commit rHGb80915b52476: remotefilelog: use progress helper in repack (authored by martinvonz). · Explain WhyDec 6 2018, 6:41 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mhgext/remotefilelog/repack.py (31 lines)

Diff 12740

hgext/remotefilelog/repack.py

maxchainlen = ui.configint('packs', 'maxchainlen', 1000) maxchainlen = ui.configint('packs', 'maxchainlen', 1000)
byfile = {} byfile = {}
for entry in ledger.entries.itervalues(): for entry in ledger.entries.itervalues():
if entry.datasource: if entry.datasource:
byfile.setdefault(entry.filename, {})[entry.node] = entry byfile.setdefault(entry.filename, {})[entry.node] = entry
count = 0 count = 0
for filename, entries in sorted(byfile.iteritems()): repackprogress = ui.makeprogress(_("repacking data"), unit=self.unit,
ui.progress(_("repacking data"), count, unit=self.unit,
total=len(byfile)) total=len(byfile))
for filename, entries in sorted(byfile.iteritems()):
repackprogress.update(count)
ancestors = {} ancestors = {}
nodes = list(node for node in entries) nodes = list(node for node in entries)
nohistory = [] nohistory = []
buildprogress = ui.makeprogress(_("building history"), unit='nodes',
total=len(nodes))
for i, node in enumerate(nodes): for i, node in enumerate(nodes):
if node in ancestors: if node in ancestors:
continue continue
ui.progress(_("building history"), i, unit='nodes', buildprogress.update(i)
total=len(nodes))
try: try:
ancestors.update(self.fullhistory.getancestors(filename, ancestors.update(self.fullhistory.getancestors(filename,
node, known=ancestors)) node, known=ancestors))
except KeyError: except KeyError:
# Since we're packing data entries, we may not have the # Since we're packing data entries, we may not have the
# corresponding history entries for them. It's not a big # corresponding history entries for them. It's not a big
# deal, but the entries won't be delta'd perfectly. # deal, but the entries won't be delta'd perfectly.
nohistory.append(node) nohistory.append(node)
ui.progress(_("building history"), None) buildprogress.complete()
# Order the nodes children first, so we can produce reverse deltas # Order the nodes children first, so we can produce reverse deltas
orderednodes = list(reversed(self._toposort(ancestors))) orderednodes = list(reversed(self._toposort(ancestors)))
if len(nohistory) > 0: if len(nohistory) > 0:
ui.debug('repackdata: %d nodes without history\n' % ui.debug('repackdata: %d nodes without history\n' %
len(nohistory)) len(nohistory))
orderednodes.extend(sorted(nohistory)) orderednodes.extend(sorted(nohistory))
neworderednodes.append(node) neworderednodes.append(node)
orderednodes = neworderednodes orderednodes = neworderednodes
# Compute delta bases for nodes: # Compute delta bases for nodes:
deltabases = {} deltabases = {}
nobase = set() nobase = set()
referenced = set() referenced = set()
nodes = set(nodes) nodes = set(nodes)
for i, node in enumerate(orderednodes): processprogress = ui.makeprogress(_("processing nodes"),
ui.progress(_("processing nodes"), i, unit='nodes', unit='nodes',
total=len(orderednodes)) total=len(orderednodes))
for i, node in enumerate(orderednodes):
processprogress.update(i)
# Find delta base # Find delta base
# TODO: allow delta'ing against most recent descendant instead # TODO: allow delta'ing against most recent descendant instead
# of immediate child # of immediate child
deltatuple = deltabases.get(node, None) deltatuple = deltabases.get(node, None)
if deltatuple is None: if deltatuple is None:
deltabase, chainlen = nullid, 0 deltabase, chainlen = nullid, 0
deltabases[node] = (nullid, 0) deltabases[node] = (nullid, 0)
nobase.add(node) nobase.add(node)
# TODO: don't use the delta if it's larger than the fulltext # TODO: don't use the delta if it's larger than the fulltext
if constants.METAKEYSIZE not in meta: if constants.METAKEYSIZE not in meta:
meta[constants.METAKEYSIZE] = size meta[constants.METAKEYSIZE] = size
target.add(filename, node, deltabase, delta, meta) target.add(filename, node, deltabase, delta, meta)
entries[node].datarepacked = True entries[node].datarepacked = True
ui.progress(_("processing nodes"), None) processprogress.complete()
count += 1 count += 1
ui.progress(_("repacking data"), None) repackprogress.complete()
target.close(ledger=ledger) target.close(ledger=ledger)
def repackhistory(self, ledger, target): def repackhistory(self, ledger, target):
ui = self.repo.ui ui = self.repo.ui
byfile = {} byfile = {}
for entry in ledger.entries.itervalues(): for entry in ledger.entries.itervalues():
if entry.historysource: if entry.historysource:
byfile.setdefault(entry.filename, {})[entry.node] = entry byfile.setdefault(entry.filename, {})[entry.node] = entry
count = 0 progress = ui.makeprogress(_("repacking history"), unit=self.unit,
total=len(byfile))
for filename, entries in sorted(byfile.iteritems()): for filename, entries in sorted(byfile.iteritems()):
ancestors = {} ancestors = {}
nodes = list(node for node in entries) nodes = list(node for node in entries)
for node in nodes: for node in nodes:
if node in ancestors: if node in ancestors:
continue continue
ancestors.update(self.history.getancestors(filename, node, ancestors.update(self.history.getancestors(filename, node,
if copyfrom: if copyfrom:
dontprocess.add(p1) dontprocess.add(p1)
target.add(filename, node, p1, p2, linknode, copyfrom) target.add(filename, node, p1, p2, linknode, copyfrom)
if node in entries: if node in entries:
entries[node].historyrepacked = True entries[node].historyrepacked = True
count += 1 progress.increment()
ui.progress(_("repacking history"), count, unit=self.unit,
total=len(byfile))
ui.progress(_("repacking history"), None) progress.complete()
target.close(ledger=ledger) target.close(ledger=ledger)
def _toposort(self, ancestors): def _toposort(self, ancestors):
def parentfunc(node): def parentfunc(node):
p1, p2, linknode, copyfrom = ancestors[node] p1, p2, linknode, copyfrom = ancestors[node]
parents = [] parents = []
if p1 != nullid: if p1 != nullid:
parents.append(p1) parents.append(p1)