This is an archive of the discontinued Mercurial Phabricator instance.

Differential D4129

includematcher: separate "parents" from "dirs"
ClosedPublic

Authored by spectral on Aug 6 2018, 2:50 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG5a7df82de142: includematcher: separate "parents" from "dirs"
Summary

A future patch will make use of this separation so that we can make more
intelligent decisions about what to investigate/load when the matcher is in use.

Currently, even with this patch, we typically use the 'visitdir' call to identify if
we can skip some directory, something along the lines of:

for f in all_items:
    if match.visitdir(f):
       <do stuff>

This can be slower than we'd like if there are a lot of items; it requires N
calls to match.visitdir in the best case. Commonly, especially with 'narrow',
we have a situation where we do some work for the directory, possibly just
loading it from disk (when using treemanifests) and then check if we should be
interacting with it at all, which can be a huge slowdown in some pathological
cases.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

spectral created this revision.Aug 6 2018, 2:50 PM
spectral planned changes to this revision.Aug 6 2018, 3:56 PM
spectral updated this revision to Diff 10016.Aug 6 2018, 5:17 PM
yuja added a subscriber: yuja.Aug 8 2018, 11:16 AM

-def _rootsanddirs(kindpats):
+def _rootsdirsandparents(kindpats):

'''Returns roots and exact directories from patterns.
roots are directories to match recursively, whereas exact directories should
be matched non-recursively. The returned (roots, dirs) tuple will also
include directories that need to be implicitly considered as either, such as
parent directories.

Can you update the doc to reflect this change?

Closed by commit rHG5a7df82de142: includematcher: separate "parents" from "dirs" (authored by spectral). · Explain WhyAug 8 2018, 11:19 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mmercurial/match.py (33 lines)

Diff 10016

mercurial/match.py

class includematcher(basematcher): class includematcher(basematcher):
def __init__(self, root, cwd, kindpats, listsubrepos=False, badfn=None): def __init__(self, root, cwd, kindpats, listsubrepos=False, badfn=None):
super(includematcher, self).__init__(root, cwd, badfn) super(includematcher, self).__init__(root, cwd, badfn)
self._pats, self.matchfn = _buildmatch(kindpats, '(?:/|$)', self._pats, self.matchfn = _buildmatch(kindpats, '(?:/|$)',
listsubrepos, root) listsubrepos, root)
self._prefix = _prefix(kindpats) self._prefix = _prefix(kindpats)
roots, dirs = _rootsanddirs(kindpats) roots, dirs, parents = _rootsdirsandparents(kindpats)
# roots are directories which are recursively included. # roots are directories which are recursively included.
self._roots = set(roots) self._roots = set(roots)
# dirs are directories which are non-recursively included. # dirs are directories which are non-recursively included.
self._dirs = set(dirs) self._dirs = set(dirs)
# parents are directories which are non-recursively included because
# they are needed to get to items in _dirs or _roots.
self._parents = set(parents)
def visitdir(self, dir): def visitdir(self, dir):
if self._prefix and dir in self._roots: if self._prefix and dir in self._roots:
return 'all' return 'all'
return ('.' in self._roots or return ('.' in self._roots or
dir in self._roots or dir in self._roots or
dir in self._dirs or dir in self._dirs or
dir in self._parents or
any(parentdir in self._roots any(parentdir in self._roots
for parentdir in util.finddirs(dir))) for parentdir in util.finddirs(dir)))
@encoding.strmethod @encoding.strmethod
def __repr__(self): def __repr__(self):
return ('<includematcher includes=%r>' % pycompat.bytestr(self._pats)) return ('<includematcher includes=%r>' % pycompat.bytestr(self._pats))
class exactmatcher(basematcher): class exactmatcher(basematcher):
r.append('.') r.append('.')
return r, d return r, d
def _roots(kindpats): def _roots(kindpats):
'''Returns root directories to match recursively from the given patterns.''' '''Returns root directories to match recursively from the given patterns.'''
roots, dirs = _patternrootsanddirs(kindpats) roots, dirs = _patternrootsanddirs(kindpats)
return roots return roots
def _rootsanddirs(kindpats): def _rootsdirsandparents(kindpats):
'''Returns roots and exact directories from patterns. '''Returns roots and exact directories from patterns.
roots are directories to match recursively, whereas exact directories should roots are directories to match recursively, whereas exact directories should
be matched non-recursively. The returned (roots, dirs) tuple will also be matched non-recursively. The returned (roots, dirs) tuple will also
include directories that need to be implicitly considered as either, such as include directories that need to be implicitly considered as either, such as
parent directories. parent directories.
>>> _rootsanddirs( >>> _rootsdirsandparents(
... [(b'glob', b'g/h/*', b''), (b'glob', b'g/h', b''), ... [(b'glob', b'g/h/*', b''), (b'glob', b'g/h', b''),
... (b'glob', b'g*', b'')]) ... (b'glob', b'g*', b'')])
(['g/h', 'g/h', '.'], ['g', '.']) (['g/h', 'g/h', '.'], [], ['g', '.'])
>>> _rootsanddirs( >>> _rootsdirsandparents(
... [(b'rootfilesin', b'g/h', b''), (b'rootfilesin', b'', b'')]) ... [(b'rootfilesin', b'g/h', b''), (b'rootfilesin', b'', b'')])
([], ['g/h', '.', 'g', '.']) ([], ['g/h', '.'], ['g', '.'])
>>> _rootsanddirs( >>> _rootsdirsandparents(
... [(b'relpath', b'r', b''), (b'path', b'p/p', b''), ... [(b'relpath', b'r', b''), (b'path', b'p/p', b''),
... (b'path', b'', b'')]) ... (b'path', b'', b'')])
(['r', 'p/p', '.'], ['p', '.']) (['r', 'p/p', '.'], [], ['p', '.'])
>>> _rootsanddirs( >>> _rootsdirsandparents(
... [(b'relglob', b'rg*', b''), (b're', b're/', b''), ... [(b'relglob', b'rg*', b''), (b're', b're/', b''),
... (b'relre', b'rr', b'')]) ... (b'relre', b'rr', b'')])
(['.', '.', '.'], ['.']) (['.', '.', '.'], [], ['.'])
''' '''
r, d = _patternrootsanddirs(kindpats) r, d = _patternrootsanddirs(kindpats)
p = []
# Append the parents as non-recursive/exact directories, since they must be # Append the parents as non-recursive/exact directories, since they must be
# scanned to get to either the roots or the other exact directories. # scanned to get to either the roots or the other exact directories.
d.extend(util.dirs(d)) p.extend(util.dirs(d))
d.extend(util.dirs(r)) p.extend(util.dirs(r))
# util.dirs() does not include the root directory, so add it manually # util.dirs() does not include the root directory, so add it manually
d.append('.') p.append('.')
return r, d return r, d, p
def _explicitfiles(kindpats): def _explicitfiles(kindpats):
'''Returns the potential explicit filenames from the patterns. '''Returns the potential explicit filenames from the patterns.
>>> _explicitfiles([(b'path', b'foo/bar', b'')]) >>> _explicitfiles([(b'path', b'foo/bar', b'')])
['foo/bar'] ['foo/bar']
>>> _explicitfiles([(b'rootfilesin', b'foo/bar', b'')]) >>> _explicitfiles([(b'rootfilesin', b'foo/bar', b'')])
[] []