This is an archive of the discontinued Mercurial Phabricator instance.

Differential D7931

rust-status: use bare hg status fastpath from Python
ClosedPublic

Authored by Alphare on Jan 17 2020, 11:53 AM.

Details

Reviewers
marmoute
Group Reviewers
hg-reviewers
Commits
rHG4d1634e59f13: rust-status: use bare hg status fastpath from Python
Summary

This change also adds a test case for subrepos.
Repeating the benchmark information from the hg-core commit:

On the Netbeans repository:
C: 840ms
Rust+C: 556ms

Mozilla Central with the one pattern that causes a fallback removed:
C: 2.315s
Rust+C: 1.700s

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

Alphare created this revision.Jan 17 2020, 11:53 AM
marmoute requested changes to this revision.Jan 23 2020, 1:48 PM
marmoute added a subscriber: marmoute.

hooo, I am a bot

This revision now requires changes to proceed.Jan 23 2020, 1:48 PM
marmoute resigned from this revision.Jan 23 2020, 1:49 PM
This revision now requires review to proceed.Jan 23 2020, 1:49 PM
Alphare updated this revision to Diff 19949.Feb 6 2020, 10:16 AM
Alphare updated this revision to Diff 20041.Feb 10 2020, 10:47 AM
Alphare updated this revision to Diff 20159.Feb 11 2020, 5:55 AM
Alphare updated this revision to Diff 20167.Feb 11 2020, 9:20 AM
Alphare updated this revision to Diff 20559.Mar 6 2020, 11:31 AM
Alphare updated this revision to Diff 20706.Mar 11 2020, 11:51 AM
Alphare added a commit: rHG4d1634e59f13: rust-status: use bare hg status fastpath from Python.Mar 11 2020, 12:24 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG4d1634e59f13: rust-status: use bare hg status fastpath from Python (authored by Alphare). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 20723

mercurial/dirstate.py

from . import ( from . import (
encoding, encoding,
error, error,
match as matchmod, match as matchmod,
pathutil, pathutil,
policy, policy,
pycompat, pycompat,
scmutil, scmutil,
sparse,
txnutil, txnutil,
util, util,
) )
from .interfaces import ( from .interfaces import (
dirstate as intdirstate, dirstate as intdirstate,
util as interfaceutil, util as interfaceutil,
) )
else: else:
# We may not have walked the full directory tree above, # We may not have walked the full directory tree above,
# so stat and check everything we missed. # so stat and check everything we missed.
iv = iter(visit) iv = iter(visit)
for st in util.statfiles([join(i) for i in visit]): for st in util.statfiles([join(i) for i in visit]):
results[next(iv)] = st results[next(iv)] = st
return results return results
def _rust_status(self, matcher, list_clean): def _rust_status(self, matcher, list_clean, list_ignored, list_unknown):
# Force Rayon (Rust parallelism library) to respect the number of # Force Rayon (Rust parallelism library) to respect the number of
# workers. This is a temporary workaround until Rust code knows # workers. This is a temporary workaround until Rust code knows
# how to read the config file. # how to read the config file.
numcpus = self._ui.configint(b"worker", b"numcpus") numcpus = self._ui.configint(b"worker", b"numcpus")
if numcpus is not None: if numcpus is not None:
encoding.environ.setdefault(b'RAYON_NUM_THREADS', b'%d' % numcpus) encoding.environ.setdefault(b'RAYON_NUM_THREADS', b'%d' % numcpus)
workers_enabled = self._ui.configbool(b"worker", b"enabled", True) workers_enabled = self._ui.configbool(b"worker", b"enabled", True)
if not workers_enabled: if not workers_enabled:
encoding.environ[b"RAYON_NUM_THREADS"] = b"1" encoding.environ[b"RAYON_NUM_THREADS"] = b"1"
( (
lookup, lookup,
modified, modified,
added, added,
removed, removed,
deleted, deleted,
unknown,
clean, clean,
ignored,
unknown,
warnings,
bad,
) = rustmod.status( ) = rustmod.status(
self._map._rustmap, self._map._rustmap,
matcher, matcher,
self._rootdir, self._rootdir,
bool(list_clean), self._ignorefiles(),
self._lastnormaltime,
self._checkexec, self._checkexec,
self._lastnormaltime,
bool(list_clean),
bool(list_ignored),
bool(list_unknown),
)
if self._ui.warn:
for item in warnings:
if isinstance(item, tuple):
file_path, syntax = item
msg = _(b"%s: ignoring invalid syntax '%s'\n") % (
file_path,
syntax,
)
self._ui.warn(msg)
else:
msg = _(b"skipping unreadable pattern file '%s': %s\n")
self._ui.warn(
msg
% (
pathutil.canonpath(
self._rootdir, self._rootdir, item
),
b"No such file or directory",
) )
)
for (fn, message) in bad:
matcher.bad(fn, encoding.strtolocal(message))
status = scmutil.status( status = scmutil.status(
modified=modified, modified=modified,
added=added, added=added,
removed=removed, removed=removed,
deleted=deleted, deleted=deleted,
unknown=unknown, unknown=unknown,
ignored=[], ignored=ignored,
clean=clean, clean=clean,
) )
return (lookup, status) return (lookup, status)
def status(self, match, subrepos, ignored, clean, unknown): def status(self, match, subrepos, ignored, clean, unknown):
'''Determine the status of the working copy relative to the '''Determine the status of the working copy relative to the
dirstate and return a pair of (unsure, status), where status is of type dirstate and return a pair of (unsure, status), where status is of type
scmutil.status and: scmutil.status and:
lookup, modified, added, unknown, ignored = [], [], [], [], [] lookup, modified, added, unknown, ignored = [], [], [], [], []
removed, deleted, clean = [], [], [] removed, deleted, clean = [], [], []
dmap = self._map dmap = self._map
dmap.preload() dmap.preload()
use_rust = True use_rust = True
allowed_matchers = (matchmod.alwaysmatcher, matchmod.exactmatcher) allowed_matchers = (
matchmod.alwaysmatcher,
matchmod.exactmatcher,
matchmod.includematcher,
)
if rustmod is None: if rustmod is None:
use_rust = False use_rust = False
elif self._checkcase:
# Case-insensitive filesystems are not handled yet
use_rust = False
elif subrepos: elif subrepos:
use_rust = False use_rust = False
elif bool(listunknown): elif sparse.enabled:
# Pathauditor does not exist yet in Rust, unknown files
# can't be trusted.
use_rust = False use_rust = False
elif self._ignorefiles() and listignored: elif match.traversedir is not None:
# Rust has no ignore mechanism yet, so don't use Rust for
# commands that need ignore.
use_rust = False use_rust = False
elif not isinstance(match, allowed_matchers): elif not isinstance(match, allowed_matchers):
# Matchers have yet to be implemented # Matchers have yet to be implemented
use_rust = False use_rust = False
if use_rust: if use_rust:
return self._rust_status(match, listclean) try:
return self._rust_status(
match, listclean, listignored, listunknown
)
except rustmod.FallbackError:
pass
def noop(f): def noop(f):
pass pass
dcontains = dmap.__contains__ dcontains = dmap.__contains__
dget = dmap.__getitem__ dget = dmap.__getitem__
ladd = lookup.append # aka "unsure" ladd = lookup.append # aka "unsure"
madd = modified.append madd = modified.append
elif listclean: elif listclean:
cadd(fn) cadd(fn)
elif state == b'm': elif state == b'm':
madd(fn) madd(fn)
elif state == b'a': elif state == b'a':
aadd(fn) aadd(fn)
elif state == b'r': elif state == b'r':
radd(fn) radd(fn)
status = scmutil.status(
return (
lookup,
scmutil.status(
modified, added, removed, deleted, unknown, ignored, clean modified, added, removed, deleted, unknown, ignored, clean
),
) )
return (lookup, status)
def matches(self, match): def matches(self, match):
''' '''
return files in the dirstate (in whatever state) filtered by match return files in the dirstate (in whatever state) filtered by match
''' '''
dmap = self._map dmap = self._map
if rustmod is not None: if rustmod is not None:
dmap = self._map._rustmap dmap = self._map._rustmap

mercurial/match.py

def get(self, path): def get(self, path):
return self._dirs.get(path, set()) return self._dirs.get(path, set())
class includematcher(basematcher): class includematcher(basematcher):
def __init__(self, root, kindpats, badfn=None): def __init__(self, root, kindpats, badfn=None):
super(includematcher, self).__init__(badfn) super(includematcher, self).__init__(badfn)
if rustmod is not None:
# We need to pass the patterns to Rust because they can contain
# patterns from the user interface
self._kindpats = kindpats
self._pats, self.matchfn = _buildmatch(kindpats, b'(?:/|$)', root) self._pats, self.matchfn = _buildmatch(kindpats, b'(?:/|$)', root)
self._prefix = _prefix(kindpats) self._prefix = _prefix(kindpats)
roots, dirs, parents = _rootsdirsandparents(kindpats) roots, dirs, parents = _rootsdirsandparents(kindpats)
# roots are directories which are recursively included. # roots are directories which are recursively included.
self._roots = set(roots) self._roots = set(roots)
# dirs are directories which are non-recursively included. # dirs are directories which are non-recursively included.
self._dirs = set(dirs) self._dirs = set(dirs)
# parents are directories which are non-recursively included because # parents are directories which are non-recursively included because

tests/test-subrepo-deep-nested-change.t

\r (no-eol) (esc) \r (no-eol) (esc)
$ mv sub1/.hgsub sub1/x.hgsub $ mv sub1/.hgsub sub1/x.hgsub
$ hg status -S $ hg status -S
warning: subrepo spec file 'sub1/.hgsub' not found warning: subrepo spec file 'sub1/.hgsub' not found
R sub1/.hgsubstate R sub1/.hgsubstate
R sub1/sub2/folder/test.txt R sub1/sub2/folder/test.txt
! sub1/.hgsub ! sub1/.hgsub
? sub1/x.hgsub ? sub1/x.hgsub
$ hg status -R sub1
warning: subrepo spec file 'sub1/.hgsub' not found
R .hgsubstate
! .hgsub
? x.hgsub
$ mv sub1/x.hgsub sub1/.hgsub $ mv sub1/x.hgsub sub1/.hgsub
$ hg update -Cq $ hg update -Cq
$ touch sub1/foo $ touch sub1/foo
$ hg forget sub1/sub2/folder/test.txt $ hg forget sub1/sub2/folder/test.txt
$ rm sub1/sub2/test.txt $ rm sub1/sub2/test.txt
Test relative path printing + subrepos Test relative path printing + subrepos
$ mkdir -p foo/bar $ mkdir -p foo/bar