This is an archive of the discontinued Mercurial Phabricator instance.

Differential D8323

rust-matchers: use the `regex` crate
ClosedPublic

Authored by Alphare on Mar 24 2020, 1:55 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG496868f1030c: rust-matchers: use the `regex` crate
Summary

Instead of falling back to Python when a code path with "ignore" functionality
is reached and Re2 is not installed, the default compilation (i.e. without
the with-re2 feature) will use the regex crate for all regular expressions
business.

As with the introduction of Re2 in a previous series, this yields a big
performance boost compared to the Python + C code in status, diff, commit,
update, and maybe others.

For now Re2 looks to be faster at compiling the DFA (1.5ms vs 5ms for
Netbeans' .hgignore) and a bit faster in actual use: (123ms vs 137ms for
the parallel traversal of Netbeans' clean repo). I am in talks with the author
of regex to see whether that performance difference is a bug, a "won't fix",
or a tuning issue.

The regex crate is already one of our dependencies and using this code does
not require any additional work from the end-user than to use the Rust
extensions.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

Alphare created this revision.Mar 24 2020, 1:55 PM
Alphare added a commit: rHG496868f1030c: rust-matchers: use the `regex` crate.Mar 24 2020, 7:01 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG496868f1030c: rust-matchers: use the `regex` crate (authored by Alphare). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 20868

rust/hg-core/src/filepatterns.rs

fn _build_single_regex(entry: &IgnorePattern) -> Vec<u8> { fn _build_single_regex(entry: &IgnorePattern) -> Vec<u8> {
let IgnorePattern { let IgnorePattern {
syntax, pattern, .. syntax, pattern, ..
} = entry; } = entry;
if pattern.is_empty() { if pattern.is_empty() {
return vec![]; return vec![];
} }
match syntax { match syntax {
PatternSyntax::Regexp => pattern.to_owned(), // The `regex` crate adds `.*` to the start and end of expressions
// if there are no anchors, so add them.
PatternSyntax::Regexp => [b"^", &pattern[..], b"$"].concat(),
PatternSyntax::RelRegexp => { PatternSyntax::RelRegexp => {
if pattern[0] == b'^' { // The `regex` crate accepts `**` while `re2` and Python's `re`
// do not. Checking for `*` correctly triggers the same error all
// engines.
if pattern[0] == b'^' || pattern[0] == b'*' {
return pattern.to_owned(); return pattern.to_owned();
} }
[&b".*"[..], pattern].concat() [&b".*"[..], pattern].concat()
} }
PatternSyntax::Path | PatternSyntax::RelPath => { PatternSyntax::Path | PatternSyntax::RelPath => {
if pattern == b"." { if pattern == b"." {
return vec![]; return vec![];
} }
[escape_pattern(pattern).as_slice(), b"(?:/|$)"].concat() [escape_pattern(pattern).as_slice(), b"(?:/|$)"].concat()
} }
PatternSyntax::RootFiles => { PatternSyntax::RootFiles => {
let mut res = if pattern == b"." { let mut res = if pattern == b"." {
vec![] vec![b'^']
} else { } else {
// Pattern is a directory name. // Pattern is a directory name.
[escape_pattern(pattern).as_slice(), b"/"].concat() [b"^", escape_pattern(pattern).as_slice(), b"/"].concat()
}; };
// Anything after the pattern must be a non-directory. // Anything after the pattern must be a non-directory.
res.extend(b"[^/]+$"); res.extend(b"[^/]+$");
res.push(b'$');
res res
} }
PatternSyntax::RelGlob => { PatternSyntax::RelGlob => {
let glob_re = glob_to_re(pattern); let glob_re = glob_to_re(pattern);
if let Some(rest) = glob_re.drop_prefix(b"[^/]*") { if let Some(rest) = glob_re.drop_prefix(b"[^/]*") {
[b".*", rest, GLOB_SUFFIX].concat() [b".*", rest, GLOB_SUFFIX].concat()
} else { } else {
[b"(?:|.*/)", glob_re.as_slice(), GLOB_SUFFIX].concat() [b"(?:.*/)?", glob_re.as_slice(), GLOB_SUFFIX].concat()
} }
} }
PatternSyntax::Glob | PatternSyntax::RootGlob => { PatternSyntax::Glob | PatternSyntax::RootGlob => {
[glob_to_re(pattern).as_slice(), GLOB_SUFFIX].concat() [b"^", glob_to_re(pattern).as_slice(), GLOB_SUFFIX].concat()
} }
PatternSyntax::Include | PatternSyntax::SubInclude => unreachable!(), PatternSyntax::Include | PatternSyntax::SubInclude => unreachable!(),
} }
} }
const GLOB_SPECIAL_CHARACTERS: [u8; 7] = const GLOB_SPECIAL_CHARACTERS: [u8; 7] =
[b'*', b'?', b'[', b']', b'{', b'}', b'\\']; [b'*', b'?', b'[', b']', b'{', b'}', b'\\'];
PatternSyntax::Include | PatternSyntax::SubInclude => { PatternSyntax::Include | PatternSyntax::SubInclude => {
return Err(PatternError::NonRegexPattern(entry.clone())) return Err(PatternError::NonRegexPattern(entry.clone()))
} }
_ => pattern.to_owned(), _ => pattern.to_owned(),
}; };
if *syntax == PatternSyntax::RootGlob if *syntax == PatternSyntax::RootGlob
&& !pattern.iter().any(|b| GLOB_SPECIAL_CHARACTERS.contains(b)) && !pattern.iter().any(|b| GLOB_SPECIAL_CHARACTERS.contains(b))
{ {
let mut escaped = escape_pattern(&pattern); // The `regex` crate adds `.*` to the start and end of expressions
// if there are no anchors, so add the start anchor.
let mut escaped = vec![b'^'];
escaped.extend(escape_pattern(&pattern));
escaped.extend(GLOB_SUFFIX); escaped.extend(GLOB_SUFFIX);
Ok(escaped) Ok(escaped)
} else { } else {
let mut entry = entry.clone(); let mut entry = entry.clone();
entry.pattern = pattern; entry.pattern = pattern;
Ok(_build_single_regex(&entry)) Ok(_build_single_regex(&entry))
} }
} }
fn test_build_single_regex() { fn test_build_single_regex() {
assert_eq!( assert_eq!(
build_single_regex(&IgnorePattern::new( build_single_regex(&IgnorePattern::new(
PatternSyntax::RelGlob, PatternSyntax::RelGlob,
b"rust/target/", b"rust/target/",
Path::new("") Path::new("")
)) ))
.unwrap(), .unwrap(),
br"(?:|.*/)rust/target(?:/|$)".to_vec(), br"(?:.*/)?rust/target(?:/|$)".to_vec(),
); );
} }
#[test] #[test]
fn test_build_single_regex_shortcut() { fn test_build_single_regex_shortcut() {
assert_eq!( assert_eq!(
build_single_regex(&IgnorePattern::new( build_single_regex(&IgnorePattern::new(
PatternSyntax::RootGlob, PatternSyntax::RootGlob,
b"", b"",
Path::new("") Path::new("")
)) ))
.unwrap(), .unwrap(),
br"\.(?:/|$)".to_vec(), br"^\.(?:/|$)".to_vec(),
); );
assert_eq!( assert_eq!(
build_single_regex(&IgnorePattern::new( build_single_regex(&IgnorePattern::new(
PatternSyntax::RootGlob, PatternSyntax::RootGlob,
b"whatever", b"whatever",
Path::new("") Path::new("")
)) ))
.unwrap(), .unwrap(),
br"whatever(?:/|$)".to_vec(), br"^whatever(?:/|$)".to_vec(),
); );
assert_eq!( assert_eq!(
build_single_regex(&IgnorePattern::new( build_single_regex(&IgnorePattern::new(
PatternSyntax::RootGlob, PatternSyntax::RootGlob,
b"*.o", b"*.o",
Path::new("") Path::new("")
)) ))
.unwrap(), .unwrap(),
br"[^/]*\.o(?:/|$)".to_vec(), br"^[^/]*\.o(?:/|$)".to_vec(),
); );
} }
} }

rust/hg-core/src/matchers.rs

pattern: &[u8], pattern: &[u8],
) -> PatternResult<impl Fn(&HgPath) -> bool + Sync> { ) -> PatternResult<impl Fn(&HgPath) -> bool + Sync> {
let regex = Re2::new(pattern); let regex = Re2::new(pattern);
let regex = regex.map_err(|e| PatternError::UnsupportedSyntax(e))?; let regex = regex.map_err(|e| PatternError::UnsupportedSyntax(e))?;
Ok(move |path: &HgPath| regex.is_match(path.as_bytes())) Ok(move |path: &HgPath| regex.is_match(path.as_bytes()))
} }
#[cfg(not(feature = "with-re2"))] #[cfg(not(feature = "with-re2"))]
fn re_matcher(_: &[u8]) -> PatternResult<Box<dyn Fn(&HgPath) -> bool + Sync>> { /// Returns a function that matches an `HgPath` against the given regex
Err(PatternError::Re2NotInstalled) /// pattern.
///
/// This can fail when the pattern is invalid or not supported by the
/// underlying engine (the `regex` crate), for instance anything with
/// back-references.
fn re_matcher(
pattern: &[u8],
) -> PatternResult<impl Fn(&HgPath) -> bool + Sync> {
use std::io::Write;
let mut escaped_bytes = vec![];
for byte in pattern {
if *byte > 127 {
write!(escaped_bytes, "\\x{:x}", *byte).unwrap();
} else {
escaped_bytes.push(*byte);
}
}
// Avoid the cost of UTF8 checking
//
// # Safety
// This is safe because we escaped all non-ASCII bytes.
let pattern_string = unsafe { String::from_utf8_unchecked(escaped_bytes) };
let re = regex::bytes::RegexBuilder::new(&pattern_string)
.unicode(false)
.build()
.map_err(|e| PatternError::UnsupportedSyntax(e.to_string()))?;
Ok(move |path: &HgPath| re.is_match(path.as_bytes()))
} }
/// Returns the regex pattern and a function that matches an `HgPath` against /// Returns the regex pattern and a function that matches an `HgPath` against
/// said regex formed by the given ignore patterns. /// said regex formed by the given ignore patterns.
fn build_regex_match<'a>( fn build_regex_match<'a>(
ignore_patterns: &'a [&'a IgnorePattern], ignore_patterns: &'a [&'a IgnorePattern],
) -> PatternResult<(Vec<u8>, Box<dyn Fn(&HgPath) -> bool + Sync>)> { ) -> PatternResult<(Vec<u8>, Box<dyn Fn(&HgPath) -> bool + Sync>)> {
let regexps: Result<Vec<_>, PatternError> = ignore_patterns let regexps: Result<Vec<_>, PatternError> = ignore_patterns