regexp: fix pathological slowdown in String for negated classes
The existing implementation of regexp.String() is very inefficient
for large classes, such as those generated by negating small classes.
(For example, [^.] or \W.) To determine whether it can apply the
case-insensitive ?i flag to such classes, calcFlags() iterates
across every case-fold variant of every rune within such classes.
Avoid this by checking if a class covers the majority of the
foldable space (between minFold and maxFold, 'A' and 0x1e943) and,
if so, check runes *not* within the class to see if they fold to
a rune which *is* in the class.
Fixes #69303
diff --git a/src/regexp/syntax/regexp.go b/src/regexp/syntax/regexp.go
index 4994928..f1763eb 100644
--- a/src/regexp/syntax/regexp.go
+++ b/src/regexp/syntax/regexp.go
@@ -147,18 +147,10 @@
return 0, 0
case OpCharClass:
- // If literal is fold-sensitive, return 0, flagI - (?i) has been compiled out.
- // If literal is not fold-sensitive, return 0, 0.
- for i := 0; i < len(re.Rune); i += 2 {
- lo := max(minFold, re.Rune[i])
- hi := min(maxFold, re.Rune[i+1])
- for r := lo; r <= hi; r++ {
- for f := unicode.SimpleFold(r); f != r; f = unicode.SimpleFold(f) {
- if !(lo <= f && f <= hi) && !inCharClass(f, re.Rune) {
- return 0, flagI
- }
- }
- }
+ // If literal is fold-sensitive, forbid flagI - it's not safe to
+ // include this with other subexpressions that use (?i).
+ if foldSensitive(re) {
+ return 0, flagI
}
return 0, 0
@@ -221,6 +213,77 @@
}
}
+func runesInFoldableRange(re *Regexp) int32 {
+ var total int32
+ for i := range len(re.Rune) / 2 {
+ lowRune := re.Rune[i*2]
+ highRune := re.Rune[i*2+1]
+ if lowRune <= maxFold && highRune >= minFold {
+ total += min(highRune, maxFold) - max(lowRune, minFold) + 1
+ }
+ }
+ return total
+}
+
+func includesMostFoldableRunes(re *Regexp) bool {
+ if len(re.Rune) == 0 {
+ return false
+ }
+ // To avoid having to iterate in the common case, check the distance from
+ // the highest to lowest rune (which will generally be very small) before
+ // checking each range.
+ totalRangeWidth := min(maxFold, re.Rune[len(re.Rune)-1]) - max(minFold, re.Rune[0])
+ const foldableRangeSize = maxFold - minFold
+ return totalRangeWidth >= foldableRangeSize/2 &&
+ runesInFoldableRange(re) >= foldableRangeSize/2
+}
+
+func foldSensitive(re *Regexp) bool {
+ // Being fold-sensitive means that some rune within one of the regexp's
+ // ranges has a fold variant which is not in any of its ranges.
+
+ // Many common expressions, such as \W, include a huge range of runes,
+ // and exclude only a few runes within [minFold, maxFold]. In these
+ // cases, to avoid needing to check hundreds of thousands of runes,
+ // loop over runes which are *not* in the regexp, and check if *they*
+ // have a fold variant which *is* in the regexp (a symmetric property).
+ if !includesMostFoldableRunes(re) {
+ for i := range len(re.Rune) / 2 {
+ low := max(minFold, re.Rune[i*2])
+ high := min(maxFold, re.Rune[i*2+1])
+ for r := low; r <= high; r++ {
+ for f := unicode.SimpleFold(r); f != r; f = unicode.SimpleFold(f) {
+ if !(low <= f && f <= high) && !inCharClass(f, re.Rune) {
+ return true
+ }
+ }
+ }
+ }
+ return false
+ }
+
+ for i := range len(re.Rune)/2 + 1 {
+ // Check outside the class.
+ // [minFold, re.Rune[0]-1], [re.Rune[1]+1, re.Rune[2]-1], ..., [re.Rune[len(re.Rune)-1]+1, maxFold]
+ low := int32(minFold)
+ high := int32(maxFold)
+ if i > 0 {
+ low = max(low, re.Rune[i*2-1]+1)
+ }
+ if i*2 < len(re.Rune) {
+ high = min(high, re.Rune[i*2]-1)
+ }
+ for r := low; r <= high; r++ {
+ for f := unicode.SimpleFold(r); f != r; f = unicode.SimpleFold(f) {
+ if inCharClass(f, re.Rune) {
+ return true
+ }
+ }
+ }
+ }
+ return false
+}
+
// writeRegexp writes the Perl syntax for the regular expression re to b.
func writeRegexp(b *strings.Builder, re *Regexp, f printFlags, flags map[*Regexp]printFlags) {
f |= flags[re]
diff --git a/src/regexp/syntax/regexp_test.go b/src/regexp/syntax/regexp_test.go
new file mode 100644
index 0000000..9c7c705
--- /dev/null
+++ b/src/regexp/syntax/regexp_test.go
@@ -0,0 +1,78 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package syntax
+
+import (
+ "fmt"
+ "testing"
+)
+
+func TestRunesInFoldableRange(t *testing.T) {
+ for _, tt := range []struct {
+ regexp string
+ want int32
+ }{
+ {`[a-z]`, 26},
+ {`[a-bd-eg-h]`, 6},
+ {`[\x00-A]`, 1}, // end at minFold
+ {`[\x{1e943}-\x{1e949}]`, 1}, // start at maxFold
+ } {
+ re, err := Parse(tt.regexp, MatchNL|Perl&^OneLine)
+ if err != nil {
+ t.Errorf("Parse(%#q) = error %v", tt.regexp, err)
+ continue
+ }
+ if got := runesInFoldableRange(re); got != tt.want {
+ t.Errorf("runesInFoldableRange(%#q) = %d, want %d", tt.regexp, got, tt.want)
+ }
+ }
+}
+
+func TestIncludesMostFoldableRunes(t *testing.T) {
+ for _, tt := range []struct {
+ regexp string
+ want bool
+ }{
+ {``, false},
+ {`[A-z]`, false},
+ {`[A-\x{10000}]`, true},
+ {`\W`, true},
+ {`[\x{1e900}-\x{3e949}]`, false}, // large, but mostly non-foldable
+ } {
+ re, err := Parse(tt.regexp, MatchNL|Perl&^OneLine)
+ if err != nil {
+ t.Errorf("Parse(%#q) = error %v", tt.regexp, err)
+ continue
+ }
+ if got := includesMostFoldableRunes(re); got != tt.want {
+ t.Errorf("includesMostFoldableRunes(%#q) = %t, want %t", tt.regexp, got, tt.want)
+ }
+ }
+}
+
+func TestFoldSensitive(t *testing.T) {
+ for _, tt := range []struct {
+ regexp string
+ want bool
+ }{
+ {`\d`, false}, // no foldables -> safe to use ?i
+ {`[A-B]`, true}, // doesn't have lowercase -> unsafe
+ {`[kK]`, true}, // doesn't have Kelvin rune
+ {`[A-Ba-b]`, false},
+ {`\W`, true}, // some non-word runes fold to A-Za-z
+ {`[A-\x{1e943}]`, false}, // entire foldable range
+ {`\D`, false}, // excluded runes (digits) don't fold
+ } {
+ fmt.Println(tt.regexp) // XXX DEBUG
+ re, err := Parse(tt.regexp, MatchNL|Perl&^OneLine)
+ if err != nil {
+ t.Errorf("Parse(%#q) = error %v", tt.regexp, err)
+ continue
+ }
+ if got := foldSensitive(re); got != tt.want {
+ t.Errorf("foldSensitive(%#q) = %t, want %t", tt.regexp, got, tt.want)
+ }
+ }
+}
diff --git a/src/regexp/syntax/simplify_test.go b/src/regexp/syntax/simplify_test.go
index 6d06f99..c84b8e9 100644
--- a/src/regexp/syntax/simplify_test.go
+++ b/src/regexp/syntax/simplify_test.go
@@ -114,6 +114,10 @@
{`(?i)[a-z]`, "[A-Za-z\u017F\u212A]"},
{`(?i)[\x00-\x{FFFD}]`, "[\\x00-\uFFFD]"},
{`(?i)[\x00-\x{10FFFF}]`, `(?s:.)`},
+ {`[Aa][Bb][Kk]`, `(?i:AB)[Kk]`},
+ {`[Aa][Bb][KkK]`, `(?i:AB[KkK])`},
+ {`[Aa][Bb][O-u]`, `(?i:AB)[O-u]`},
+ {`(?i)[0-24-6A]`, `[0-24-6Aa]`},
// Empty string as a regular expression.
// The empty string must be preserved inside parens in order
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
Congratulations on opening your first change. Thank you for your contribution!
Next steps:
A maintainer will review your change and provide feedback. See
https://go.dev/doc/contribute#review for more info and tips to get your
patch through code review.
Most changes in the Go project go through a few rounds of revision. This can be
surprising to people new to the project. The careful, iterative review process
is our way of helping mentor contributors and ensuring that their contributions
have a lasting impact.
During May-July and Nov-Jan the Go project is in a code freeze, during which
little code gets reviewed or merged. If a reviewer responds with a comment like
R=go1.11 or adds a tag like "wait-release", it means that this CL will be
reviewed as part of the next development cycle. See https://go.dev/s/release
for more details.
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
| Code-Review | +2 |
regexp: fix pathological slowdown in String for negated classesThis should be regexp/syntax, not regexp.
The existing implementation of regexp.String() is very inefficientI'd just write this as "String" (and no "()" -- that's the standard Go style).
It's not regexp.String, it's really (*syntax.Regexp).String. But String is probably understandable enough.
case-insensitive ?i flag to such classes, calcFlags() iteratess/()//
for i := range len(re.Rune) / 2 {Throughout, let's stick with iterating by 2 as the original code did rather than the `i*2` thing.
lowRune := re.Rune[i*2]Nit: throughout, instead of `lowRune`/`highRune` or `low`/`high`, how about just `lo`/`hi` -- it's a common naming choice that doesn't lose any clarity and it's what the code you're editing already used.
func foldSensitive(re *Regexp) bool {Nit: let's reverse the order of these three functions. That puts the more important function first and its helper after.
// Being fold-sensitive means that some rune within one of the regexp's
// ranges has a fold variant which is not in any of its ranges.I'd rewrite this as a doc comment.
Also, since this is now far away from where it's called, might be worth mentioning that it's specific to a char class node (or even rename this function to mention it).
// Many common expressions, such as \W, include a huge range of runes,Maybe s/expressions/classes/ would be more precise?
// loop over runes which are *not* in the regexp, and check if *they*
// have a fold variant which *is* in the regexp (a symmetric property).I think s/regexp/class/ both times.
if !includesMostFoldableRunes(re) {If I'm reading it correctly, the big comment which precedes this is mostly about how to handle the `if includesMostFoldableRunes(re)` case, which is further down the page. I think it would make more sense to put that one first, both to get rid of the `!` here and to make the comment better align with the code.
No actual test would fail without this change, right?
Should we add a new benchmark, at least?
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
| Code-Review | +0 |
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
Whoops, didn't realize that the big +2 button would just submit my comments and approve without any further confirmation.
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
I believe I've handled all requested changes. Thanks for the review!
Ran a quick benchmark on my laptop - not very precise, but should be good enough for a change like this :)
```
goos: darwin
goarch: arm64
pkg: std/regexp/syntax
cpu: Apple M5 Max
│ old.txt │ new.txt │
│ sec/op │ sec/op vs base │
String-18 1115880.0n ± 3% 379.2n ± 6% -99.97% (p=0.000 n=10)
```
regexp: fix pathological slowdown in String for negated classesThis should be regexp/syntax, not regexp.
Done
The existing implementation of regexp.String() is very inefficientI'd just write this as "String" (and no "()" -- that's the standard Go style).
It's not regexp.String, it's really (*syntax.Regexp).String. But String is probably understandable enough.
Done
case-insensitive ?i flag to such classes, calcFlags() iteratesNicholas Feinbergs/()//
Done
Throughout, let's stick with iterating by 2 as the original code did rather than the `i*2` thing.
Done
Nit: throughout, instead of `lowRune`/`highRune` or `low`/`high`, how about just `lo`/`hi` -- it's a common naming choice that doesn't lose any clarity and it's what the code you're editing already used.
Done
Nit: let's reverse the order of these three functions. That puts the more important function first and its helper after.
Done
// Being fold-sensitive means that some rune within one of the regexp's
// ranges has a fold variant which is not in any of its ranges.I'd rewrite this as a doc comment.
Also, since this is now far away from where it's called, might be worth mentioning that it's specific to a char class node (or even rename this function to mention it).
Done
// Many common expressions, such as \W, include a huge range of runes,Maybe s/expressions/classes/ would be more precise?
Done
// loop over runes which are *not* in the regexp, and check if *they*
// have a fold variant which *is* in the regexp (a symmetric property).I think s/regexp/class/ both times.
Done
If I'm reading it correctly, the big comment which precedes this is mostly about how to handle the `if includesMostFoldableRunes(re)` case, which is further down the page. I think it would make more sense to put that one first, both to get rid of the `!` here and to make the comment better align with the code.
Done
fmt.Println(tt.regexp) // XXX DEBUGNicholas FeinbergDelete
Done
No actual test would fail without this change, right?
Should we add a new benchmark, at least?
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
| Code-Review | +1 |
| Commit-Queue | +1 |
// foldSensitiveCharClass returns, for a charClass node, whether some rune ins/returns/reports/
func BenchmarkString(b *testing.B) {Add a comment here linking to the issue (as `https://go.dev/issue/69303`) to explain the existence of this benchmark. Or maybe even rename it to `BenchmarkIssue69303` -- it's not a general-purpose benchmark for `String`, it checks for a regression in this specific issue.
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
Adding rsc as the reviewer per s/owners.
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
// foldSensitiveCharClass returns, for a charClass node, whether some rune inNicholas Feinbergs/returns/reports/
Done
Add a comment here linking to the issue (as `https://go.dev/issue/69303`) to explain the existence of this benchmark. Or maybe even rename it to `BenchmarkIssue69303` -- it's not a general-purpose benchmark for `String`, it checks for a regression in this specific issue.
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
// Copyright 2024 The Go Authors. All rights reserved.2026
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
// Copyright 2024 The Go Authors. All rights reserved.Nicholas Feinberg2026
Done
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |