mirror of
https://github.com/runkids/skillshare.git
synced 2026-11-18 18:04:17 +00:00
Add compile-time literal prefilters for audit rules using regexp AST analysis and apply prefilter gating before MatchString in hot scanning loops. Changes: - extend compiled rule with prefilter metadata (literal + case-fold flag) - derive conservative must-match literals via regexp/syntax (fallback to LiteralPrefix) - apply prefilter checks in unified scanners, legacy ScanContent paths, and markdown link rule checks - add unit tests for prefilter extraction and case-fold matching Verification: - docker compose -f .devcontainer/docker-compose.yml exec -T --user "501:20" skillshare-devcontainer bash -lc 'cd /workspace && GOCACHE=/tmp/go-build go test ./internal/audit -count=1' - docker compose -f .devcontainer/docker-compose.yml exec -T --user "501:20" skillshare-devcontainer bash -lc 'cd /workspace && GOCACHE=/tmp/go-build go build -o /tmp/skillshare ./cmd/skillshare && SKILLSHARE_TEST_BINARY=/tmp/skillshare GOCACHE=/tmp/go-build go test ./tests/integration -run TestAudit_ -count=1'
193 lines
4.1 KiB
Go
193 lines
4.1 KiB
Go
package audit
|
|
|
|
import (
|
|
"regexp"
|
|
"regexp/syntax"
|
|
"strings"
|
|
"unicode"
|
|
)
|
|
|
|
type literalConstraint struct {
|
|
text string
|
|
fold bool
|
|
}
|
|
|
|
// deriveRulePrefilter extracts a conservative literal prefilter from regex.
|
|
// The prefilter must be present in every possible match; otherwise "" is used.
|
|
func deriveRulePrefilter(raw string, re *regexp.Regexp) (string, bool) {
|
|
if re == nil {
|
|
return "", false
|
|
}
|
|
|
|
ast, err := syntax.Parse(raw, syntax.Perl)
|
|
if err == nil {
|
|
if best := pickBestPrefilter(requiredLiterals(ast.Simplify())); best.text != "" {
|
|
if best.fold {
|
|
return strings.ToLower(best.text), true
|
|
}
|
|
return best.text, false
|
|
}
|
|
}
|
|
|
|
// Fallback to Go regexp's literal prefix acceleration when available.
|
|
prefix, _ := re.LiteralPrefix()
|
|
if !isSelectiveLiteral(prefix) {
|
|
return "", false
|
|
}
|
|
if regexMayFoldCase(raw) {
|
|
return strings.ToLower(prefix), true
|
|
}
|
|
return prefix, false
|
|
}
|
|
|
|
func requiredLiterals(re *syntax.Regexp) []literalConstraint {
|
|
if re == nil {
|
|
return nil
|
|
}
|
|
|
|
switch re.Op {
|
|
case syntax.OpLiteral:
|
|
if len(re.Rune) == 0 {
|
|
return nil
|
|
}
|
|
return []literalConstraint{{
|
|
text: string(re.Rune),
|
|
fold: re.Flags&syntax.FoldCase != 0,
|
|
}}
|
|
case syntax.OpCapture:
|
|
if len(re.Sub) == 0 {
|
|
return nil
|
|
}
|
|
return requiredLiterals(re.Sub[0])
|
|
case syntax.OpConcat:
|
|
var out []literalConstraint
|
|
for _, sub := range re.Sub {
|
|
out = unionConstraints(out, requiredLiterals(sub))
|
|
}
|
|
return out
|
|
case syntax.OpAlternate:
|
|
if len(re.Sub) == 0 {
|
|
return nil
|
|
}
|
|
out := requiredLiterals(re.Sub[0])
|
|
for _, sub := range re.Sub[1:] {
|
|
out = intersectConstraints(out, requiredLiterals(sub))
|
|
if len(out) == 0 {
|
|
return nil
|
|
}
|
|
}
|
|
return out
|
|
case syntax.OpPlus:
|
|
if len(re.Sub) == 0 {
|
|
return nil
|
|
}
|
|
return requiredLiterals(re.Sub[0])
|
|
case syntax.OpRepeat:
|
|
if re.Min > 0 && len(re.Sub) > 0 {
|
|
return requiredLiterals(re.Sub[0])
|
|
}
|
|
return nil
|
|
case syntax.OpStar, syntax.OpQuest:
|
|
return nil
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func pickBestPrefilter(candidates []literalConstraint) literalConstraint {
|
|
best := literalConstraint{}
|
|
bestLen := 0
|
|
for _, c := range candidates {
|
|
c = normalizeConstraint(c)
|
|
if !isSelectiveLiteral(c.text) {
|
|
continue
|
|
}
|
|
l := len(c.text)
|
|
if l > bestLen {
|
|
best = c
|
|
bestLen = l
|
|
continue
|
|
}
|
|
if l == bestLen && best.fold && !c.fold {
|
|
// Prefer case-sensitive prefilters when selectivity is identical.
|
|
best = c
|
|
}
|
|
}
|
|
return best
|
|
}
|
|
|
|
func unionConstraints(base, extra []literalConstraint) []literalConstraint {
|
|
out := append([]literalConstraint{}, base...)
|
|
for _, c := range extra {
|
|
out = appendUniqueConstraint(out, c)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func intersectConstraints(a, b []literalConstraint) []literalConstraint {
|
|
if len(a) == 0 || len(b) == 0 {
|
|
return nil
|
|
}
|
|
var out []literalConstraint
|
|
for _, ca := range a {
|
|
for _, cb := range b {
|
|
if merged, ok := mergeConstraintPair(ca, cb); ok {
|
|
out = appendUniqueConstraint(out, merged)
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func mergeConstraintPair(a, b literalConstraint) (literalConstraint, bool) {
|
|
a = normalizeConstraint(a)
|
|
b = normalizeConstraint(b)
|
|
|
|
if strings.EqualFold(a.text, b.text) {
|
|
if a.fold || b.fold || a.text != b.text {
|
|
return literalConstraint{text: strings.ToLower(a.text), fold: true}, true
|
|
}
|
|
return literalConstraint{text: a.text, fold: false}, true
|
|
}
|
|
return literalConstraint{}, false
|
|
}
|
|
|
|
func appendUniqueConstraint(out []literalConstraint, c literalConstraint) []literalConstraint {
|
|
c = normalizeConstraint(c)
|
|
for _, existing := range out {
|
|
if equivalentConstraint(existing, c) {
|
|
return out
|
|
}
|
|
}
|
|
return append(out, c)
|
|
}
|
|
|
|
func equivalentConstraint(a, b literalConstraint) bool {
|
|
a = normalizeConstraint(a)
|
|
b = normalizeConstraint(b)
|
|
return a.fold == b.fold && a.text == b.text
|
|
}
|
|
|
|
func normalizeConstraint(c literalConstraint) literalConstraint {
|
|
if c.fold {
|
|
c.text = strings.ToLower(c.text)
|
|
}
|
|
return c
|
|
}
|
|
|
|
func isSelectiveLiteral(s string) bool {
|
|
if len(s) < 3 {
|
|
return false
|
|
}
|
|
for _, r := range s {
|
|
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func regexMayFoldCase(raw string) bool {
|
|
return strings.Contains(strings.ToLower(raw), "(?i")
|
|
}
|