Files
skillshare/internal/audit/audit_rules_test.go
T
Willie 6e86537987 fix(audit): honor enabled:false for synthetic tier and cross-skill findings
Tier and cross-skill findings are generated programmatically rather than
compiled from the rule list, so they bypassed the disabled-ID filter applied
at rule-compile time. Setting enabled:false in audit-rules.yaml marked the
rule disabled in the listing but the scan still fired it at full severity.

Filter synthetic findings against the disabled-ID set in both the skill-scan
and single-file paths, threading the set through the cross-skill call sites
(analyzer, cmd, server). Fixes #204.
2026-06-05 20:51:32 +08:00

497 lines
14 KiB
Go

package audit
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestLoadBuiltinRules(t *testing.T) {
resetForTest()
rules, err := loadBuiltinRules()
if err != nil {
t.Fatalf("loadBuiltinRules() error: %v", err)
}
if len(rules) == 0 {
t.Fatal("expected non-empty rules")
}
// Verify each rule has required fields
for _, r := range rules {
if r.ID == "" {
t.Error("rule has empty ID")
}
if r.Severity == "" {
t.Error("rule has empty Severity")
}
if r.Pattern == "" {
t.Errorf("rule %s has empty Pattern", r.ID)
}
if r.Message == "" {
t.Errorf("rule %s has empty Message", r.ID)
}
if r.Regex == nil {
t.Errorf("rule %s has nil Regex", r.ID)
}
}
}
func TestRuleCount(t *testing.T) {
resetForTest()
rules, err := loadBuiltinRules()
if err != nil {
t.Fatalf("loadBuiltinRules() error: %v", err)
}
expected := len(builtinYAML())
if got := len(rules); got != expected {
t.Errorf("expected %d builtin rules, got %d", expected, got)
}
}
func TestCompileRules_InvalidRegex(t *testing.T) {
yr := []yamlRule{{
ID: "bad-regex",
Severity: SeverityHigh,
Pattern: "test",
Message: "test",
Regex: "[invalid",
}}
_, err := compileRules(yr)
if err == nil {
t.Fatal("expected error for invalid regex")
}
if !strings.Contains(err.Error(), "bad-regex") {
t.Errorf("error should mention rule ID, got: %s", err)
}
}
func TestCompileRules_InvalidExcludeRegex(t *testing.T) {
yr := []yamlRule{{
ID: "bad-exclude",
Severity: SeverityHigh,
Pattern: "test",
Message: "test",
Regex: "ok",
Exclude: "[invalid",
}}
_, err := compileRules(yr)
if err == nil {
t.Fatal("expected error for invalid exclude regex")
}
if !strings.Contains(err.Error(), "bad-exclude") {
t.Errorf("error should mention rule ID, got: %s", err)
}
}
func TestCompileRules_ValidateSeverity(t *testing.T) {
yr := []yamlRule{{
ID: "typo-sev",
Severity: "CRTICAL",
Pattern: "test",
Message: "test",
Regex: "test",
}}
_, err := compileRules(yr)
if err == nil {
t.Fatal("expected error for invalid severity")
}
if !strings.Contains(err.Error(), "CRTICAL") {
t.Errorf("error should mention bad severity, got: %s", err)
}
}
func TestCompileRules_EmptyRegex(t *testing.T) {
yr := []yamlRule{{
ID: "empty-regex",
Severity: SeverityHigh,
Pattern: "test",
Message: "test",
Regex: "",
}}
_, err := compileRules(yr)
if err == nil {
t.Fatal("expected error for empty regex")
}
}
func TestExcludeRegex(t *testing.T) {
// suspicious-fetch should have an exclude for localhost
resetForTest()
rules, err := loadBuiltinRules()
if err != nil {
t.Fatalf("loadBuiltinRules() error: %v", err)
}
var fetchRule *rule
for i := range rules {
if rules[i].ID == "suspicious-fetch-0" {
fetchRule = &rules[i]
break
}
}
if fetchRule == nil {
t.Fatal("suspicious-fetch-0 rule not found")
}
if fetchRule.Exclude == nil {
t.Fatal("suspicious-fetch-0 should have Exclude regex")
}
// External URL should match regex but NOT exclude
if !fetchRule.Regex.MatchString("curl https://evil.com") {
t.Error("should match external URL")
}
if fetchRule.Exclude.MatchString("curl https://evil.com") {
t.Error("should not exclude external URL")
}
// Localhost should match both regex and exclude
if !fetchRule.Regex.MatchString("curl http://localhost:3000/api") {
t.Error("should match localhost URL")
}
if !fetchRule.Exclude.MatchString("curl http://localhost:3000/api") {
t.Error("should exclude localhost URL")
}
}
func TestMergeRules_Append(t *testing.T) {
base := []yamlRule{
{ID: "rule-1", Severity: SeverityHigh, Pattern: "p1", Message: "m1", Regex: "r1"},
}
overlay := []yamlRule{
{ID: "rule-2", Severity: SeverityMedium, Pattern: "p2", Message: "m2", Regex: "r2"},
}
merged := mergeYAMLRules(base, overlay)
if len(merged) != 2 {
t.Fatalf("expected 2 rules, got %d", len(merged))
}
if merged[1].ID != "rule-2" {
t.Errorf("expected appended rule-2, got %s", merged[1].ID)
}
}
func TestMergeRules_Disable(t *testing.T) {
base := []yamlRule{
{ID: "rule-1", Severity: SeverityHigh, Pattern: "p1", Message: "m1", Regex: "r1"},
{ID: "rule-2", Severity: SeverityHigh, Pattern: "p2", Message: "m2", Regex: "r2"},
}
f := false
overlay := []yamlRule{
{ID: "rule-1", Enabled: &f},
}
merged := mergeYAMLRules(base, overlay)
// After compiling, rule-1 should be excluded
compiled, err := compileRules(merged)
if err != nil {
t.Fatalf("compileRules error: %v", err)
}
if len(compiled) != 1 {
t.Fatalf("expected 1 rule after disabling, got %d", len(compiled))
}
if compiled[0].ID != "rule-2" {
t.Errorf("expected rule-2 to survive, got %s", compiled[0].ID)
}
}
func TestMergeRules_Override(t *testing.T) {
base := []yamlRule{
{ID: "rule-1", Severity: SeverityHigh, Pattern: "p1", Message: "original", Regex: "r1"},
}
overlay := []yamlRule{
{ID: "rule-1", Severity: SeverityCritical, Pattern: "p1", Message: "overridden", Regex: "r1"},
}
merged := mergeYAMLRules(base, overlay)
if len(merged) != 1 {
t.Fatalf("expected 1 rule, got %d", len(merged))
}
if merged[0].Message != "overridden" {
t.Errorf("expected overridden message, got %s", merged[0].Message)
}
if merged[0].Severity != SeverityCritical {
t.Errorf("expected CRITICAL severity, got %s", merged[0].Severity)
}
}
func TestMergeRules_PatternDisable(t *testing.T) {
resetForTest()
boolFalse := false
base := []yamlRule{
{ID: "pi-0", Severity: SeverityCritical, Pattern: "prompt-injection", Message: "m0", Regex: "r0"},
{ID: "pi-1", Severity: SeverityHigh, Pattern: "prompt-injection", Message: "m1", Regex: "r1"},
{ID: "dc-0", Severity: SeverityHigh, Pattern: "destructive-commands", Message: "m2", Regex: "r2"},
}
overlay := []yamlRule{
{Pattern: "prompt-injection", Enabled: &boolFalse},
}
merged := mergeYAMLRules(base, overlay)
compiled, err := compileRules(merged)
if err != nil {
t.Fatalf("compile: %v", err)
}
if len(compiled) != 1 {
t.Fatalf("want 1 rule, got %d", len(compiled))
}
if compiled[0].ID != "dc-0" {
t.Fatalf("want dc-0, got %s", compiled[0].ID)
}
}
func TestMergeRules_PatternSeverityOverride(t *testing.T) {
resetForTest()
base := []yamlRule{
{ID: "dc-0", Severity: SeverityHigh, Pattern: "destructive-commands", Message: "m0", Regex: "r0"},
{ID: "dc-1", Severity: SeverityHigh, Pattern: "destructive-commands", Message: "m1", Regex: "r1"},
{ID: "pi-0", Severity: SeverityCritical, Pattern: "prompt-injection", Message: "m2", Regex: "r2"},
}
overlay := []yamlRule{
{Pattern: "destructive-commands", Severity: SeverityMedium},
}
merged := mergeYAMLRules(base, overlay)
compiled, err := compileRules(merged)
if err != nil {
t.Fatalf("compile: %v", err)
}
if len(compiled) != 3 {
t.Fatalf("want 3 rules, got %d", len(compiled))
}
for _, r := range compiled {
if r.Pattern == "destructive-commands" && r.Severity != SeverityMedium {
t.Fatalf("want MEDIUM for %s, got %s", r.ID, r.Severity)
}
}
if compiled[2].Severity != SeverityCritical {
t.Fatalf("pi-0 severity should be unchanged, got %s", compiled[2].Severity)
}
}
func TestMergeRules_PatternDisableThenIDEnable(t *testing.T) {
resetForTest()
boolFalse := false
boolTrue := true
base := []yamlRule{
{ID: "ca-0", Severity: SeverityCritical, Pattern: "credential-access", Message: "m0", Regex: "r0"},
{ID: "ca-1", Severity: SeverityCritical, Pattern: "credential-access", Message: "m1", Regex: "r1"},
{ID: "ca-2", Severity: SeverityHigh, Pattern: "credential-access", Message: "m2", Regex: "r2"},
}
overlay := []yamlRule{
{Pattern: "credential-access", Enabled: &boolFalse},
{ID: "ca-1", Enabled: &boolTrue},
}
merged := mergeYAMLRules(base, overlay)
compiled, err := compileRules(merged)
if err != nil {
t.Fatalf("compile: %v", err)
}
if len(compiled) != 1 {
t.Fatalf("want 1 rule, got %d", len(compiled))
}
if compiled[0].ID != "ca-1" {
t.Fatalf("want ca-1, got %s", compiled[0].ID)
}
}
func TestMergeRules_SeverityOnlyOverride(t *testing.T) {
base := []yamlRule{
{ID: "rule-1", Severity: SeverityHigh, Pattern: "p1", Message: "msg", Regex: "r1"},
}
overlay := []yamlRule{
{ID: "rule-1", Severity: SeverityLow},
}
merged := mergeYAMLRules(base, overlay)
if len(merged) != 1 {
t.Fatalf("expected 1 rule, got %d", len(merged))
}
if merged[0].Severity != SeverityLow {
t.Errorf("severity: want LOW, got %s", merged[0].Severity)
}
if merged[0].Regex != "r1" {
t.Errorf("regex should be preserved, got %q", merged[0].Regex)
}
if merged[0].Message != "msg" {
t.Errorf("message should be preserved, got %q", merged[0].Message)
}
if merged[0].Pattern != "p1" {
t.Errorf("pattern should be preserved, got %q", merged[0].Pattern)
}
// Verify it compiles successfully.
compiled, err := compileRules(merged)
if err != nil {
t.Fatalf("compileRules should succeed: %v", err)
}
if len(compiled) != 1 {
t.Fatalf("expected 1 compiled rule, got %d", len(compiled))
}
if compiled[0].Severity != SeverityLow {
t.Errorf("compiled severity: want LOW, got %s", compiled[0].Severity)
}
}
func TestLoadUserRules_NotFound(t *testing.T) {
rules, err := loadUserRules("/nonexistent/path/audit-rules.yaml")
if err != nil {
t.Fatalf("expected nil error for missing file, got: %v", err)
}
if rules != nil {
t.Errorf("expected nil rules for missing file, got %d rules", len(rules))
}
}
func TestRulesWithProject(t *testing.T) {
resetForTest()
// Set up a temp dir as config home (so global user rules don't interfere)
tmpDir := t.TempDir()
cfgDir := filepath.Join(tmpDir, "config")
os.MkdirAll(cfgDir, 0755)
t.Setenv("SKILLSHARE_CONFIG", filepath.Join(cfgDir, "config.yaml"))
// Create a project with custom rules
projectRoot := filepath.Join(tmpDir, "project")
os.MkdirAll(filepath.Join(projectRoot, ".skillshare"), 0755)
os.WriteFile(filepath.Join(projectRoot, ".skillshare", "audit-rules.yaml"), []byte(`rules:
- id: custom-project-rule
severity: HIGH
pattern: custom
message: "Custom project rule"
regex: 'CUSTOM_PATTERN'
`), 0644)
rules, err := RulesWithProject(projectRoot)
if err != nil {
t.Fatalf("RulesWithProject() error: %v", err)
}
// Should have builtin rules + 1 custom rule
builtinCount := len(builtinYAML())
if len(rules) != builtinCount+1 {
t.Errorf("expected %d rules (%d builtin + 1 custom), got %d", builtinCount+1, builtinCount, len(rules))
}
// Find the custom rule
found := false
for _, r := range rules {
if r.ID == "custom-project-rule" {
found = true
if r.Pattern != "custom" {
t.Errorf("expected pattern 'custom', got %s", r.Pattern)
}
}
}
if !found {
t.Error("custom-project-rule not found in merged rules")
}
}
func TestGlobalUserRules(t *testing.T) {
resetForTest()
tmpDir := t.TempDir()
cfgDir := filepath.Join(tmpDir, "config")
os.MkdirAll(cfgDir, 0755)
t.Setenv("SKILLSHARE_CONFIG", filepath.Join(cfgDir, "config.yaml"))
// Create global user rules that add a custom rule
os.WriteFile(filepath.Join(cfgDir, "audit-rules.yaml"), []byte(`rules:
- id: global-custom
severity: MEDIUM
pattern: global-test
message: "Global custom rule"
regex: 'GLOBAL_TEST'
`), 0644)
rules, err := Rules()
if err != nil {
t.Fatalf("Rules() error: %v", err)
}
expected := len(builtinYAML()) + 1
if len(rules) != expected {
t.Errorf("expected %d rules (%d builtin + 1 global custom), got %d", expected, len(builtinYAML()), len(rules))
}
found := false
for _, r := range rules {
if r.ID == "global-custom" {
found = true
}
}
if !found {
t.Error("global-custom rule not found")
}
}
func TestGlobalUserRules_DisableBuiltin(t *testing.T) {
resetForTest()
tmpDir := t.TempDir()
cfgDir := filepath.Join(tmpDir, "config")
os.MkdirAll(cfgDir, 0755)
t.Setenv("SKILLSHARE_CONFIG", filepath.Join(cfgDir, "config.yaml"))
// Disable a builtin rule
os.WriteFile(filepath.Join(cfgDir, "audit-rules.yaml"), []byte(`rules:
- id: insecure-http-0
enabled: false
`), 0644)
rules, err := Rules()
if err != nil {
t.Fatalf("Rules() error: %v", err)
}
expected := len(builtinYAML()) - 1
if len(rules) != expected {
t.Errorf("expected %d rules (%d builtin - 1 disabled), got %d", expected, len(builtinYAML()), len(rules))
}
for _, r := range rules {
if r.ID == "insecure-http-0" {
t.Error("insecure-http-0 should be disabled")
}
}
}
// TestProjectRules_DisableTierRule reproduces issue #204: setting
// `enabled: false` for a tier rule in a project audit-rules.yaml must
// suppress that tier finding during a skill scan.
func TestProjectRules_DisableTierRule(t *testing.T) {
resetForTest()
tmpDir := t.TempDir()
cfgDir := filepath.Join(tmpDir, "config")
os.MkdirAll(cfgDir, 0755)
t.Setenv("SKILLSHARE_CONFIG", filepath.Join(cfgDir, "config.yaml"))
// Skill whose SKILL.md combines an interpreter (python3) with a
// network command (curl) inside a code fence → tier-interpreter-network.
skillDir := filepath.Join(tmpDir, "skill")
os.MkdirAll(skillDir, 0755)
skillMD := "---\nname: demo\ndescription: demo\n---\n\n```bash\npython3 script.py\ncurl https://example.com\n```\n"
os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte(skillMD), 0644)
projectRoot := filepath.Join(tmpDir, "project")
os.MkdirAll(filepath.Join(projectRoot, ".skillshare"), 0755)
// Baseline: no rule overrides → tier-interpreter-network fires.
res, err := ScanSkillForProject(skillDir, projectRoot)
if err != nil {
t.Fatalf("ScanSkillForProject: %v", err)
}
if !hasFinding(res.Findings, "tier-interpreter-network", SeverityMedium) {
t.Fatal("expected tier-interpreter-network finding before disabling")
}
// Disable the tier rule via the project audit-rules.yaml.
resetForTest()
os.WriteFile(filepath.Join(projectRoot, ".skillshare", "audit-rules.yaml"), []byte(
"rules:\n - id: tier-interpreter-network\n enabled: false\n"), 0644)
res, err = ScanSkillForProject(skillDir, projectRoot)
if err != nil {
t.Fatalf("ScanSkillForProject (disabled): %v", err)
}
if hasFinding(res.Findings, "tier-interpreter-network", SeverityMedium) {
t.Error("tier-interpreter-network should be suppressed when disabled in audit-rules.yaml")
}
}