mirror of
https://github.com/runkids/skillshare.git
synced 2026-11-19 02:17:09 +00:00
Tier and cross-skill findings are generated programmatically rather than compiled from the rule list, so they bypassed the disabled-ID filter applied at rule-compile time. Setting enabled:false in audit-rules.yaml marked the rule disabled in the listing but the scan still fired it at full severity. Filter synthetic findings against the disabled-ID set in both the skill-scan and single-file paths, threading the set through the cross-skill call sites (analyzer, cmd, server). Fixes #204.
497 lines
14 KiB
Go
497 lines
14 KiB
Go
package audit
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestLoadBuiltinRules(t *testing.T) {
|
|
resetForTest()
|
|
rules, err := loadBuiltinRules()
|
|
if err != nil {
|
|
t.Fatalf("loadBuiltinRules() error: %v", err)
|
|
}
|
|
if len(rules) == 0 {
|
|
t.Fatal("expected non-empty rules")
|
|
}
|
|
// Verify each rule has required fields
|
|
for _, r := range rules {
|
|
if r.ID == "" {
|
|
t.Error("rule has empty ID")
|
|
}
|
|
if r.Severity == "" {
|
|
t.Error("rule has empty Severity")
|
|
}
|
|
if r.Pattern == "" {
|
|
t.Errorf("rule %s has empty Pattern", r.ID)
|
|
}
|
|
if r.Message == "" {
|
|
t.Errorf("rule %s has empty Message", r.ID)
|
|
}
|
|
if r.Regex == nil {
|
|
t.Errorf("rule %s has nil Regex", r.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRuleCount(t *testing.T) {
|
|
resetForTest()
|
|
rules, err := loadBuiltinRules()
|
|
if err != nil {
|
|
t.Fatalf("loadBuiltinRules() error: %v", err)
|
|
}
|
|
expected := len(builtinYAML())
|
|
if got := len(rules); got != expected {
|
|
t.Errorf("expected %d builtin rules, got %d", expected, got)
|
|
}
|
|
}
|
|
|
|
func TestCompileRules_InvalidRegex(t *testing.T) {
|
|
yr := []yamlRule{{
|
|
ID: "bad-regex",
|
|
Severity: SeverityHigh,
|
|
Pattern: "test",
|
|
Message: "test",
|
|
Regex: "[invalid",
|
|
}}
|
|
_, err := compileRules(yr)
|
|
if err == nil {
|
|
t.Fatal("expected error for invalid regex")
|
|
}
|
|
if !strings.Contains(err.Error(), "bad-regex") {
|
|
t.Errorf("error should mention rule ID, got: %s", err)
|
|
}
|
|
}
|
|
|
|
func TestCompileRules_InvalidExcludeRegex(t *testing.T) {
|
|
yr := []yamlRule{{
|
|
ID: "bad-exclude",
|
|
Severity: SeverityHigh,
|
|
Pattern: "test",
|
|
Message: "test",
|
|
Regex: "ok",
|
|
Exclude: "[invalid",
|
|
}}
|
|
_, err := compileRules(yr)
|
|
if err == nil {
|
|
t.Fatal("expected error for invalid exclude regex")
|
|
}
|
|
if !strings.Contains(err.Error(), "bad-exclude") {
|
|
t.Errorf("error should mention rule ID, got: %s", err)
|
|
}
|
|
}
|
|
|
|
func TestCompileRules_ValidateSeverity(t *testing.T) {
|
|
yr := []yamlRule{{
|
|
ID: "typo-sev",
|
|
Severity: "CRTICAL",
|
|
Pattern: "test",
|
|
Message: "test",
|
|
Regex: "test",
|
|
}}
|
|
_, err := compileRules(yr)
|
|
if err == nil {
|
|
t.Fatal("expected error for invalid severity")
|
|
}
|
|
if !strings.Contains(err.Error(), "CRTICAL") {
|
|
t.Errorf("error should mention bad severity, got: %s", err)
|
|
}
|
|
}
|
|
|
|
func TestCompileRules_EmptyRegex(t *testing.T) {
|
|
yr := []yamlRule{{
|
|
ID: "empty-regex",
|
|
Severity: SeverityHigh,
|
|
Pattern: "test",
|
|
Message: "test",
|
|
Regex: "",
|
|
}}
|
|
_, err := compileRules(yr)
|
|
if err == nil {
|
|
t.Fatal("expected error for empty regex")
|
|
}
|
|
}
|
|
|
|
func TestExcludeRegex(t *testing.T) {
|
|
// suspicious-fetch should have an exclude for localhost
|
|
resetForTest()
|
|
rules, err := loadBuiltinRules()
|
|
if err != nil {
|
|
t.Fatalf("loadBuiltinRules() error: %v", err)
|
|
}
|
|
|
|
var fetchRule *rule
|
|
for i := range rules {
|
|
if rules[i].ID == "suspicious-fetch-0" {
|
|
fetchRule = &rules[i]
|
|
break
|
|
}
|
|
}
|
|
if fetchRule == nil {
|
|
t.Fatal("suspicious-fetch-0 rule not found")
|
|
}
|
|
if fetchRule.Exclude == nil {
|
|
t.Fatal("suspicious-fetch-0 should have Exclude regex")
|
|
}
|
|
|
|
// External URL should match regex but NOT exclude
|
|
if !fetchRule.Regex.MatchString("curl https://evil.com") {
|
|
t.Error("should match external URL")
|
|
}
|
|
if fetchRule.Exclude.MatchString("curl https://evil.com") {
|
|
t.Error("should not exclude external URL")
|
|
}
|
|
|
|
// Localhost should match both regex and exclude
|
|
if !fetchRule.Regex.MatchString("curl http://localhost:3000/api") {
|
|
t.Error("should match localhost URL")
|
|
}
|
|
if !fetchRule.Exclude.MatchString("curl http://localhost:3000/api") {
|
|
t.Error("should exclude localhost URL")
|
|
}
|
|
}
|
|
|
|
func TestMergeRules_Append(t *testing.T) {
|
|
base := []yamlRule{
|
|
{ID: "rule-1", Severity: SeverityHigh, Pattern: "p1", Message: "m1", Regex: "r1"},
|
|
}
|
|
overlay := []yamlRule{
|
|
{ID: "rule-2", Severity: SeverityMedium, Pattern: "p2", Message: "m2", Regex: "r2"},
|
|
}
|
|
merged := mergeYAMLRules(base, overlay)
|
|
if len(merged) != 2 {
|
|
t.Fatalf("expected 2 rules, got %d", len(merged))
|
|
}
|
|
if merged[1].ID != "rule-2" {
|
|
t.Errorf("expected appended rule-2, got %s", merged[1].ID)
|
|
}
|
|
}
|
|
|
|
func TestMergeRules_Disable(t *testing.T) {
|
|
base := []yamlRule{
|
|
{ID: "rule-1", Severity: SeverityHigh, Pattern: "p1", Message: "m1", Regex: "r1"},
|
|
{ID: "rule-2", Severity: SeverityHigh, Pattern: "p2", Message: "m2", Regex: "r2"},
|
|
}
|
|
f := false
|
|
overlay := []yamlRule{
|
|
{ID: "rule-1", Enabled: &f},
|
|
}
|
|
merged := mergeYAMLRules(base, overlay)
|
|
|
|
// After compiling, rule-1 should be excluded
|
|
compiled, err := compileRules(merged)
|
|
if err != nil {
|
|
t.Fatalf("compileRules error: %v", err)
|
|
}
|
|
if len(compiled) != 1 {
|
|
t.Fatalf("expected 1 rule after disabling, got %d", len(compiled))
|
|
}
|
|
if compiled[0].ID != "rule-2" {
|
|
t.Errorf("expected rule-2 to survive, got %s", compiled[0].ID)
|
|
}
|
|
}
|
|
|
|
func TestMergeRules_Override(t *testing.T) {
|
|
base := []yamlRule{
|
|
{ID: "rule-1", Severity: SeverityHigh, Pattern: "p1", Message: "original", Regex: "r1"},
|
|
}
|
|
overlay := []yamlRule{
|
|
{ID: "rule-1", Severity: SeverityCritical, Pattern: "p1", Message: "overridden", Regex: "r1"},
|
|
}
|
|
merged := mergeYAMLRules(base, overlay)
|
|
if len(merged) != 1 {
|
|
t.Fatalf("expected 1 rule, got %d", len(merged))
|
|
}
|
|
if merged[0].Message != "overridden" {
|
|
t.Errorf("expected overridden message, got %s", merged[0].Message)
|
|
}
|
|
if merged[0].Severity != SeverityCritical {
|
|
t.Errorf("expected CRITICAL severity, got %s", merged[0].Severity)
|
|
}
|
|
}
|
|
|
|
func TestMergeRules_PatternDisable(t *testing.T) {
|
|
resetForTest()
|
|
boolFalse := false
|
|
base := []yamlRule{
|
|
{ID: "pi-0", Severity: SeverityCritical, Pattern: "prompt-injection", Message: "m0", Regex: "r0"},
|
|
{ID: "pi-1", Severity: SeverityHigh, Pattern: "prompt-injection", Message: "m1", Regex: "r1"},
|
|
{ID: "dc-0", Severity: SeverityHigh, Pattern: "destructive-commands", Message: "m2", Regex: "r2"},
|
|
}
|
|
overlay := []yamlRule{
|
|
{Pattern: "prompt-injection", Enabled: &boolFalse},
|
|
}
|
|
merged := mergeYAMLRules(base, overlay)
|
|
compiled, err := compileRules(merged)
|
|
if err != nil {
|
|
t.Fatalf("compile: %v", err)
|
|
}
|
|
if len(compiled) != 1 {
|
|
t.Fatalf("want 1 rule, got %d", len(compiled))
|
|
}
|
|
if compiled[0].ID != "dc-0" {
|
|
t.Fatalf("want dc-0, got %s", compiled[0].ID)
|
|
}
|
|
}
|
|
|
|
func TestMergeRules_PatternSeverityOverride(t *testing.T) {
|
|
resetForTest()
|
|
base := []yamlRule{
|
|
{ID: "dc-0", Severity: SeverityHigh, Pattern: "destructive-commands", Message: "m0", Regex: "r0"},
|
|
{ID: "dc-1", Severity: SeverityHigh, Pattern: "destructive-commands", Message: "m1", Regex: "r1"},
|
|
{ID: "pi-0", Severity: SeverityCritical, Pattern: "prompt-injection", Message: "m2", Regex: "r2"},
|
|
}
|
|
overlay := []yamlRule{
|
|
{Pattern: "destructive-commands", Severity: SeverityMedium},
|
|
}
|
|
merged := mergeYAMLRules(base, overlay)
|
|
compiled, err := compileRules(merged)
|
|
if err != nil {
|
|
t.Fatalf("compile: %v", err)
|
|
}
|
|
if len(compiled) != 3 {
|
|
t.Fatalf("want 3 rules, got %d", len(compiled))
|
|
}
|
|
for _, r := range compiled {
|
|
if r.Pattern == "destructive-commands" && r.Severity != SeverityMedium {
|
|
t.Fatalf("want MEDIUM for %s, got %s", r.ID, r.Severity)
|
|
}
|
|
}
|
|
if compiled[2].Severity != SeverityCritical {
|
|
t.Fatalf("pi-0 severity should be unchanged, got %s", compiled[2].Severity)
|
|
}
|
|
}
|
|
|
|
func TestMergeRules_PatternDisableThenIDEnable(t *testing.T) {
|
|
resetForTest()
|
|
boolFalse := false
|
|
boolTrue := true
|
|
base := []yamlRule{
|
|
{ID: "ca-0", Severity: SeverityCritical, Pattern: "credential-access", Message: "m0", Regex: "r0"},
|
|
{ID: "ca-1", Severity: SeverityCritical, Pattern: "credential-access", Message: "m1", Regex: "r1"},
|
|
{ID: "ca-2", Severity: SeverityHigh, Pattern: "credential-access", Message: "m2", Regex: "r2"},
|
|
}
|
|
overlay := []yamlRule{
|
|
{Pattern: "credential-access", Enabled: &boolFalse},
|
|
{ID: "ca-1", Enabled: &boolTrue},
|
|
}
|
|
merged := mergeYAMLRules(base, overlay)
|
|
compiled, err := compileRules(merged)
|
|
if err != nil {
|
|
t.Fatalf("compile: %v", err)
|
|
}
|
|
if len(compiled) != 1 {
|
|
t.Fatalf("want 1 rule, got %d", len(compiled))
|
|
}
|
|
if compiled[0].ID != "ca-1" {
|
|
t.Fatalf("want ca-1, got %s", compiled[0].ID)
|
|
}
|
|
}
|
|
|
|
func TestMergeRules_SeverityOnlyOverride(t *testing.T) {
|
|
base := []yamlRule{
|
|
{ID: "rule-1", Severity: SeverityHigh, Pattern: "p1", Message: "msg", Regex: "r1"},
|
|
}
|
|
overlay := []yamlRule{
|
|
{ID: "rule-1", Severity: SeverityLow},
|
|
}
|
|
merged := mergeYAMLRules(base, overlay)
|
|
if len(merged) != 1 {
|
|
t.Fatalf("expected 1 rule, got %d", len(merged))
|
|
}
|
|
if merged[0].Severity != SeverityLow {
|
|
t.Errorf("severity: want LOW, got %s", merged[0].Severity)
|
|
}
|
|
if merged[0].Regex != "r1" {
|
|
t.Errorf("regex should be preserved, got %q", merged[0].Regex)
|
|
}
|
|
if merged[0].Message != "msg" {
|
|
t.Errorf("message should be preserved, got %q", merged[0].Message)
|
|
}
|
|
if merged[0].Pattern != "p1" {
|
|
t.Errorf("pattern should be preserved, got %q", merged[0].Pattern)
|
|
}
|
|
// Verify it compiles successfully.
|
|
compiled, err := compileRules(merged)
|
|
if err != nil {
|
|
t.Fatalf("compileRules should succeed: %v", err)
|
|
}
|
|
if len(compiled) != 1 {
|
|
t.Fatalf("expected 1 compiled rule, got %d", len(compiled))
|
|
}
|
|
if compiled[0].Severity != SeverityLow {
|
|
t.Errorf("compiled severity: want LOW, got %s", compiled[0].Severity)
|
|
}
|
|
}
|
|
|
|
func TestLoadUserRules_NotFound(t *testing.T) {
|
|
rules, err := loadUserRules("/nonexistent/path/audit-rules.yaml")
|
|
if err != nil {
|
|
t.Fatalf("expected nil error for missing file, got: %v", err)
|
|
}
|
|
if rules != nil {
|
|
t.Errorf("expected nil rules for missing file, got %d rules", len(rules))
|
|
}
|
|
}
|
|
|
|
func TestRulesWithProject(t *testing.T) {
|
|
resetForTest()
|
|
|
|
// Set up a temp dir as config home (so global user rules don't interfere)
|
|
tmpDir := t.TempDir()
|
|
cfgDir := filepath.Join(tmpDir, "config")
|
|
os.MkdirAll(cfgDir, 0755)
|
|
t.Setenv("SKILLSHARE_CONFIG", filepath.Join(cfgDir, "config.yaml"))
|
|
|
|
// Create a project with custom rules
|
|
projectRoot := filepath.Join(tmpDir, "project")
|
|
os.MkdirAll(filepath.Join(projectRoot, ".skillshare"), 0755)
|
|
os.WriteFile(filepath.Join(projectRoot, ".skillshare", "audit-rules.yaml"), []byte(`rules:
|
|
- id: custom-project-rule
|
|
severity: HIGH
|
|
pattern: custom
|
|
message: "Custom project rule"
|
|
regex: 'CUSTOM_PATTERN'
|
|
`), 0644)
|
|
|
|
rules, err := RulesWithProject(projectRoot)
|
|
if err != nil {
|
|
t.Fatalf("RulesWithProject() error: %v", err)
|
|
}
|
|
|
|
// Should have builtin rules + 1 custom rule
|
|
builtinCount := len(builtinYAML())
|
|
if len(rules) != builtinCount+1 {
|
|
t.Errorf("expected %d rules (%d builtin + 1 custom), got %d", builtinCount+1, builtinCount, len(rules))
|
|
}
|
|
|
|
// Find the custom rule
|
|
found := false
|
|
for _, r := range rules {
|
|
if r.ID == "custom-project-rule" {
|
|
found = true
|
|
if r.Pattern != "custom" {
|
|
t.Errorf("expected pattern 'custom', got %s", r.Pattern)
|
|
}
|
|
}
|
|
}
|
|
if !found {
|
|
t.Error("custom-project-rule not found in merged rules")
|
|
}
|
|
}
|
|
|
|
func TestGlobalUserRules(t *testing.T) {
|
|
resetForTest()
|
|
|
|
tmpDir := t.TempDir()
|
|
cfgDir := filepath.Join(tmpDir, "config")
|
|
os.MkdirAll(cfgDir, 0755)
|
|
t.Setenv("SKILLSHARE_CONFIG", filepath.Join(cfgDir, "config.yaml"))
|
|
|
|
// Create global user rules that add a custom rule
|
|
os.WriteFile(filepath.Join(cfgDir, "audit-rules.yaml"), []byte(`rules:
|
|
- id: global-custom
|
|
severity: MEDIUM
|
|
pattern: global-test
|
|
message: "Global custom rule"
|
|
regex: 'GLOBAL_TEST'
|
|
`), 0644)
|
|
|
|
rules, err := Rules()
|
|
if err != nil {
|
|
t.Fatalf("Rules() error: %v", err)
|
|
}
|
|
|
|
expected := len(builtinYAML()) + 1
|
|
if len(rules) != expected {
|
|
t.Errorf("expected %d rules (%d builtin + 1 global custom), got %d", expected, len(builtinYAML()), len(rules))
|
|
}
|
|
|
|
found := false
|
|
for _, r := range rules {
|
|
if r.ID == "global-custom" {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Error("global-custom rule not found")
|
|
}
|
|
}
|
|
|
|
func TestGlobalUserRules_DisableBuiltin(t *testing.T) {
|
|
resetForTest()
|
|
|
|
tmpDir := t.TempDir()
|
|
cfgDir := filepath.Join(tmpDir, "config")
|
|
os.MkdirAll(cfgDir, 0755)
|
|
t.Setenv("SKILLSHARE_CONFIG", filepath.Join(cfgDir, "config.yaml"))
|
|
|
|
// Disable a builtin rule
|
|
os.WriteFile(filepath.Join(cfgDir, "audit-rules.yaml"), []byte(`rules:
|
|
- id: insecure-http-0
|
|
enabled: false
|
|
`), 0644)
|
|
|
|
rules, err := Rules()
|
|
if err != nil {
|
|
t.Fatalf("Rules() error: %v", err)
|
|
}
|
|
|
|
expected := len(builtinYAML()) - 1
|
|
if len(rules) != expected {
|
|
t.Errorf("expected %d rules (%d builtin - 1 disabled), got %d", expected, len(builtinYAML()), len(rules))
|
|
}
|
|
|
|
for _, r := range rules {
|
|
if r.ID == "insecure-http-0" {
|
|
t.Error("insecure-http-0 should be disabled")
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestProjectRules_DisableTierRule reproduces issue #204: setting
|
|
// `enabled: false` for a tier rule in a project audit-rules.yaml must
|
|
// suppress that tier finding during a skill scan.
|
|
func TestProjectRules_DisableTierRule(t *testing.T) {
|
|
resetForTest()
|
|
|
|
tmpDir := t.TempDir()
|
|
cfgDir := filepath.Join(tmpDir, "config")
|
|
os.MkdirAll(cfgDir, 0755)
|
|
t.Setenv("SKILLSHARE_CONFIG", filepath.Join(cfgDir, "config.yaml"))
|
|
|
|
// Skill whose SKILL.md combines an interpreter (python3) with a
|
|
// network command (curl) inside a code fence → tier-interpreter-network.
|
|
skillDir := filepath.Join(tmpDir, "skill")
|
|
os.MkdirAll(skillDir, 0755)
|
|
skillMD := "---\nname: demo\ndescription: demo\n---\n\n```bash\npython3 script.py\ncurl https://example.com\n```\n"
|
|
os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte(skillMD), 0644)
|
|
|
|
projectRoot := filepath.Join(tmpDir, "project")
|
|
os.MkdirAll(filepath.Join(projectRoot, ".skillshare"), 0755)
|
|
|
|
// Baseline: no rule overrides → tier-interpreter-network fires.
|
|
res, err := ScanSkillForProject(skillDir, projectRoot)
|
|
if err != nil {
|
|
t.Fatalf("ScanSkillForProject: %v", err)
|
|
}
|
|
if !hasFinding(res.Findings, "tier-interpreter-network", SeverityMedium) {
|
|
t.Fatal("expected tier-interpreter-network finding before disabling")
|
|
}
|
|
|
|
// Disable the tier rule via the project audit-rules.yaml.
|
|
resetForTest()
|
|
os.WriteFile(filepath.Join(projectRoot, ".skillshare", "audit-rules.yaml"), []byte(
|
|
"rules:\n - id: tier-interpreter-network\n enabled: false\n"), 0644)
|
|
|
|
res, err = ScanSkillForProject(skillDir, projectRoot)
|
|
if err != nil {
|
|
t.Fatalf("ScanSkillForProject (disabled): %v", err)
|
|
}
|
|
if hasFinding(res.Findings, "tier-interpreter-network", SeverityMedium) {
|
|
t.Error("tier-interpreter-network should be suppressed when disabled in audit-rules.yaml")
|
|
}
|
|
}
|