Files
skillshare/internal/audit/crossskill_test.go
T
Willie 6e86537987 fix(audit): honor enabled:false for synthetic tier and cross-skill findings
Tier and cross-skill findings are generated programmatically rather than
compiled from the rule list, so they bypassed the disabled-ID filter applied
at rule-compile time. Setting enabled:false in audit-rules.yaml marked the
rule disabled in the listing but the scan still fired it at full severity.

Filter synthetic findings against the disabled-ID set in both the skill-scan
and single-file paths, threading the set through the cross-skill call sites
(analyzer, cmd, server). Fixes #204.
2026-06-05 20:51:32 +08:00

388 lines
11 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package audit
import (
"fmt"
"testing"
)
// helper to build a Result with given findings and tier profile.
func makeResult(name string, tp TierProfile, findings ...Finding) *Result {
r := &Result{
SkillName: name,
Findings: findings,
TierProfile: tp,
}
r.updateRisk()
return r
}
func tierWith(tiers ...CommandTier) TierProfile {
var p TierProfile
for _, t := range tiers {
p.Add(t)
}
return p
}
func credFinding() Finding {
return Finding{Severity: SeverityHigh, Pattern: "credential-access", Message: "reads creds", File: "SKILL.md", Line: 1}
}
func dataflowTaintFinding() Finding {
return Finding{Severity: SeverityHigh, Pattern: "dataflow-taint", Message: "taint flow", File: "SKILL.md", Line: 2}
}
func highFinding() Finding {
return Finding{Severity: SeverityHigh, Pattern: "shell-execution", Message: "shell exec", File: "SKILL.md", Line: 3}
}
func lowFinding() Finding {
return Finding{Severity: SeverityLow, Pattern: "dangling-link", Message: "broken link", File: "SKILL.md", Line: 4}
}
func TestExtractCapability(t *testing.T) {
tests := []struct {
name string
result *Result
wantCred bool
wantNet bool
wantPriv bool
wantInterpreter bool
wantHigh bool
}{
{
name: "credential finding sets HasCredReads",
result: makeResult("a", TierProfile{}, credFinding()),
wantCred: true,
wantHigh: true,
},
{
name: "dataflow-taint sets HasCredReads",
result: makeResult("a", TierProfile{}, dataflowTaintFinding()),
wantCred: true,
wantHigh: true,
},
{
name: "tier profile sets capabilities",
result: makeResult("a", tierWith(TierNetwork, TierPrivilege)),
wantNet: true, wantPriv: true,
},
{
name: "mixed findings and tiers",
result: makeResult("a", tierWith(TierNetwork), credFinding()),
wantCred: true, wantNet: true, wantHigh: true,
},
{
name: "low-only findings do not set HasHighPlus",
result: makeResult("a", TierProfile{}, lowFinding()),
},
{
name: "interpreter tier sets HasInterpreter",
result: makeResult("a", tierWith(TierInterpreter)),
wantInterpreter: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cap := extractCapability(tt.result)
if cap.HasCredReads != tt.wantCred {
t.Errorf("HasCredReads = %v, want %v", cap.HasCredReads, tt.wantCred)
}
if cap.HasNetSend != tt.wantNet {
t.Errorf("HasNetSend = %v, want %v", cap.HasNetSend, tt.wantNet)
}
if cap.HasPrivilege != tt.wantPriv {
t.Errorf("HasPrivilege = %v, want %v", cap.HasPrivilege, tt.wantPriv)
}
if cap.HasInterpreter != tt.wantInterpreter {
t.Errorf("HasInterpreter = %v, want %v", cap.HasInterpreter, tt.wantInterpreter)
}
if cap.HasHighPlus != tt.wantHigh {
t.Errorf("HasHighPlus = %v, want %v", cap.HasHighPlus, tt.wantHigh)
}
})
}
}
func TestCrossSkillAnalysis(t *testing.T) {
tests := []struct {
name string
results []*Result
wantNil bool
wantPatterns []string // expected patterns in findings (one per rule)
}{
{
name: "nil on empty input",
results: nil,
wantNil: true,
},
{
name: "nil on single skill",
results: []*Result{makeResult("a", tierWith(TierNetwork), credFinding())},
wantNil: true,
},
{
name: "source + sink basic pair",
results: []*Result{
makeResult("reader", TierProfile{}, credFinding()), // cred read, no network
makeResult("sender", tierWith(TierNetwork)), // network, no cred read
},
wantPatterns: []string{"cross-skill-exfiltration"},
},
{
name: "source + sink: both have both caps — no finding",
results: []*Result{
makeResult("a", tierWith(TierNetwork), credFinding()), // has both
makeResult("b", tierWith(TierNetwork), credFinding()), // has both
},
wantNil: true,
},
{
name: "source + sink: 3-skill chain produces 1 summary finding",
results: []*Result{
makeResult("reader1", TierProfile{}, credFinding()),
makeResult("sender1", tierWith(TierNetwork)),
makeResult("reader2", TierProfile{}, dataflowTaintFinding()),
},
// Set-based: one summary finding listing both readers × the sender.
wantPatterns: []string{"cross-skill-exfiltration"},
},
{
name: "privilege + network basic pair",
results: []*Result{
makeResult("admin", tierWith(TierPrivilege)),
makeResult("fetcher", tierWith(TierNetwork)),
},
wantPatterns: []string{"cross-skill-privilege-network"},
},
{
name: "privilege + network: same skill has both — no finding",
results: []*Result{
makeResult("a", tierWith(TierPrivilege, TierNetwork)),
makeResult("b", TierProfile{}),
},
wantNil: true,
},
{
name: "stealth + high-risk basic pair",
results: []*Result{
makeResult("sneaky", tierWith(TierStealth)),
makeResult("dangerous", TierProfile{}, highFinding()),
},
wantPatterns: []string{"cross-skill-stealth"},
},
{
name: "stealth alone — no high-risk partner — no finding",
results: []*Result{
makeResult("sneaky", tierWith(TierStealth)),
makeResult("clean", TierProfile{}, lowFinding()),
},
wantNil: true,
},
{
name: "stealth skill with high findings paired with another high-risk skill — one finding",
results: []*Result{
makeResult("sneaky", tierWith(TierStealth), highFinding()),
makeResult("dangerous", TierProfile{}, highFinding()),
},
wantPatterns: []string{"cross-skill-stealth"},
},
{
name: "stealth self-pair only — single skill is both stealth and high — no finding",
results: []*Result{
makeResult("solo-stealth", tierWith(TierStealth), highFinding()),
makeResult("clean", TierProfile{}, lowFinding()),
},
wantNil: true,
},
{
name: "all clean skills — no findings",
results: []*Result{
makeResult("a", TierProfile{}),
makeResult("b", TierProfile{}, lowFinding()),
makeResult("c", tierWith(TierReadOnly)),
},
wantNil: true,
},
{
name: "combined rules: cred+net and stealth+high",
results: []*Result{
makeResult("reader", TierProfile{}, credFinding()), // cred read, high finding
makeResult("sender", tierWith(TierNetwork)), // network
makeResult("sneaky", tierWith(TierStealth)), // stealth
},
wantPatterns: []string{
"cross-skill-exfiltration", // reader × sender
"cross-skill-stealth", // sneaky × reader (reader has high finding)
},
},
{
name: "credential × interpreter basic pair",
results: []*Result{
makeResult("reader", TierProfile{}, credFinding()), // cred read, no network
makeResult("scripting", tierWith(TierInterpreter)), // interpreter, no network
},
wantPatterns: []string{"cross-skill-cred-interpreter"},
},
{
name: "credential × interpreter: interpreter also has network — no finding (interpreter is not in interpreterOnly)",
results: []*Result{
makeResult("reader", TierProfile{}, credFinding()),
makeResult("scripting", tierWith(TierInterpreter, TierNetwork)),
},
// interpreter has network → not in interpreterOnly set, but in netNoCred → exfiltration fires instead
wantPatterns: []string{"cross-skill-exfiltration"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := CrossSkillAnalysis(tt.results, nil)
if tt.wantNil {
if result != nil {
t.Fatalf("expected nil, got %d findings: %v", len(result.Findings), result.Findings)
}
return
}
if result == nil {
t.Fatal("expected non-nil result")
}
if result.SkillName != "_cross-skill" {
t.Errorf("SkillName = %q, want _cross-skill", result.SkillName)
}
// Collect actual patterns.
var gotPatterns []string
for _, f := range result.Findings {
gotPatterns = append(gotPatterns, f.Pattern)
if f.File != "." {
t.Errorf("finding File = %q, want \".\"", f.File)
}
if f.Line != 0 {
t.Errorf("finding Line = %d, want 0", f.Line)
}
}
// Check exact pattern match (set-based produces exactly one per rule).
if len(gotPatterns) != len(tt.wantPatterns) {
t.Fatalf("got %d findings %v, want %d %v", len(gotPatterns), gotPatterns, len(tt.wantPatterns), tt.wantPatterns)
}
patternCounts := make(map[string]int)
for _, p := range gotPatterns {
patternCounts[p]++
}
wantCounts := make(map[string]int)
for _, p := range tt.wantPatterns {
wantCounts[p]++
}
for p, wc := range wantCounts {
if gc := patternCounts[p]; gc != wc {
t.Errorf("pattern %q: got %d, want %d", p, gc, wc)
}
}
})
}
}
func TestCrossSkillAnalysis_RiskScore(t *testing.T) {
results := []*Result{
makeResult("reader", TierProfile{}, credFinding()),
makeResult("sender", tierWith(TierNetwork)),
}
xr := CrossSkillAnalysis(results, nil)
if xr == nil {
t.Fatal("expected non-nil result")
}
if xr.RiskScore <= 0 {
t.Errorf("RiskScore = %d, want > 0", xr.RiskScore)
}
if xr.RiskLabel == "clean" {
t.Error("RiskLabel = clean, want non-clean")
}
}
func TestCrossSkillAnalysis_HonorsDisabledIDs(t *testing.T) {
results := []*Result{
makeResult("reader", TierProfile{}, credFinding()),
makeResult("sender", tierWith(TierNetwork)),
}
// Baseline: cross-skill-exfiltration fires.
if xr := CrossSkillAnalysis(results, nil); xr == nil ||
!hasFinding(xr.Findings, "cross-skill-exfiltration", SeverityHigh) {
t.Fatal("expected cross-skill-exfiltration when not disabled")
}
// Disabling the rule suppresses the only finding → nil result.
if xr := CrossSkillAnalysis(results, map[string]bool{"cross-skill-exfiltration": true}); xr != nil {
t.Errorf("cross-skill-exfiltration should be suppressed when disabled, got %v", xr.Findings)
}
}
func TestFormatNameList(t *testing.T) {
tests := []struct {
names []string
want string
}{
{[]string{"a"}, "a"},
{[]string{"c", "a", "b"}, "a, b, c"},
{[]string{"a", "b", "c", "d", "e"}, "a, b, c, d, e"},
{[]string{"a", "b", "c", "d", "e", "f"}, "a, b, c, d, e and 1 more"},
{[]string{"a", "b", "c", "d", "e", "f", "g", "h"}, "a, b, c, d, e and 3 more"},
}
for _, tt := range tests {
t.Run(tt.want, func(t *testing.T) {
got := formatNameList(tt.names)
if got != tt.want {
t.Errorf("formatNameList(%v) = %q, want %q", tt.names, got, tt.want)
}
})
}
}
func TestIsSingleSelfPair(t *testing.T) {
tests := []struct {
a, b []string
want bool
}{
{[]string{"x"}, []string{"x"}, true},
{[]string{"x"}, []string{"y"}, false},
{[]string{"x", "y"}, []string{"x"}, false},
{[]string{"x"}, []string{"x", "y"}, false},
}
for _, tt := range tests {
if got := isSingleSelfPair(tt.a, tt.b); got != tt.want {
t.Errorf("isSingleSelfPair(%v, %v) = %v, want %v", tt.a, tt.b, got, tt.want)
}
}
}
// BenchmarkCrossSkillAnalysis_100k verifies the set-based approach handles
// 100k skills in well under 100ms.
func BenchmarkCrossSkillAnalysis_100k(b *testing.B) {
const n = 100_000
results := make([]*Result, n)
for i := range results {
name := fmt.Sprintf("skill-%d", i)
switch i % 4 {
case 0:
results[i] = makeResult(name, TierProfile{}, credFinding())
case 1:
results[i] = makeResult(name, tierWith(TierNetwork))
case 2:
results[i] = makeResult(name, tierWith(TierPrivilege, TierStealth), highFinding())
default:
results[i] = makeResult(name, TierProfile{}, lowFinding())
}
}
b.ResetTimer()
for range b.N {
result := CrossSkillAnalysis(results, nil)
if result == nil {
b.Fatal("expected non-nil result")
}
}
}