diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml new file mode 100644 index 000000000..2a62b683b --- /dev/null +++ b/.pre-commit-hooks.yaml @@ -0,0 +1,8 @@ +- id: skillshare-audit + name: Skillshare Audit + description: Scan skills for security threats before committing + entry: skillshare audit -p --format text + language: system + files: '(\.skillshare/|skills/)' + pass_filenames: false + stages: [pre-commit] diff --git a/cmd/skillshare/audit.go b/cmd/skillshare/audit.go index 045da1935..e262aa9e3 100644 --- a/cmd/skillshare/audit.go +++ b/cmd/skillshare/audit.go @@ -64,8 +64,9 @@ type auditRunSummary struct { Mode string `json:"mode,omitempty"` Threshold string `json:"threshold,omitempty"` MaxSeverity string `json:"maxSeverity,omitempty"` - RiskScore int `json:"riskScore"` - RiskLabel string `json:"riskLabel,omitempty"` + RiskScore int `json:"riskScore"` + RiskLabel string `json:"riskLabel,omitempty"` + AvgAnalyzability float64 `json:"avgAnalyzability"` } type auditJSONOutput struct { @@ -755,6 +756,7 @@ func summarizeAuditResults(total int, results []*audit.Result, threshold string) maxRisk := 0 maxSeverity := "" + sumAnalyzability := 0.0 for _, r := range results { c, h, m, l, i := r.CountBySeverityAll() summary.Critical += c @@ -788,10 +790,14 @@ func summarizeAuditResults(total int, results []*audit.Result, threshold string) maxSeverity = rs } } + sumAnalyzability += r.Analyzability } summary.RiskScore = maxRisk summary.RiskLabel = audit.RiskLabelFromScoreAndMaxSeverity(maxRisk, maxSeverity) summary.MaxSeverity = maxSeverity + if len(results) > 0 { + summary.AvgAnalyzability = sumAnalyzability / float64(len(results)) + } return summary } @@ -923,9 +929,11 @@ func printSkillResult(result *audit.Result, elapsed time.Duration) { } if ui.IsTTY() { fmt.Printf("%s→%s Aggregate risk: %s%s (%d/100)%s\n", ui.Cyan, ui.Reset, color, strings.ToUpper(result.RiskLabel), result.RiskScore, ui.Reset) + fmt.Printf("%s→%s Auditable: %.0f%%\n", ui.Cyan, ui.Reset, result.Analyzability*100) fmt.Printf("%s→%s Block decision: %s (max severity %s %s threshold %s)\n", ui.Cyan, ui.Reset, decision, maxSeverity, compare, threshold) } else { fmt.Printf("→ Aggregate risk: %s (%d/100)\n", strings.ToUpper(result.RiskLabel), result.RiskScore) + fmt.Printf("→ Auditable: %.0f%%\n", result.Analyzability*100) fmt.Printf("→ Block decision: %s (max severity %s %s threshold %s)\n", decision, maxSeverity, compare, threshold) } } @@ -960,6 +968,7 @@ func buildAuditSummaryLines(summary auditRunSummary) []string { riskLabel := strings.ToUpper(summary.RiskLabel) riskText := fmt.Sprintf("%s (%d/100)", riskLabel, summary.RiskScore) lines = append(lines, fmt.Sprintf(" Aggregate: %s", ui.Colorize(riskColor(summary.RiskLabel), riskText))) + lines = append(lines, fmt.Sprintf(" Auditable: %.0f%% avg", summary.AvgAnalyzability*100)) lines = append(lines, " Note: Failed uses severity gate; aggregate is informational") if summary.ScanErrors > 0 { lines = append(lines, fmt.Sprintf(" Scan errs: %d", summary.ScanErrors)) diff --git a/internal/audit/analyzability_test.go b/internal/audit/analyzability_test.go new file mode 100644 index 000000000..360398aff --- /dev/null +++ b/internal/audit/analyzability_test.go @@ -0,0 +1,110 @@ +package audit + +import ( + "os" + "path/filepath" + "testing" +) + +func TestAnalyzability_PureMarkdown(t *testing.T) { + dir := t.TempDir() + skillDir := filepath.Join(dir, "clean-skill") + os.MkdirAll(skillDir, 0755) + os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("# My Skill\nDoes stuff"), 0644) + os.WriteFile(filepath.Join(skillDir, "utils.sh"), []byte("echo hello"), 0644) + + result, err := ScanSkill(skillDir) + if err != nil { + t.Fatal(err) + } + if result.Analyzability < 0.99 { + t.Errorf("pure text skill should be ~1.0 analyzable, got %.2f", result.Analyzability) + } + if result.TotalBytes == 0 { + t.Error("TotalBytes should be > 0") + } + if result.AuditableBytes == 0 { + t.Error("AuditableBytes should be > 0") + } +} + +func TestAnalyzability_WithBinary(t *testing.T) { + dir := t.TempDir() + skillDir := filepath.Join(dir, "mixed-skill") + os.MkdirAll(skillDir, 0755) + os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("# Skill"), 0644) + + // Binary file: 1000 bytes with null bytes + bin := make([]byte, 1000) + bin[0] = 0 // null byte → binary + os.WriteFile(filepath.Join(skillDir, "data.bin"), bin, 0644) + + result, err := ScanSkill(skillDir) + if err != nil { + t.Fatal(err) + } + // SKILL.md = 7 bytes auditable, data.bin = 1000 bytes non-auditable + // Analyzability = 7 / 1007 ≈ 0.007 + if result.Analyzability > 0.05 { + t.Errorf("skill with large binary should have low analyzability, got %.2f", result.Analyzability) + } +} + +func TestAnalyzability_EmptySkill(t *testing.T) { + dir := t.TempDir() + skillDir := filepath.Join(dir, "empty-skill") + os.MkdirAll(skillDir, 0755) + + result, err := ScanSkill(skillDir) + if err != nil { + t.Fatal(err) + } + if result.Analyzability != 1.0 { + t.Errorf("empty skill should be 1.0 analyzable, got %.2f", result.Analyzability) + } +} + +func TestAnalyzability_LowScoreEmitsWarning(t *testing.T) { + dir := t.TempDir() + skillDir := filepath.Join(dir, "opaque-skill") + os.MkdirAll(skillDir, 0755) + os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("# X"), 0644) + + // Create large non-scannable file to push analyzability below 70% + os.WriteFile(filepath.Join(skillDir, "logo.png"), make([]byte, 100), 0644) + + result, err := ScanSkill(skillDir) + if err != nil { + t.Fatal(err) + } + found := false + for _, f := range result.Findings { + if f.Pattern == "low-analyzability" { + found = true + if f.Severity != SeverityInfo { + t.Errorf("expected INFO severity, got %s", f.Severity) + } + break + } + } + if !found { + t.Errorf("expected low-analyzability finding when score=%.2f", result.Analyzability) + } +} + +func TestAnalyzability_HighScoreNoWarning(t *testing.T) { + dir := t.TempDir() + skillDir := filepath.Join(dir, "transparent-skill") + os.MkdirAll(skillDir, 0755) + os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("# My Skill\nLong content here"), 0644) + + result, err := ScanSkill(skillDir) + if err != nil { + t.Fatal(err) + } + for _, f := range result.Findings { + if f.Pattern == "low-analyzability" { + t.Error("should not emit low-analyzability for high score") + } + } +} diff --git a/internal/audit/audit.go b/internal/audit/audit.go index b24d3e6d5..3d293bb3b 100644 --- a/internal/audit/audit.go +++ b/internal/audit/audit.go @@ -19,8 +19,9 @@ import ( var ErrBlocked = errors.New("blocked by security audit") const ( - maxScanFileSize = 1_000_000 // 1MB - maxScanDepth = 6 + maxScanFileSize = 1_000_000 // 1MB + maxScanDepth = 6 + analyzabilityThreshold = 0.70 ) // mdFileInfo holds data collected during the walk for structural checks. @@ -50,13 +51,16 @@ type Finding struct { // Result holds all findings for a single skill. type Result struct { - SkillName string `json:"skillName"` - Findings []Finding `json:"findings"` - RiskScore int `json:"riskScore"` - RiskLabel string `json:"riskLabel"` // "clean", "low", "medium", "high", "critical" - Threshold string `json:"threshold,omitempty"` - IsBlocked bool `json:"isBlocked,omitempty"` - ScanTarget string `json:"scanTarget,omitempty"` + SkillName string `json:"skillName"` + Findings []Finding `json:"findings"` + RiskScore int `json:"riskScore"` + RiskLabel string `json:"riskLabel"` // "clean", "low", "medium", "high", "critical" + Threshold string `json:"threshold,omitempty"` + IsBlocked bool `json:"isBlocked,omitempty"` + ScanTarget string `json:"scanTarget,omitempty"` + TotalBytes int64 `json:"totalBytes"` + AuditableBytes int64 `json:"auditableBytes"` + Analyzability float64 `json:"analyzability"` // AuditableBytes / TotalBytes (1.0 when TotalBytes == 0) } func (r *Result) updateRisk() { @@ -268,6 +272,8 @@ func scanSkillImpl(skillPath string, activeRules []rule, disabled map[string]boo // checkContentIntegrity can reuse them instead of re-reading from disk. fileCache := make(map[string][]byte) + var totalBytes, auditableBytes int64 + err = filepath.Walk(skillPath, func(path string, fi os.FileInfo, walkErr error) error { if walkErr != nil { return nil @@ -292,10 +298,15 @@ func scanSkillImpl(skillPath string, activeRules []rule, disabled map[string]boo if depth > maxScanDepth { return nil } + // Files exceeding maxScanFileSize are excluded from totalBytes so that + // analyzability reflects the ratio among files the scanner considers, + // not raw on-disk size. Oversized files are a separate concern. if fi.Size() > maxScanFileSize { return nil } + totalBytes += fi.Size() + if !isScannable(fi.Name()) { return nil } @@ -308,6 +319,8 @@ func scanSkillImpl(skillPath string, activeRules []rule, disabled map[string]boo return nil } + auditableBytes += int64(len(data)) + // Cache content for content-integrity check reuse. fileCache[filepath.ToSlash(relPath)] = data @@ -352,6 +365,24 @@ func scanSkillImpl(skillPath string, activeRules []rule, disabled map[string]boo result.Findings = append(result.Findings, checkContentIntegrity(skillPath, fileCache)...) } + result.TotalBytes = totalBytes + result.AuditableBytes = auditableBytes + if totalBytes > 0 { + result.Analyzability = float64(auditableBytes) / float64(totalBytes) + } else { + result.Analyzability = 1.0 + } + + if totalBytes > 0 && result.Analyzability < analyzabilityThreshold { + result.Findings = append(result.Findings, Finding{ + Severity: SeverityInfo, + Pattern: "low-analyzability", + Message: fmt.Sprintf("only %.0f%% of skill content is auditable (%.0f%% is binary/non-scannable)", result.Analyzability*100, (1-result.Analyzability)*100), + File: ".", + Line: 0, + }) + } + result.updateRisk() return result, nil } @@ -367,13 +398,21 @@ func ScanFileWithRules(filePath string, activeRules []rule) (*Result, error) { return nil, fmt.Errorf("not a file: %s", filePath) } + fileSize := info.Size() result := &Result{ SkillName: filepath.Base(filePath), ScanTarget: filePath, + TotalBytes: fileSize, } // Keep parity with directory scan boundaries. - if info.Size() > maxScanFileSize || !isScannable(info.Name()) { + if fileSize > maxScanFileSize || !isScannable(info.Name()) { + // Non-scannable or oversized: nothing is auditable. + if fileSize > 0 { + result.Analyzability = 0.0 + } else { + result.Analyzability = 1.0 + } result.updateRisk() return result, nil } @@ -383,10 +422,15 @@ func ScanFileWithRules(filePath string, activeRules []rule) (*Result, error) { return nil, fmt.Errorf("read file: %w", err) } if isBinaryContent(data) { + // Binary content: counted in TotalBytes but not auditable. + result.Analyzability = 0.0 result.updateRisk() return result, nil } + result.AuditableBytes = int64(len(data)) + result.Analyzability = 1.0 + resolvedRules := activeRules if resolvedRules == nil { rules, err := Rules() diff --git a/internal/audit/audit_scan_skill_test.go b/internal/audit/audit_scan_skill_test.go index d0ee9215b..0906568d7 100644 --- a/internal/audit/audit_scan_skill_test.go +++ b/internal/audit/audit_scan_skill_test.go @@ -9,6 +9,17 @@ import ( "testing" ) +// filterOutPattern returns findings excluding those with the given pattern. +func filterOutPattern(findings []Finding, pattern string) []Finding { + var out []Finding + for _, f := range findings { + if f.Pattern != pattern { + out = append(out, f) + } + } + return out +} + func TestScanSkill_CleanDirectory(t *testing.T) { dir := t.TempDir() os.MkdirAll(filepath.Join(dir, "my-skill"), 0755) @@ -70,8 +81,9 @@ func TestScanSkill_SkipsMetaJSON(t *testing.T) { if err != nil { t.Fatal(err) } - if len(result.Findings) != 0 { - t.Errorf("expected 0 findings (.skillshare-meta.json should be skipped), got %d: %+v", len(result.Findings), result.Findings) + nonAnalyzability := filterOutPattern(result.Findings, "low-analyzability") + if len(nonAnalyzability) != 0 { + t.Errorf("expected 0 findings (.skillshare-meta.json should be skipped), got %d: %+v", len(nonAnalyzability), nonAnalyzability) } } @@ -86,8 +98,9 @@ func TestScanSkill_SkipsBinaryFiles(t *testing.T) { if err != nil { t.Fatal(err) } - if len(result.Findings) != 0 { - t.Errorf("expected 0 findings (.png should be skipped), got %d", len(result.Findings)) + nonAnalyzability := filterOutPattern(result.Findings, "low-analyzability") + if len(nonAnalyzability) != 0 { + t.Errorf("expected 0 findings (.png should be skipped), got %d", len(nonAnalyzability)) } } diff --git a/internal/server/handler_audit.go b/internal/server/handler_audit.go index 2b76447da..49165db78 100644 --- a/internal/server/handler_audit.go +++ b/internal/server/handler_audit.go @@ -22,29 +22,33 @@ type auditFindingResponse struct { } type auditResultResponse struct { - SkillName string `json:"skillName"` - Findings []auditFindingResponse `json:"findings"` - RiskScore int `json:"riskScore"` - RiskLabel string `json:"riskLabel"` - Threshold string `json:"threshold"` - IsBlocked bool `json:"isBlocked"` - ScanTarget string `json:"scanTarget,omitempty"` + SkillName string `json:"skillName"` + Findings []auditFindingResponse `json:"findings"` + RiskScore int `json:"riskScore"` + RiskLabel string `json:"riskLabel"` + Threshold string `json:"threshold"` + IsBlocked bool `json:"isBlocked"` + ScanTarget string `json:"scanTarget,omitempty"` + TotalBytes int64 `json:"totalBytes"` + AuditableBytes int64 `json:"auditableBytes"` + Analyzability float64 `json:"analyzability"` } type auditSummary struct { - Total int `json:"total"` - Passed int `json:"passed"` - Warning int `json:"warning"` - Failed int `json:"failed"` - Critical int `json:"critical"` - High int `json:"high"` - Medium int `json:"medium"` - Low int `json:"low"` - Info int `json:"info"` - Threshold string `json:"threshold"` - RiskScore int `json:"riskScore"` - RiskLabel string `json:"riskLabel"` - ScanErrors int `json:"scanErrors,omitempty"` + Total int `json:"total"` + Passed int `json:"passed"` + Warning int `json:"warning"` + Failed int `json:"failed"` + Critical int `json:"critical"` + High int `json:"high"` + Medium int `json:"medium"` + Low int `json:"low"` + Info int `json:"info"` + Threshold string `json:"threshold"` + RiskScore int `json:"riskScore"` + RiskLabel string `json:"riskLabel"` + ScanErrors int `json:"scanErrors,omitempty"` + AvgAnalyzability float64 `json:"avgAnalyzability"` } type skillEntry struct { @@ -105,6 +109,7 @@ func (s *Server) handleAuditAll(w http.ResponseWriter, r *http.Request) { scanErrors := 0 maxRisk := 0 maxSeverity := "" + sumAnalyzability := 0.0 // Phase 1: parallel scan with bounded workers. projectRoot := s.projectRoot @@ -157,6 +162,7 @@ func (s *Server) handleAuditAll(w http.ResponseWriter, r *http.Request) { maxSeverity = ms } } + sumAnalyzability += result.Analyzability } status := "ok" @@ -173,6 +179,9 @@ func (s *Server) handleAuditAll(w http.ResponseWriter, r *http.Request) { summary.ScanErrors = scanErrors summary.RiskScore = maxRisk summary.RiskLabel = audit.RiskLabelFromScoreAndMaxSeverity(maxRisk, maxSeverity) + if len(results) > 0 { + summary.AvgAnalyzability = sumAnalyzability / float64(len(results)) + } args := map[string]any{ "scope": "all", @@ -302,18 +311,19 @@ func (s *Server) handleAuditSkill(w http.ResponseWriter, r *http.Request) { writeJSON(w, map[string]any{ "result": toAuditResponse(result), "summary": auditSummary{ - Total: 1, - Passed: boolToInt(len(result.Findings) == 0), - Warning: warningCount, - Failed: failedCount, - Critical: c, - High: h, - Medium: m, - Low: l, - Info: i, - Threshold: threshold, - RiskScore: result.RiskScore, - RiskLabel: result.RiskLabel, + Total: 1, + Passed: boolToInt(len(result.Findings) == 0), + Warning: warningCount, + Failed: failedCount, + Critical: c, + High: h, + Medium: m, + Low: l, + Info: i, + Threshold: threshold, + RiskScore: result.RiskScore, + RiskLabel: result.RiskLabel, + AvgAnalyzability: result.Analyzability, }, }) } @@ -444,12 +454,15 @@ func toAuditResponse(result *audit.Result) auditResultResponse { }) } return auditResultResponse{ - SkillName: result.SkillName, - Findings: findings, - RiskScore: result.RiskScore, - RiskLabel: result.RiskLabel, - Threshold: result.Threshold, - IsBlocked: result.IsBlocked, - ScanTarget: result.ScanTarget, + SkillName: result.SkillName, + Findings: findings, + RiskScore: result.RiskScore, + RiskLabel: result.RiskLabel, + Threshold: result.Threshold, + IsBlocked: result.IsBlocked, + ScanTarget: result.ScanTarget, + TotalBytes: result.TotalBytes, + AuditableBytes: result.AuditableBytes, + Analyzability: result.Analyzability, } } diff --git a/website/docs/how-to/recipes/pre-commit-hook.md b/website/docs/how-to/recipes/pre-commit-hook.md new file mode 100644 index 000000000..62e9914f2 --- /dev/null +++ b/website/docs/how-to/recipes/pre-commit-hook.md @@ -0,0 +1,59 @@ +--- +sidebar_position: 3 +--- + +# Recipe: Pre-commit Hook + +> Run `skillshare audit` automatically on every commit using the [pre-commit](https://pre-commit.com/) framework. + +## Scenario + +You want to catch security issues in skills **before** they're committed, rather than relying solely on CI to catch problems after the fact. + +## Setup + +Add to your project's `.pre-commit-config.yaml`: + +```yaml +repos: + - repo: https://github.com/runkids/skillshare + rev: v0.16.8 # use latest release tag + hooks: + - id: skillshare-audit +``` + +Then install the hook: + +```bash +pre-commit install +``` + +## How It Works + +The hook runs `skillshare audit -p` whenever you commit changes to files matching `.skillshare/` or `skills/` directories. If any findings exceed the configured threshold, the commit is blocked. + +## Configuration + +The hook respects your project's `.skillshare/config.yaml` settings: + +```yaml +audit: + block_threshold: high # block on HIGH+ findings +``` + +## Skipping the Hook + +For a one-time skip: + +```bash +SKIP=skillshare-audit git commit -m "your message" +``` + +## Requirements + +- `skillshare` CLI must be installed and available in `PATH` +- Project must be initialized with `skillshare init -p` + +## Combining with CI + +The pre-commit hook catches issues locally, while [CI/CD validation](ci-cd-skill-validation.md) provides a safety net for the whole team. Use both for defense in depth. diff --git a/website/docs/reference/commands/audit.md b/website/docs/reference/commands/audit.md index 8ed0caf7f..95112ad2b 100644 --- a/website/docs/reference/commands/audit.md +++ b/website/docs/reference/commands/audit.md @@ -534,6 +534,21 @@ This creates: - **Security tab** entries visible at Repository → Security → Code scanning alerts - **Historical tracking** of findings across commits +### Pre-commit Hook + +Run `skillshare audit` automatically on every commit using the [pre-commit](https://pre-commit.com/) framework. The hook scans files matching `.skillshare/` or `skills/` directories and blocks the commit if findings exceed your configured threshold. + +```yaml +# .pre-commit-config.yaml +repos: + - repo: https://github.com/runkids/skillshare + rev: v0.16.8 # use latest release tag + hooks: + - id: skillshare-audit +``` + +See the [Pre-commit Hook recipe](/docs/how-to/recipes/pre-commit-hook) for full setup instructions. + ## Best Practices ### For Individual Developers @@ -552,9 +567,10 @@ This creates: 1. **Install**: Skills are automatically scanned — blocked if threshold exceeded 2. **Periodic scan**: Run `skillshare audit` regularly to catch rules updated after install -3. **CI gate**: Add audit to your CI pipeline for shared skill repositories -4. **Custom rules**: Tailor detection to your organization's threat model -5. **Review reports**: Use `--format json` for compliance or `--format sarif` for GitHub Code Scanning +3. **Pre-commit hook**: Catch issues before they're committed with the [pre-commit framework](/docs/how-to/recipes/pre-commit-hook) +4. **CI gate**: Add audit to your CI pipeline for shared skill repositories +5. **Custom rules**: Tailor detection to your organization's threat model +6. **Review reports**: Use `--format json` for compliance or `--format sarif` for GitHub Code Scanning ### Threshold Configuration @@ -903,3 +919,4 @@ Source of truth for regex-based rules: - [doctor](/docs/reference/commands/doctor) — Diagnose setup issues - [list](/docs/reference/commands/list) — List installed skills - [Securing Your Skills](/docs/how-to/advanced/security) — Security guide for teams and organizations +- [Pre-commit Hook](/docs/how-to/recipes/pre-commit-hook) — Automatic audit on every commit