feat(audit): add analyzability score and pre-commit hook

Analyzability score measures what percentage of a skill's content is
auditable plaintext vs binary/non-scannable. Surfaces in CLI text
output, JSON, SARIF, and Web UI API responses.

- Add TotalBytes, AuditableBytes, Analyzability fields to audit.Result
- Emit INFO-level low-analyzability finding when score < 70%
- Show "Auditable: X%" in per-skill and summary text output
- Expose analyzability in server API (auditResultResponse + auditSummary)
- Add .pre-commit-hooks.yaml for pre-commit framework integration
- Add pre-commit hook recipe docs and audit.md cross-references
This commit is contained in:
Willie
2026-03-02 23:01:30 +08:00
parent afb8951359
commit 8e83495200
8 changed files with 331 additions and 58 deletions
+8
View File
@@ -0,0 +1,8 @@
- id: skillshare-audit
name: Skillshare Audit
description: Scan skills for security threats before committing
entry: skillshare audit -p --format text
language: system
files: '(\.skillshare/|skills/)'
pass_filenames: false
stages: [pre-commit]
+11 -2
View File
@@ -64,8 +64,9 @@ type auditRunSummary struct {
Mode string `json:"mode,omitempty"`
Threshold string `json:"threshold,omitempty"`
MaxSeverity string `json:"maxSeverity,omitempty"`
RiskScore int `json:"riskScore"`
RiskLabel string `json:"riskLabel,omitempty"`
RiskScore int `json:"riskScore"`
RiskLabel string `json:"riskLabel,omitempty"`
AvgAnalyzability float64 `json:"avgAnalyzability"`
}
type auditJSONOutput struct {
@@ -755,6 +756,7 @@ func summarizeAuditResults(total int, results []*audit.Result, threshold string)
maxRisk := 0
maxSeverity := ""
sumAnalyzability := 0.0
for _, r := range results {
c, h, m, l, i := r.CountBySeverityAll()
summary.Critical += c
@@ -788,10 +790,14 @@ func summarizeAuditResults(total int, results []*audit.Result, threshold string)
maxSeverity = rs
}
}
sumAnalyzability += r.Analyzability
}
summary.RiskScore = maxRisk
summary.RiskLabel = audit.RiskLabelFromScoreAndMaxSeverity(maxRisk, maxSeverity)
summary.MaxSeverity = maxSeverity
if len(results) > 0 {
summary.AvgAnalyzability = sumAnalyzability / float64(len(results))
}
return summary
}
@@ -923,9 +929,11 @@ func printSkillResult(result *audit.Result, elapsed time.Duration) {
}
if ui.IsTTY() {
fmt.Printf("%s→%s Aggregate risk: %s%s (%d/100)%s\n", ui.Cyan, ui.Reset, color, strings.ToUpper(result.RiskLabel), result.RiskScore, ui.Reset)
fmt.Printf("%s→%s Auditable: %.0f%%\n", ui.Cyan, ui.Reset, result.Analyzability*100)
fmt.Printf("%s→%s Block decision: %s (max severity %s %s threshold %s)\n", ui.Cyan, ui.Reset, decision, maxSeverity, compare, threshold)
} else {
fmt.Printf("→ Aggregate risk: %s (%d/100)\n", strings.ToUpper(result.RiskLabel), result.RiskScore)
fmt.Printf("→ Auditable: %.0f%%\n", result.Analyzability*100)
fmt.Printf("→ Block decision: %s (max severity %s %s threshold %s)\n", decision, maxSeverity, compare, threshold)
}
}
@@ -960,6 +968,7 @@ func buildAuditSummaryLines(summary auditRunSummary) []string {
riskLabel := strings.ToUpper(summary.RiskLabel)
riskText := fmt.Sprintf("%s (%d/100)", riskLabel, summary.RiskScore)
lines = append(lines, fmt.Sprintf(" Aggregate: %s", ui.Colorize(riskColor(summary.RiskLabel), riskText)))
lines = append(lines, fmt.Sprintf(" Auditable: %.0f%% avg", summary.AvgAnalyzability*100))
lines = append(lines, " Note: Failed uses severity gate; aggregate is informational")
if summary.ScanErrors > 0 {
lines = append(lines, fmt.Sprintf(" Scan errs: %d", summary.ScanErrors))
+110
View File
@@ -0,0 +1,110 @@
package audit
import (
"os"
"path/filepath"
"testing"
)
func TestAnalyzability_PureMarkdown(t *testing.T) {
dir := t.TempDir()
skillDir := filepath.Join(dir, "clean-skill")
os.MkdirAll(skillDir, 0755)
os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("# My Skill\nDoes stuff"), 0644)
os.WriteFile(filepath.Join(skillDir, "utils.sh"), []byte("echo hello"), 0644)
result, err := ScanSkill(skillDir)
if err != nil {
t.Fatal(err)
}
if result.Analyzability < 0.99 {
t.Errorf("pure text skill should be ~1.0 analyzable, got %.2f", result.Analyzability)
}
if result.TotalBytes == 0 {
t.Error("TotalBytes should be > 0")
}
if result.AuditableBytes == 0 {
t.Error("AuditableBytes should be > 0")
}
}
func TestAnalyzability_WithBinary(t *testing.T) {
dir := t.TempDir()
skillDir := filepath.Join(dir, "mixed-skill")
os.MkdirAll(skillDir, 0755)
os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("# Skill"), 0644)
// Binary file: 1000 bytes with null bytes
bin := make([]byte, 1000)
bin[0] = 0 // null byte → binary
os.WriteFile(filepath.Join(skillDir, "data.bin"), bin, 0644)
result, err := ScanSkill(skillDir)
if err != nil {
t.Fatal(err)
}
// SKILL.md = 7 bytes auditable, data.bin = 1000 bytes non-auditable
// Analyzability = 7 / 1007 ≈ 0.007
if result.Analyzability > 0.05 {
t.Errorf("skill with large binary should have low analyzability, got %.2f", result.Analyzability)
}
}
func TestAnalyzability_EmptySkill(t *testing.T) {
dir := t.TempDir()
skillDir := filepath.Join(dir, "empty-skill")
os.MkdirAll(skillDir, 0755)
result, err := ScanSkill(skillDir)
if err != nil {
t.Fatal(err)
}
if result.Analyzability != 1.0 {
t.Errorf("empty skill should be 1.0 analyzable, got %.2f", result.Analyzability)
}
}
func TestAnalyzability_LowScoreEmitsWarning(t *testing.T) {
dir := t.TempDir()
skillDir := filepath.Join(dir, "opaque-skill")
os.MkdirAll(skillDir, 0755)
os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("# X"), 0644)
// Create large non-scannable file to push analyzability below 70%
os.WriteFile(filepath.Join(skillDir, "logo.png"), make([]byte, 100), 0644)
result, err := ScanSkill(skillDir)
if err != nil {
t.Fatal(err)
}
found := false
for _, f := range result.Findings {
if f.Pattern == "low-analyzability" {
found = true
if f.Severity != SeverityInfo {
t.Errorf("expected INFO severity, got %s", f.Severity)
}
break
}
}
if !found {
t.Errorf("expected low-analyzability finding when score=%.2f", result.Analyzability)
}
}
func TestAnalyzability_HighScoreNoWarning(t *testing.T) {
dir := t.TempDir()
skillDir := filepath.Join(dir, "transparent-skill")
os.MkdirAll(skillDir, 0755)
os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("# My Skill\nLong content here"), 0644)
result, err := ScanSkill(skillDir)
if err != nil {
t.Fatal(err)
}
for _, f := range result.Findings {
if f.Pattern == "low-analyzability" {
t.Error("should not emit low-analyzability for high score")
}
}
}
+54 -10
View File
@@ -19,8 +19,9 @@ import (
var ErrBlocked = errors.New("blocked by security audit")
const (
maxScanFileSize = 1_000_000 // 1MB
maxScanDepth = 6
maxScanFileSize = 1_000_000 // 1MB
maxScanDepth = 6
analyzabilityThreshold = 0.70
)
// mdFileInfo holds data collected during the walk for structural checks.
@@ -50,13 +51,16 @@ type Finding struct {
// Result holds all findings for a single skill.
type Result struct {
SkillName string `json:"skillName"`
Findings []Finding `json:"findings"`
RiskScore int `json:"riskScore"`
RiskLabel string `json:"riskLabel"` // "clean", "low", "medium", "high", "critical"
Threshold string `json:"threshold,omitempty"`
IsBlocked bool `json:"isBlocked,omitempty"`
ScanTarget string `json:"scanTarget,omitempty"`
SkillName string `json:"skillName"`
Findings []Finding `json:"findings"`
RiskScore int `json:"riskScore"`
RiskLabel string `json:"riskLabel"` // "clean", "low", "medium", "high", "critical"
Threshold string `json:"threshold,omitempty"`
IsBlocked bool `json:"isBlocked,omitempty"`
ScanTarget string `json:"scanTarget,omitempty"`
TotalBytes int64 `json:"totalBytes"`
AuditableBytes int64 `json:"auditableBytes"`
Analyzability float64 `json:"analyzability"` // AuditableBytes / TotalBytes (1.0 when TotalBytes == 0)
}
func (r *Result) updateRisk() {
@@ -268,6 +272,8 @@ func scanSkillImpl(skillPath string, activeRules []rule, disabled map[string]boo
// checkContentIntegrity can reuse them instead of re-reading from disk.
fileCache := make(map[string][]byte)
var totalBytes, auditableBytes int64
err = filepath.Walk(skillPath, func(path string, fi os.FileInfo, walkErr error) error {
if walkErr != nil {
return nil
@@ -292,10 +298,15 @@ func scanSkillImpl(skillPath string, activeRules []rule, disabled map[string]boo
if depth > maxScanDepth {
return nil
}
// Files exceeding maxScanFileSize are excluded from totalBytes so that
// analyzability reflects the ratio among files the scanner considers,
// not raw on-disk size. Oversized files are a separate concern.
if fi.Size() > maxScanFileSize {
return nil
}
totalBytes += fi.Size()
if !isScannable(fi.Name()) {
return nil
}
@@ -308,6 +319,8 @@ func scanSkillImpl(skillPath string, activeRules []rule, disabled map[string]boo
return nil
}
auditableBytes += int64(len(data))
// Cache content for content-integrity check reuse.
fileCache[filepath.ToSlash(relPath)] = data
@@ -352,6 +365,24 @@ func scanSkillImpl(skillPath string, activeRules []rule, disabled map[string]boo
result.Findings = append(result.Findings, checkContentIntegrity(skillPath, fileCache)...)
}
result.TotalBytes = totalBytes
result.AuditableBytes = auditableBytes
if totalBytes > 0 {
result.Analyzability = float64(auditableBytes) / float64(totalBytes)
} else {
result.Analyzability = 1.0
}
if totalBytes > 0 && result.Analyzability < analyzabilityThreshold {
result.Findings = append(result.Findings, Finding{
Severity: SeverityInfo,
Pattern: "low-analyzability",
Message: fmt.Sprintf("only %.0f%% of skill content is auditable (%.0f%% is binary/non-scannable)", result.Analyzability*100, (1-result.Analyzability)*100),
File: ".",
Line: 0,
})
}
result.updateRisk()
return result, nil
}
@@ -367,13 +398,21 @@ func ScanFileWithRules(filePath string, activeRules []rule) (*Result, error) {
return nil, fmt.Errorf("not a file: %s", filePath)
}
fileSize := info.Size()
result := &Result{
SkillName: filepath.Base(filePath),
ScanTarget: filePath,
TotalBytes: fileSize,
}
// Keep parity with directory scan boundaries.
if info.Size() > maxScanFileSize || !isScannable(info.Name()) {
if fileSize > maxScanFileSize || !isScannable(info.Name()) {
// Non-scannable or oversized: nothing is auditable.
if fileSize > 0 {
result.Analyzability = 0.0
} else {
result.Analyzability = 1.0
}
result.updateRisk()
return result, nil
}
@@ -383,10 +422,15 @@ func ScanFileWithRules(filePath string, activeRules []rule) (*Result, error) {
return nil, fmt.Errorf("read file: %w", err)
}
if isBinaryContent(data) {
// Binary content: counted in TotalBytes but not auditable.
result.Analyzability = 0.0
result.updateRisk()
return result, nil
}
result.AuditableBytes = int64(len(data))
result.Analyzability = 1.0
resolvedRules := activeRules
if resolvedRules == nil {
rules, err := Rules()
+17 -4
View File
@@ -9,6 +9,17 @@ import (
"testing"
)
// filterOutPattern returns findings excluding those with the given pattern.
func filterOutPattern(findings []Finding, pattern string) []Finding {
var out []Finding
for _, f := range findings {
if f.Pattern != pattern {
out = append(out, f)
}
}
return out
}
func TestScanSkill_CleanDirectory(t *testing.T) {
dir := t.TempDir()
os.MkdirAll(filepath.Join(dir, "my-skill"), 0755)
@@ -70,8 +81,9 @@ func TestScanSkill_SkipsMetaJSON(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(result.Findings) != 0 {
t.Errorf("expected 0 findings (.skillshare-meta.json should be skipped), got %d: %+v", len(result.Findings), result.Findings)
nonAnalyzability := filterOutPattern(result.Findings, "low-analyzability")
if len(nonAnalyzability) != 0 {
t.Errorf("expected 0 findings (.skillshare-meta.json should be skipped), got %d: %+v", len(nonAnalyzability), nonAnalyzability)
}
}
@@ -86,8 +98,9 @@ func TestScanSkill_SkipsBinaryFiles(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(result.Findings) != 0 {
t.Errorf("expected 0 findings (.png should be skipped), got %d", len(result.Findings))
nonAnalyzability := filterOutPattern(result.Findings, "low-analyzability")
if len(nonAnalyzability) != 0 {
t.Errorf("expected 0 findings (.png should be skipped), got %d", len(nonAnalyzability))
}
}
+52 -39
View File
@@ -22,29 +22,33 @@ type auditFindingResponse struct {
}
type auditResultResponse struct {
SkillName string `json:"skillName"`
Findings []auditFindingResponse `json:"findings"`
RiskScore int `json:"riskScore"`
RiskLabel string `json:"riskLabel"`
Threshold string `json:"threshold"`
IsBlocked bool `json:"isBlocked"`
ScanTarget string `json:"scanTarget,omitempty"`
SkillName string `json:"skillName"`
Findings []auditFindingResponse `json:"findings"`
RiskScore int `json:"riskScore"`
RiskLabel string `json:"riskLabel"`
Threshold string `json:"threshold"`
IsBlocked bool `json:"isBlocked"`
ScanTarget string `json:"scanTarget,omitempty"`
TotalBytes int64 `json:"totalBytes"`
AuditableBytes int64 `json:"auditableBytes"`
Analyzability float64 `json:"analyzability"`
}
type auditSummary struct {
Total int `json:"total"`
Passed int `json:"passed"`
Warning int `json:"warning"`
Failed int `json:"failed"`
Critical int `json:"critical"`
High int `json:"high"`
Medium int `json:"medium"`
Low int `json:"low"`
Info int `json:"info"`
Threshold string `json:"threshold"`
RiskScore int `json:"riskScore"`
RiskLabel string `json:"riskLabel"`
ScanErrors int `json:"scanErrors,omitempty"`
Total int `json:"total"`
Passed int `json:"passed"`
Warning int `json:"warning"`
Failed int `json:"failed"`
Critical int `json:"critical"`
High int `json:"high"`
Medium int `json:"medium"`
Low int `json:"low"`
Info int `json:"info"`
Threshold string `json:"threshold"`
RiskScore int `json:"riskScore"`
RiskLabel string `json:"riskLabel"`
ScanErrors int `json:"scanErrors,omitempty"`
AvgAnalyzability float64 `json:"avgAnalyzability"`
}
type skillEntry struct {
@@ -105,6 +109,7 @@ func (s *Server) handleAuditAll(w http.ResponseWriter, r *http.Request) {
scanErrors := 0
maxRisk := 0
maxSeverity := ""
sumAnalyzability := 0.0
// Phase 1: parallel scan with bounded workers.
projectRoot := s.projectRoot
@@ -157,6 +162,7 @@ func (s *Server) handleAuditAll(w http.ResponseWriter, r *http.Request) {
maxSeverity = ms
}
}
sumAnalyzability += result.Analyzability
}
status := "ok"
@@ -173,6 +179,9 @@ func (s *Server) handleAuditAll(w http.ResponseWriter, r *http.Request) {
summary.ScanErrors = scanErrors
summary.RiskScore = maxRisk
summary.RiskLabel = audit.RiskLabelFromScoreAndMaxSeverity(maxRisk, maxSeverity)
if len(results) > 0 {
summary.AvgAnalyzability = sumAnalyzability / float64(len(results))
}
args := map[string]any{
"scope": "all",
@@ -302,18 +311,19 @@ func (s *Server) handleAuditSkill(w http.ResponseWriter, r *http.Request) {
writeJSON(w, map[string]any{
"result": toAuditResponse(result),
"summary": auditSummary{
Total: 1,
Passed: boolToInt(len(result.Findings) == 0),
Warning: warningCount,
Failed: failedCount,
Critical: c,
High: h,
Medium: m,
Low: l,
Info: i,
Threshold: threshold,
RiskScore: result.RiskScore,
RiskLabel: result.RiskLabel,
Total: 1,
Passed: boolToInt(len(result.Findings) == 0),
Warning: warningCount,
Failed: failedCount,
Critical: c,
High: h,
Medium: m,
Low: l,
Info: i,
Threshold: threshold,
RiskScore: result.RiskScore,
RiskLabel: result.RiskLabel,
AvgAnalyzability: result.Analyzability,
},
})
}
@@ -444,12 +454,15 @@ func toAuditResponse(result *audit.Result) auditResultResponse {
})
}
return auditResultResponse{
SkillName: result.SkillName,
Findings: findings,
RiskScore: result.RiskScore,
RiskLabel: result.RiskLabel,
Threshold: result.Threshold,
IsBlocked: result.IsBlocked,
ScanTarget: result.ScanTarget,
SkillName: result.SkillName,
Findings: findings,
RiskScore: result.RiskScore,
RiskLabel: result.RiskLabel,
Threshold: result.Threshold,
IsBlocked: result.IsBlocked,
ScanTarget: result.ScanTarget,
TotalBytes: result.TotalBytes,
AuditableBytes: result.AuditableBytes,
Analyzability: result.Analyzability,
}
}
@@ -0,0 +1,59 @@
---
sidebar_position: 3
---
# Recipe: Pre-commit Hook
> Run `skillshare audit` automatically on every commit using the [pre-commit](https://pre-commit.com/) framework.
## Scenario
You want to catch security issues in skills **before** they're committed, rather than relying solely on CI to catch problems after the fact.
## Setup
Add to your project's `.pre-commit-config.yaml`:
```yaml
repos:
- repo: https://github.com/runkids/skillshare
rev: v0.16.8 # use latest release tag
hooks:
- id: skillshare-audit
```
Then install the hook:
```bash
pre-commit install
```
## How It Works
The hook runs `skillshare audit -p` whenever you commit changes to files matching `.skillshare/` or `skills/` directories. If any findings exceed the configured threshold, the commit is blocked.
## Configuration
The hook respects your project's `.skillshare/config.yaml` settings:
```yaml
audit:
block_threshold: high # block on HIGH+ findings
```
## Skipping the Hook
For a one-time skip:
```bash
SKIP=skillshare-audit git commit -m "your message"
```
## Requirements
- `skillshare` CLI must be installed and available in `PATH`
- Project must be initialized with `skillshare init -p`
## Combining with CI
The pre-commit hook catches issues locally, while [CI/CD validation](ci-cd-skill-validation.md) provides a safety net for the whole team. Use both for defense in depth.
+20 -3
View File
@@ -534,6 +534,21 @@ This creates:
- **Security tab** entries visible at Repository → Security → Code scanning alerts
- **Historical tracking** of findings across commits
### Pre-commit Hook
Run `skillshare audit` automatically on every commit using the [pre-commit](https://pre-commit.com/) framework. The hook scans files matching `.skillshare/` or `skills/` directories and blocks the commit if findings exceed your configured threshold.
```yaml
# .pre-commit-config.yaml
repos:
- repo: https://github.com/runkids/skillshare
rev: v0.16.8 # use latest release tag
hooks:
- id: skillshare-audit
```
See the [Pre-commit Hook recipe](/docs/how-to/recipes/pre-commit-hook) for full setup instructions.
## Best Practices
### For Individual Developers
@@ -552,9 +567,10 @@ This creates:
1. **Install**: Skills are automatically scanned — blocked if threshold exceeded
2. **Periodic scan**: Run `skillshare audit` regularly to catch rules updated after install
3. **CI gate**: Add audit to your CI pipeline for shared skill repositories
4. **Custom rules**: Tailor detection to your organization's threat model
5. **Review reports**: Use `--format json` for compliance or `--format sarif` for GitHub Code Scanning
3. **Pre-commit hook**: Catch issues before they're committed with the [pre-commit framework](/docs/how-to/recipes/pre-commit-hook)
4. **CI gate**: Add audit to your CI pipeline for shared skill repositories
5. **Custom rules**: Tailor detection to your organization's threat model
6. **Review reports**: Use `--format json` for compliance or `--format sarif` for GitHub Code Scanning
### Threshold Configuration
@@ -903,3 +919,4 @@ Source of truth for regex-based rules:
- [doctor](/docs/reference/commands/doctor) — Diagnose setup issues
- [list](/docs/reference/commands/list) — List installed skills
- [Securing Your Skills](/docs/how-to/advanced/security) — Security guide for teams and organizations
- [Pre-commit Hook](/docs/how-to/recipes/pre-commit-hook) — Automatic audit on every commit