From ebf2aaf822a1289811bfbe22dbffcb09db56d78f Mon Sep 17 00:00:00 2001 From: Willie Date: Wed, 4 Mar 2026 18:07:13 +0800 Subject: [PATCH] feat(audit): add metadata analyzer for supply-chain trust verification Add metadataAnalyzer (skill scope) that cross-references SKILL.md metadata against the actual git source from .skillshare-meta.json: - publisher-mismatch (HIGH): description claims an org but repo owner differs (e.g. "from Acme Corp" but sourced from evil-fork) - authority-language (MEDIUM): uses "official"/"verified" language but source is not a well-known organization Both rules are disable-able via audit-rules.yaml. The analyzer reads meta from disk (zero API signature changes) and avoids import cycles by defining its own minimal metaJSON struct. --- internal/audit/analyzer_metadata.go | 291 +++++++++++++++++++ internal/audit/analyzer_metadata_test.go | 349 +++++++++++++++++++++++ internal/audit/analyzer_registry.go | 1 + internal/audit/analyzer_registry_test.go | 5 +- internal/audit/analyzers.go | 1 + internal/audit/audit.go | 2 + 6 files changed, 647 insertions(+), 2 deletions(-) create mode 100644 internal/audit/analyzer_metadata.go create mode 100644 internal/audit/analyzer_metadata_test.go diff --git a/internal/audit/analyzer_metadata.go b/internal/audit/analyzer_metadata.go new file mode 100644 index 000000000..83cbce8c6 --- /dev/null +++ b/internal/audit/analyzer_metadata.go @@ -0,0 +1,291 @@ +package audit + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" +) + +// metadataAnalyzer cross-references SKILL.md metadata (name, description) +// against the actual git source URL from .skillshare-meta.json. +// Detects social-engineering patterns: publisher mismatch and authority claims. +// Runs at skill scope after all files are walked. +type metadataAnalyzer struct{} + +func (a *metadataAnalyzer) ID() string { return AnalyzerMetadata } +func (a *metadataAnalyzer) Scope() AnalyzerScope { return ScopeSkill } + +// metaJSON is a minimal subset of install.SkillMeta to avoid import cycles. +type metaJSON struct { + RepoURL string `json:"repo_url"` +} + +// Rule IDs for disable support via audit-rules.yaml. +const ( + rulePublisherMismatch = "publisher-mismatch" + ruleAuthorityLanguage = "authority-language" +) + +// reOrgClaim matches patterns like "from Acme Corp", "by Acme", "@acme". +var reOrgClaim = regexp.MustCompile(`(?i)(?:from|by|made by|created by|published by|maintained by)\s+([A-Z][\w-]+(?:\s+(?:Corp|Inc|Ltd|Team|Labs|AI|HQ|Co|Group))?)|@([A-Za-z][\w-]+)`) + +// authorityWords are terms that imply official endorsement. +var authorityWords = []string{ + "official", + "verified", + "trusted", + "authorized", + "endorsed", + "certified", +} + +func (a *metadataAnalyzer) Analyze(ctx *AnalyzeContext) ([]Finding, error) { + if ctx.SkillPath == "" { + return nil, nil + } + + // Read .skillshare-meta.json for source URL. + repoURL := readMetaRepoURL(ctx.SkillPath) + + // Read SKILL.md frontmatter for name and description. + skillMDPath := filepath.Join(ctx.SkillPath, "SKILL.md") + name, description := readSkillFrontmatter(skillMDPath, ctx.FileCache) + + var findings []Finding + + // Rule A: Publisher mismatch — description claims an org but repo owner differs. + if !ctx.DisabledIDs[rulePublisherMismatch] && repoURL != "" { + if f := checkPublisherMismatch(name, description, repoURL); f != nil { + findings = append(findings, *f) + } + } + + // Rule B: Authority language from unrecognized source. + if !ctx.DisabledIDs[ruleAuthorityLanguage] { + if f := checkAuthorityLanguage(description, repoURL); f != nil { + findings = append(findings, *f) + } + } + + return findings, nil +} + +// readMetaRepoURL reads repo_url from .skillshare-meta.json in skillPath. +func readMetaRepoURL(skillPath string) string { + data, err := os.ReadFile(filepath.Join(skillPath, ".skillshare-meta.json")) + if err != nil { + return "" + } + var m metaJSON + if json.Unmarshal(data, &m) != nil { + return "" + } + return m.RepoURL +} + +// readSkillFrontmatter extracts name and description from SKILL.md. +// Uses fileCache if available, otherwise reads from disk. +func readSkillFrontmatter(skillMDPath string, fileCache map[string][]byte) (name, description string) { + var data []byte + if cached, ok := fileCache["SKILL.md"]; ok { + data = cached + } else { + var err error + data, err = os.ReadFile(skillMDPath) + if err != nil { + return "", "" + } + } + return parseFrontmatterNameDesc(data) +} + +// parseFrontmatterNameDesc extracts name and description from YAML frontmatter bytes. +func parseFrontmatterNameDesc(data []byte) (name, description string) { + lines := strings.Split(string(data), "\n") + inFrontmatter := false + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed == "---" { + if inFrontmatter { + break + } + inFrontmatter = true + continue + } + if !inFrontmatter { + continue + } + if strings.HasPrefix(trimmed, "name:") { + name = strings.TrimSpace(strings.TrimPrefix(trimmed, "name:")) + name = strings.Trim(name, "\"'") + } + if strings.HasPrefix(trimmed, "description:") { + description = strings.TrimSpace(strings.TrimPrefix(trimmed, "description:")) + description = strings.Trim(description, "\"'") + } + } + return name, description +} + +// extractRepoOwner returns the owner/org segment from a git URL. +// Supports HTTPS (https://github.com/owner/repo.git) and SSH (git@github.com:owner/repo.git). +func extractRepoOwner(repoURL string) string { + s := strings.TrimSpace(repoURL) + if s == "" { + return "" + } + + // SSH: git@host:owner/repo.git + if idx := strings.Index(s, ":"); idx > 0 && strings.Contains(s[:idx], "@") && !strings.Contains(s, "://") { + path := s[idx+1:] + parts := strings.SplitN(path, "/", 2) + if len(parts) >= 1 { + return strings.ToLower(parts[0]) + } + } + + // HTTPS: https://host/owner/repo.git + // Skip file:// URLs — they're local. + if strings.HasPrefix(s, "file://") { + return "" + } + // Strip scheme and host, get first path segment. + if strings.Contains(s, "://") { + afterScheme := strings.SplitN(s, "://", 2) + if len(afterScheme) == 2 { + path := afterScheme[1] + // Remove host + if slashIdx := strings.Index(path, "/"); slashIdx >= 0 { + path = path[slashIdx+1:] + } + parts := strings.SplitN(path, "/", 2) + if len(parts) >= 1 { + return strings.ToLower(parts[0]) + } + } + } + return "" +} + +// checkPublisherMismatch detects when SKILL.md claims a publisher +// that doesn't match the actual repo owner. +func checkPublisherMismatch(name, description, repoURL string) *Finding { + owner := extractRepoOwner(repoURL) + if owner == "" { + return nil + } + + // Combine name and description for claim extraction. + text := name + " " + description + matches := reOrgClaim.FindAllStringSubmatch(text, -1) + if len(matches) == 0 { + return nil + } + + for _, m := range matches { + claimed := "" + if m[1] != "" { + claimed = m[1] + } else if m[2] != "" { + claimed = m[2] + } + if claimed == "" { + continue + } + claimed = strings.TrimSpace(claimed) + claimedLower := strings.ToLower(claimed) + + // Allow if claimed name is a substring of owner or vice versa. + if strings.Contains(owner, claimedLower) || strings.Contains(claimedLower, owner) { + continue + } + + return &Finding{ + Severity: SeverityHigh, + Pattern: rulePublisherMismatch, + Message: fmt.Sprintf("skill claims origin %q but sourced from %q", claimed, owner), + File: "SKILL.md", + Line: 0, + RuleID: rulePublisherMismatch, + Analyzer: AnalyzerMetadata, + Category: CategoryTrust, + Confidence: 0.7, + } + } + return nil +} + +// checkAuthorityLanguage detects authority claims ("official", "verified") +// from sources that aren't well-known or verifiable. +func checkAuthorityLanguage(description, repoURL string) *Finding { + if description == "" { + return nil + } + lower := strings.ToLower(description) + + var found []string + for _, w := range authorityWords { + if strings.Contains(lower, w) { + found = append(found, w) + } + } + if len(found) == 0 { + return nil + } + + // If no repo URL (local skill), skip — user controls the source. + if repoURL == "" { + return nil + } + + // Well-known orgs are allowed authority claims. + owner := extractRepoOwner(repoURL) + if isWellKnownOrg(owner) { + return nil + } + + return &Finding{ + Severity: SeverityMedium, + Pattern: ruleAuthorityLanguage, + Message: fmt.Sprintf("skill uses authority language (%s) but source is unverified", strings.Join(found, ", ")), + File: "SKILL.md", + Line: 0, + RuleID: ruleAuthorityLanguage, + Analyzer: AnalyzerMetadata, + Category: CategoryTrust, + Confidence: 0.5, + } +} + +// isWellKnownOrg returns true for organizations whose authority claims +// are expected and should not trigger warnings. +func isWellKnownOrg(owner string) bool { + known := map[string]bool{ + "anthropics": true, + "openai": true, + "google": true, + "microsoft": true, + "github": true, + "gitlab": true, + "vercel": true, + "vercel-labs": true, + "meta": true, + "facebook": true, + "aws": true, + "amazon": true, + "apple": true, + "hashicorp": true, + "docker": true, + "kubernetes": true, + "rust-lang": true, + "golang": true, + "python": true, + "nodejs": true, + "composiohq": true, + "google-gemini": true, + } + return known[strings.ToLower(owner)] +} diff --git a/internal/audit/analyzer_metadata_test.go b/internal/audit/analyzer_metadata_test.go new file mode 100644 index 000000000..3caec764f --- /dev/null +++ b/internal/audit/analyzer_metadata_test.go @@ -0,0 +1,349 @@ +package audit + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +func TestExtractRepoOwner(t *testing.T) { + tests := []struct { + url string + want string + }{ + {"https://github.com/acme-corp/skills.git", "acme-corp"}, + {"https://gitlab.com/Team-X/monorepo.git", "team-x"}, + {"git@github.com:MyOrg/repo.git", "myorg"}, + {"git@gitlab.com:CompanyY/skills.git", "companyy"}, + {"file:///path/to/repo", ""}, + {"", ""}, + {"https://dev.azure.com/org/proj/_git/repo", "org"}, + } + for _, tt := range tests { + got := extractRepoOwner(tt.url) + if got != tt.want { + t.Errorf("extractRepoOwner(%q) = %q, want %q", tt.url, got, tt.want) + } + } +} + +func TestParseFrontmatterNameDesc(t *testing.T) { + tests := []struct { + name string + content string + wantName string + wantDesc string + }{ + { + name: "standard frontmatter", + content: "---\nname: my-skill\ndescription: A useful tool\n---\n# Content", + wantName: "my-skill", + wantDesc: "A useful tool", + }, + { + name: "quoted values", + content: "---\nname: \"my-skill\"\ndescription: 'Official tool from Acme Corp'\n---\n", + wantName: "my-skill", + wantDesc: "Official tool from Acme Corp", + }, + { + name: "no frontmatter", + content: "# Just content", + wantName: "", + wantDesc: "", + }, + { + name: "empty description", + content: "---\nname: test\n---\n", + wantName: "test", + wantDesc: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + name, desc := parseFrontmatterNameDesc([]byte(tt.content)) + if name != tt.wantName { + t.Errorf("name = %q, want %q", name, tt.wantName) + } + if desc != tt.wantDesc { + t.Errorf("description = %q, want %q", desc, tt.wantDesc) + } + }) + } +} + +func TestCheckPublisherMismatch(t *testing.T) { + tests := []struct { + name string + skillN string + desc string + repoURL string + wantNil bool + wantSev string + }{ + { + name: "mismatch: claims Acme but from random-user", + skillN: "formatter", + desc: "Official formatter from Acme Corp", + repoURL: "https://github.com/random-user/skills.git", + wantNil: false, + wantSev: SeverityHigh, + }, + { + name: "match: claims acme and from acme-corp", + skillN: "formatter", + desc: "formatter from acme-corp", + repoURL: "https://github.com/acme-corp/skills.git", + wantNil: true, + }, + { + name: "match: @mention matches owner", + skillN: "tool", + desc: "A tool by @myorg", + repoURL: "https://github.com/myorg/skills.git", + wantNil: true, + }, + { + name: "mismatch: @mention doesn't match", + skillN: "tool", + desc: "A tool by @bigcorp", + repoURL: "https://github.com/evil-fork/skills.git", + wantNil: false, + wantSev: SeverityHigh, + }, + { + name: "no claim in description", + skillN: "my-tool", + desc: "A great tool for coding", + repoURL: "https://github.com/someone/skills.git", + wantNil: true, + }, + { + name: "empty repo URL", + skillN: "tool", + desc: "from Acme Corp", + repoURL: "", + wantNil: true, + }, + { + name: "SSH URL mismatch", + skillN: "tool", + desc: "made by Google", + repoURL: "git@github.com:evil-fork/repo.git", + wantNil: false, + wantSev: SeverityHigh, + }, + { + name: "owner is substring of claim", + skillN: "tool", + desc: "maintained by Anthropics team", + repoURL: "https://github.com/anthropics/skills.git", + wantNil: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := checkPublisherMismatch(tt.skillN, tt.desc, tt.repoURL) + if tt.wantNil && f != nil { + t.Errorf("expected nil, got finding: %s", f.Message) + } + if !tt.wantNil { + if f == nil { + t.Fatal("expected finding, got nil") + } + if f.Severity != tt.wantSev { + t.Errorf("severity = %q, want %q", f.Severity, tt.wantSev) + } + if f.RuleID != rulePublisherMismatch { + t.Errorf("ruleID = %q, want %q", f.RuleID, rulePublisherMismatch) + } + } + }) + } +} + +func TestCheckAuthorityLanguage(t *testing.T) { + tests := []struct { + name string + desc string + repoURL string + wantNil bool + }{ + { + name: "official from unknown source", + desc: "Official database connector", + repoURL: "https://github.com/new-account-2025/skills.git", + wantNil: false, + }, + { + name: "official from well-known org", + desc: "Official Claude Code skill", + repoURL: "https://github.com/anthropics/skills.git", + wantNil: true, + }, + { + name: "no authority words", + desc: "A simple utility for formatting", + repoURL: "https://github.com/someone/skills.git", + wantNil: true, + }, + { + name: "authority but local source", + desc: "Official trusted tool", + repoURL: "", + wantNil: true, + }, + { + name: "verified from unknown", + desc: "Verified and trusted security scanner", + repoURL: "https://github.com/shady-org/skills.git", + wantNil: false, + }, + { + name: "empty description", + desc: "", + repoURL: "https://github.com/someone/skills.git", + wantNil: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := checkAuthorityLanguage(tt.desc, tt.repoURL) + if tt.wantNil && f != nil { + t.Errorf("expected nil, got finding: %s", f.Message) + } + if !tt.wantNil && f == nil { + t.Error("expected finding, got nil") + } + if !tt.wantNil && f != nil { + if f.Severity != SeverityMedium { + t.Errorf("severity = %q, want MEDIUM", f.Severity) + } + } + }) + } +} + +func TestIsWellKnownOrg(t *testing.T) { + tests := []struct { + owner string + want bool + }{ + {"anthropics", true}, + {"Anthropics", true}, + {"google", true}, + {"random-user", false}, + {"", false}, + {"google-gemini", true}, + } + for _, tt := range tests { + got := isWellKnownOrg(tt.owner) + if got != tt.want { + t.Errorf("isWellKnownOrg(%q) = %v, want %v", tt.owner, got, tt.want) + } + } +} + +func TestMetadataAnalyzer_Integration(t *testing.T) { + // Create a temp skill directory with SKILL.md and .skillshare-meta.json + dir := t.TempDir() + + // Write SKILL.md claiming "from Acme Corp" + skillContent := "---\nname: evil-skill\ndescription: Official formatter from Acme Corp\n---\n# Evil\n" + if err := os.WriteFile(filepath.Join(dir, "SKILL.md"), []byte(skillContent), 0644); err != nil { + t.Fatal(err) + } + + // Write meta pointing to a different org + meta := metaJSON{RepoURL: "https://github.com/evil-fork/skills.git"} + metaData, _ := json.Marshal(meta) + if err := os.WriteFile(filepath.Join(dir, ".skillshare-meta.json"), metaData, 0644); err != nil { + t.Fatal(err) + } + + a := &metadataAnalyzer{} + ctx := &AnalyzeContext{ + SkillPath: dir, + DisabledIDs: map[string]bool{}, + } + findings, err := a.Analyze(ctx) + if err != nil { + t.Fatal(err) + } + + // Should find both publisher mismatch (HIGH) and authority language (MEDIUM) + if len(findings) < 2 { + t.Errorf("expected >= 2 findings, got %d", len(findings)) + for _, f := range findings { + t.Logf(" %s: %s", f.Severity, f.Message) + } + return + } + + var hasMismatch, hasAuthority bool + for _, f := range findings { + if f.RuleID == rulePublisherMismatch { + hasMismatch = true + } + if f.RuleID == ruleAuthorityLanguage { + hasAuthority = true + } + } + if !hasMismatch { + t.Error("missing publisher-mismatch finding") + } + if !hasAuthority { + t.Error("missing authority-language finding") + } +} + +func TestMetadataAnalyzer_NoMeta(t *testing.T) { + // Skill without .skillshare-meta.json — should produce no findings + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "SKILL.md"), []byte("---\nname: test\n---\n# Test"), 0644); err != nil { + t.Fatal(err) + } + + a := &metadataAnalyzer{} + ctx := &AnalyzeContext{ + SkillPath: dir, + DisabledIDs: map[string]bool{}, + } + findings, err := a.Analyze(ctx) + if err != nil { + t.Fatal(err) + } + if len(findings) != 0 { + t.Errorf("expected 0 findings for skill without meta, got %d", len(findings)) + } +} + +func TestMetadataAnalyzer_DisabledRules(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "SKILL.md"), + []byte("---\nname: test\ndescription: Official tool from Acme Corp\n---\n"), 0644); err != nil { + t.Fatal(err) + } + meta := metaJSON{RepoURL: "https://github.com/evil-fork/skills.git"} + metaData, _ := json.Marshal(meta) + if err := os.WriteFile(filepath.Join(dir, ".skillshare-meta.json"), metaData, 0644); err != nil { + t.Fatal(err) + } + + a := &metadataAnalyzer{} + ctx := &AnalyzeContext{ + SkillPath: dir, + DisabledIDs: map[string]bool{ + rulePublisherMismatch: true, + ruleAuthorityLanguage: true, + }, + } + findings, err := a.Analyze(ctx) + if err != nil { + t.Fatal(err) + } + if len(findings) != 0 { + t.Errorf("expected 0 findings when rules disabled, got %d", len(findings)) + } +} diff --git a/internal/audit/analyzer_registry.go b/internal/audit/analyzer_registry.go index 9b4829c9e..d8ebb97d8 100644 --- a/internal/audit/analyzer_registry.go +++ b/internal/audit/analyzer_registry.go @@ -16,6 +16,7 @@ var defaultRegistry = &Registry{ &integrityAnalyzer{}, &tierAnalyzer{}, &crossSkillAnalyzer{}, + &metadataAnalyzer{}, }, } diff --git a/internal/audit/analyzer_registry_test.go b/internal/audit/analyzer_registry_test.go index 8626cec32..364a23b86 100644 --- a/internal/audit/analyzer_registry_test.go +++ b/internal/audit/analyzer_registry_test.go @@ -7,6 +7,7 @@ func TestDefaultRegistry_HasAllAnalyzers(t *testing.T) { for _, id := range []string{ AnalyzerStatic, AnalyzerDataflow, AnalyzerTier, AnalyzerIntegrity, AnalyzerStructure, AnalyzerCrossSkill, + AnalyzerMetadata, } { if !r.Has(id) { t.Errorf("default registry missing analyzer %q", id) @@ -19,8 +20,8 @@ func TestDefaultRegistry_ScopePartition(t *testing.T) { if len(r.FileAnalyzers()) != 2 { t.Errorf("file analyzers = %d, want 2 (static + dataflow)", len(r.FileAnalyzers())) } - if len(r.SkillAnalyzers()) != 4 { - t.Errorf("skill analyzers = %d, want 4 (markdownLink + structure + integrity + tier)", len(r.SkillAnalyzers())) + if len(r.SkillAnalyzers()) != 5 { + t.Errorf("skill analyzers = %d, want 5 (markdownLink + structure + integrity + tier + metadata)", len(r.SkillAnalyzers())) } if len(r.BundleAnalyzers()) != 1 { t.Errorf("bundle analyzers = %d, want 1 (cross-skill)", len(r.BundleAnalyzers())) diff --git a/internal/audit/analyzers.go b/internal/audit/analyzers.go index 74268592c..ddbe192e2 100644 --- a/internal/audit/analyzers.go +++ b/internal/audit/analyzers.go @@ -36,6 +36,7 @@ type AnalyzeContext struct { // --- Skill-scope fields (populated after Walk completes) --- SkillPath string + SourceURL string // git clone URL from .skillshare-meta.json (empty for local) MDFiles []mdFileInfo FileCache map[string][]byte TierProfile TierProfile diff --git a/internal/audit/audit.go b/internal/audit/audit.go index e355f55d0..d2edd3152 100644 --- a/internal/audit/audit.go +++ b/internal/audit/audit.go @@ -68,6 +68,7 @@ const ( AnalyzerIntegrity = "integrity" AnalyzerStructure = "structure" AnalyzerCrossSkill = "cross-skill" + AnalyzerMetadata = "metadata" ) // Category classifies what a finding is about. @@ -80,6 +81,7 @@ const ( CategoryIntegrity = "integrity" CategoryStructure = "structure" CategoryRisk = "risk" + CategoryTrust = "trust" ) // categoryForPattern maps a rule pattern name to a broad category.