feat(audit): add metadata analyzer for supply-chain trust verification

Add metadataAnalyzer (skill scope) that cross-references SKILL.md
metadata against the actual git source from .skillshare-meta.json:

- publisher-mismatch (HIGH): description claims an org but repo owner
  differs (e.g. "from Acme Corp" but sourced from evil-fork)
- authority-language (MEDIUM): uses "official"/"verified" language but
  source is not a well-known organization

Both rules are disable-able via audit-rules.yaml. The analyzer reads
meta from disk (zero API signature changes) and avoids import cycles
by defining its own minimal metaJSON struct.
This commit is contained in:
Willie
2026-03-05 02:13:21 +08:00
parent 272735c607
commit ebf2aaf822
6 changed files with 647 additions and 2 deletions
+291
View File
@@ -0,0 +1,291 @@
package audit
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
)
// metadataAnalyzer cross-references SKILL.md metadata (name, description)
// against the actual git source URL from .skillshare-meta.json.
// Detects social-engineering patterns: publisher mismatch and authority claims.
// Runs at skill scope after all files are walked.
type metadataAnalyzer struct{}
func (a *metadataAnalyzer) ID() string { return AnalyzerMetadata }
func (a *metadataAnalyzer) Scope() AnalyzerScope { return ScopeSkill }
// metaJSON is a minimal subset of install.SkillMeta to avoid import cycles.
type metaJSON struct {
RepoURL string `json:"repo_url"`
}
// Rule IDs for disable support via audit-rules.yaml.
const (
rulePublisherMismatch = "publisher-mismatch"
ruleAuthorityLanguage = "authority-language"
)
// reOrgClaim matches patterns like "from Acme Corp", "by Acme", "@acme".
var reOrgClaim = regexp.MustCompile(`(?i)(?:from|by|made by|created by|published by|maintained by)\s+([A-Z][\w-]+(?:\s+(?:Corp|Inc|Ltd|Team|Labs|AI|HQ|Co|Group))?)|@([A-Za-z][\w-]+)`)
// authorityWords are terms that imply official endorsement.
var authorityWords = []string{
"official",
"verified",
"trusted",
"authorized",
"endorsed",
"certified",
}
func (a *metadataAnalyzer) Analyze(ctx *AnalyzeContext) ([]Finding, error) {
if ctx.SkillPath == "" {
return nil, nil
}
// Read .skillshare-meta.json for source URL.
repoURL := readMetaRepoURL(ctx.SkillPath)
// Read SKILL.md frontmatter for name and description.
skillMDPath := filepath.Join(ctx.SkillPath, "SKILL.md")
name, description := readSkillFrontmatter(skillMDPath, ctx.FileCache)
var findings []Finding
// Rule A: Publisher mismatch — description claims an org but repo owner differs.
if !ctx.DisabledIDs[rulePublisherMismatch] && repoURL != "" {
if f := checkPublisherMismatch(name, description, repoURL); f != nil {
findings = append(findings, *f)
}
}
// Rule B: Authority language from unrecognized source.
if !ctx.DisabledIDs[ruleAuthorityLanguage] {
if f := checkAuthorityLanguage(description, repoURL); f != nil {
findings = append(findings, *f)
}
}
return findings, nil
}
// readMetaRepoURL reads repo_url from .skillshare-meta.json in skillPath.
func readMetaRepoURL(skillPath string) string {
data, err := os.ReadFile(filepath.Join(skillPath, ".skillshare-meta.json"))
if err != nil {
return ""
}
var m metaJSON
if json.Unmarshal(data, &m) != nil {
return ""
}
return m.RepoURL
}
// readSkillFrontmatter extracts name and description from SKILL.md.
// Uses fileCache if available, otherwise reads from disk.
func readSkillFrontmatter(skillMDPath string, fileCache map[string][]byte) (name, description string) {
var data []byte
if cached, ok := fileCache["SKILL.md"]; ok {
data = cached
} else {
var err error
data, err = os.ReadFile(skillMDPath)
if err != nil {
return "", ""
}
}
return parseFrontmatterNameDesc(data)
}
// parseFrontmatterNameDesc extracts name and description from YAML frontmatter bytes.
func parseFrontmatterNameDesc(data []byte) (name, description string) {
lines := strings.Split(string(data), "\n")
inFrontmatter := false
for _, line := range lines {
trimmed := strings.TrimSpace(line)
if trimmed == "---" {
if inFrontmatter {
break
}
inFrontmatter = true
continue
}
if !inFrontmatter {
continue
}
if strings.HasPrefix(trimmed, "name:") {
name = strings.TrimSpace(strings.TrimPrefix(trimmed, "name:"))
name = strings.Trim(name, "\"'")
}
if strings.HasPrefix(trimmed, "description:") {
description = strings.TrimSpace(strings.TrimPrefix(trimmed, "description:"))
description = strings.Trim(description, "\"'")
}
}
return name, description
}
// extractRepoOwner returns the owner/org segment from a git URL.
// Supports HTTPS (https://github.com/owner/repo.git) and SSH (git@github.com:owner/repo.git).
func extractRepoOwner(repoURL string) string {
s := strings.TrimSpace(repoURL)
if s == "" {
return ""
}
// SSH: git@host:owner/repo.git
if idx := strings.Index(s, ":"); idx > 0 && strings.Contains(s[:idx], "@") && !strings.Contains(s, "://") {
path := s[idx+1:]
parts := strings.SplitN(path, "/", 2)
if len(parts) >= 1 {
return strings.ToLower(parts[0])
}
}
// HTTPS: https://host/owner/repo.git
// Skip file:// URLs — they're local.
if strings.HasPrefix(s, "file://") {
return ""
}
// Strip scheme and host, get first path segment.
if strings.Contains(s, "://") {
afterScheme := strings.SplitN(s, "://", 2)
if len(afterScheme) == 2 {
path := afterScheme[1]
// Remove host
if slashIdx := strings.Index(path, "/"); slashIdx >= 0 {
path = path[slashIdx+1:]
}
parts := strings.SplitN(path, "/", 2)
if len(parts) >= 1 {
return strings.ToLower(parts[0])
}
}
}
return ""
}
// checkPublisherMismatch detects when SKILL.md claims a publisher
// that doesn't match the actual repo owner.
func checkPublisherMismatch(name, description, repoURL string) *Finding {
owner := extractRepoOwner(repoURL)
if owner == "" {
return nil
}
// Combine name and description for claim extraction.
text := name + " " + description
matches := reOrgClaim.FindAllStringSubmatch(text, -1)
if len(matches) == 0 {
return nil
}
for _, m := range matches {
claimed := ""
if m[1] != "" {
claimed = m[1]
} else if m[2] != "" {
claimed = m[2]
}
if claimed == "" {
continue
}
claimed = strings.TrimSpace(claimed)
claimedLower := strings.ToLower(claimed)
// Allow if claimed name is a substring of owner or vice versa.
if strings.Contains(owner, claimedLower) || strings.Contains(claimedLower, owner) {
continue
}
return &Finding{
Severity: SeverityHigh,
Pattern: rulePublisherMismatch,
Message: fmt.Sprintf("skill claims origin %q but sourced from %q", claimed, owner),
File: "SKILL.md",
Line: 0,
RuleID: rulePublisherMismatch,
Analyzer: AnalyzerMetadata,
Category: CategoryTrust,
Confidence: 0.7,
}
}
return nil
}
// checkAuthorityLanguage detects authority claims ("official", "verified")
// from sources that aren't well-known or verifiable.
func checkAuthorityLanguage(description, repoURL string) *Finding {
if description == "" {
return nil
}
lower := strings.ToLower(description)
var found []string
for _, w := range authorityWords {
if strings.Contains(lower, w) {
found = append(found, w)
}
}
if len(found) == 0 {
return nil
}
// If no repo URL (local skill), skip — user controls the source.
if repoURL == "" {
return nil
}
// Well-known orgs are allowed authority claims.
owner := extractRepoOwner(repoURL)
if isWellKnownOrg(owner) {
return nil
}
return &Finding{
Severity: SeverityMedium,
Pattern: ruleAuthorityLanguage,
Message: fmt.Sprintf("skill uses authority language (%s) but source is unverified", strings.Join(found, ", ")),
File: "SKILL.md",
Line: 0,
RuleID: ruleAuthorityLanguage,
Analyzer: AnalyzerMetadata,
Category: CategoryTrust,
Confidence: 0.5,
}
}
// isWellKnownOrg returns true for organizations whose authority claims
// are expected and should not trigger warnings.
func isWellKnownOrg(owner string) bool {
known := map[string]bool{
"anthropics": true,
"openai": true,
"google": true,
"microsoft": true,
"github": true,
"gitlab": true,
"vercel": true,
"vercel-labs": true,
"meta": true,
"facebook": true,
"aws": true,
"amazon": true,
"apple": true,
"hashicorp": true,
"docker": true,
"kubernetes": true,
"rust-lang": true,
"golang": true,
"python": true,
"nodejs": true,
"composiohq": true,
"google-gemini": true,
}
return known[strings.ToLower(owner)]
}
+349
View File
@@ -0,0 +1,349 @@
package audit
import (
"encoding/json"
"os"
"path/filepath"
"testing"
)
func TestExtractRepoOwner(t *testing.T) {
tests := []struct {
url string
want string
}{
{"https://github.com/acme-corp/skills.git", "acme-corp"},
{"https://gitlab.com/Team-X/monorepo.git", "team-x"},
{"git@github.com:MyOrg/repo.git", "myorg"},
{"git@gitlab.com:CompanyY/skills.git", "companyy"},
{"file:///path/to/repo", ""},
{"", ""},
{"https://dev.azure.com/org/proj/_git/repo", "org"},
}
for _, tt := range tests {
got := extractRepoOwner(tt.url)
if got != tt.want {
t.Errorf("extractRepoOwner(%q) = %q, want %q", tt.url, got, tt.want)
}
}
}
func TestParseFrontmatterNameDesc(t *testing.T) {
tests := []struct {
name string
content string
wantName string
wantDesc string
}{
{
name: "standard frontmatter",
content: "---\nname: my-skill\ndescription: A useful tool\n---\n# Content",
wantName: "my-skill",
wantDesc: "A useful tool",
},
{
name: "quoted values",
content: "---\nname: \"my-skill\"\ndescription: 'Official tool from Acme Corp'\n---\n",
wantName: "my-skill",
wantDesc: "Official tool from Acme Corp",
},
{
name: "no frontmatter",
content: "# Just content",
wantName: "",
wantDesc: "",
},
{
name: "empty description",
content: "---\nname: test\n---\n",
wantName: "test",
wantDesc: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
name, desc := parseFrontmatterNameDesc([]byte(tt.content))
if name != tt.wantName {
t.Errorf("name = %q, want %q", name, tt.wantName)
}
if desc != tt.wantDesc {
t.Errorf("description = %q, want %q", desc, tt.wantDesc)
}
})
}
}
func TestCheckPublisherMismatch(t *testing.T) {
tests := []struct {
name string
skillN string
desc string
repoURL string
wantNil bool
wantSev string
}{
{
name: "mismatch: claims Acme but from random-user",
skillN: "formatter",
desc: "Official formatter from Acme Corp",
repoURL: "https://github.com/random-user/skills.git",
wantNil: false,
wantSev: SeverityHigh,
},
{
name: "match: claims acme and from acme-corp",
skillN: "formatter",
desc: "formatter from acme-corp",
repoURL: "https://github.com/acme-corp/skills.git",
wantNil: true,
},
{
name: "match: @mention matches owner",
skillN: "tool",
desc: "A tool by @myorg",
repoURL: "https://github.com/myorg/skills.git",
wantNil: true,
},
{
name: "mismatch: @mention doesn't match",
skillN: "tool",
desc: "A tool by @bigcorp",
repoURL: "https://github.com/evil-fork/skills.git",
wantNil: false,
wantSev: SeverityHigh,
},
{
name: "no claim in description",
skillN: "my-tool",
desc: "A great tool for coding",
repoURL: "https://github.com/someone/skills.git",
wantNil: true,
},
{
name: "empty repo URL",
skillN: "tool",
desc: "from Acme Corp",
repoURL: "",
wantNil: true,
},
{
name: "SSH URL mismatch",
skillN: "tool",
desc: "made by Google",
repoURL: "git@github.com:evil-fork/repo.git",
wantNil: false,
wantSev: SeverityHigh,
},
{
name: "owner is substring of claim",
skillN: "tool",
desc: "maintained by Anthropics team",
repoURL: "https://github.com/anthropics/skills.git",
wantNil: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f := checkPublisherMismatch(tt.skillN, tt.desc, tt.repoURL)
if tt.wantNil && f != nil {
t.Errorf("expected nil, got finding: %s", f.Message)
}
if !tt.wantNil {
if f == nil {
t.Fatal("expected finding, got nil")
}
if f.Severity != tt.wantSev {
t.Errorf("severity = %q, want %q", f.Severity, tt.wantSev)
}
if f.RuleID != rulePublisherMismatch {
t.Errorf("ruleID = %q, want %q", f.RuleID, rulePublisherMismatch)
}
}
})
}
}
func TestCheckAuthorityLanguage(t *testing.T) {
tests := []struct {
name string
desc string
repoURL string
wantNil bool
}{
{
name: "official from unknown source",
desc: "Official database connector",
repoURL: "https://github.com/new-account-2025/skills.git",
wantNil: false,
},
{
name: "official from well-known org",
desc: "Official Claude Code skill",
repoURL: "https://github.com/anthropics/skills.git",
wantNil: true,
},
{
name: "no authority words",
desc: "A simple utility for formatting",
repoURL: "https://github.com/someone/skills.git",
wantNil: true,
},
{
name: "authority but local source",
desc: "Official trusted tool",
repoURL: "",
wantNil: true,
},
{
name: "verified from unknown",
desc: "Verified and trusted security scanner",
repoURL: "https://github.com/shady-org/skills.git",
wantNil: false,
},
{
name: "empty description",
desc: "",
repoURL: "https://github.com/someone/skills.git",
wantNil: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f := checkAuthorityLanguage(tt.desc, tt.repoURL)
if tt.wantNil && f != nil {
t.Errorf("expected nil, got finding: %s", f.Message)
}
if !tt.wantNil && f == nil {
t.Error("expected finding, got nil")
}
if !tt.wantNil && f != nil {
if f.Severity != SeverityMedium {
t.Errorf("severity = %q, want MEDIUM", f.Severity)
}
}
})
}
}
func TestIsWellKnownOrg(t *testing.T) {
tests := []struct {
owner string
want bool
}{
{"anthropics", true},
{"Anthropics", true},
{"google", true},
{"random-user", false},
{"", false},
{"google-gemini", true},
}
for _, tt := range tests {
got := isWellKnownOrg(tt.owner)
if got != tt.want {
t.Errorf("isWellKnownOrg(%q) = %v, want %v", tt.owner, got, tt.want)
}
}
}
func TestMetadataAnalyzer_Integration(t *testing.T) {
// Create a temp skill directory with SKILL.md and .skillshare-meta.json
dir := t.TempDir()
// Write SKILL.md claiming "from Acme Corp"
skillContent := "---\nname: evil-skill\ndescription: Official formatter from Acme Corp\n---\n# Evil\n"
if err := os.WriteFile(filepath.Join(dir, "SKILL.md"), []byte(skillContent), 0644); err != nil {
t.Fatal(err)
}
// Write meta pointing to a different org
meta := metaJSON{RepoURL: "https://github.com/evil-fork/skills.git"}
metaData, _ := json.Marshal(meta)
if err := os.WriteFile(filepath.Join(dir, ".skillshare-meta.json"), metaData, 0644); err != nil {
t.Fatal(err)
}
a := &metadataAnalyzer{}
ctx := &AnalyzeContext{
SkillPath: dir,
DisabledIDs: map[string]bool{},
}
findings, err := a.Analyze(ctx)
if err != nil {
t.Fatal(err)
}
// Should find both publisher mismatch (HIGH) and authority language (MEDIUM)
if len(findings) < 2 {
t.Errorf("expected >= 2 findings, got %d", len(findings))
for _, f := range findings {
t.Logf(" %s: %s", f.Severity, f.Message)
}
return
}
var hasMismatch, hasAuthority bool
for _, f := range findings {
if f.RuleID == rulePublisherMismatch {
hasMismatch = true
}
if f.RuleID == ruleAuthorityLanguage {
hasAuthority = true
}
}
if !hasMismatch {
t.Error("missing publisher-mismatch finding")
}
if !hasAuthority {
t.Error("missing authority-language finding")
}
}
func TestMetadataAnalyzer_NoMeta(t *testing.T) {
// Skill without .skillshare-meta.json — should produce no findings
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "SKILL.md"), []byte("---\nname: test\n---\n# Test"), 0644); err != nil {
t.Fatal(err)
}
a := &metadataAnalyzer{}
ctx := &AnalyzeContext{
SkillPath: dir,
DisabledIDs: map[string]bool{},
}
findings, err := a.Analyze(ctx)
if err != nil {
t.Fatal(err)
}
if len(findings) != 0 {
t.Errorf("expected 0 findings for skill without meta, got %d", len(findings))
}
}
func TestMetadataAnalyzer_DisabledRules(t *testing.T) {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "SKILL.md"),
[]byte("---\nname: test\ndescription: Official tool from Acme Corp\n---\n"), 0644); err != nil {
t.Fatal(err)
}
meta := metaJSON{RepoURL: "https://github.com/evil-fork/skills.git"}
metaData, _ := json.Marshal(meta)
if err := os.WriteFile(filepath.Join(dir, ".skillshare-meta.json"), metaData, 0644); err != nil {
t.Fatal(err)
}
a := &metadataAnalyzer{}
ctx := &AnalyzeContext{
SkillPath: dir,
DisabledIDs: map[string]bool{
rulePublisherMismatch: true,
ruleAuthorityLanguage: true,
},
}
findings, err := a.Analyze(ctx)
if err != nil {
t.Fatal(err)
}
if len(findings) != 0 {
t.Errorf("expected 0 findings when rules disabled, got %d", len(findings))
}
}
+1
View File
@@ -16,6 +16,7 @@ var defaultRegistry = &Registry{
&integrityAnalyzer{},
&tierAnalyzer{},
&crossSkillAnalyzer{},
&metadataAnalyzer{},
},
}
+3 -2
View File
@@ -7,6 +7,7 @@ func TestDefaultRegistry_HasAllAnalyzers(t *testing.T) {
for _, id := range []string{
AnalyzerStatic, AnalyzerDataflow, AnalyzerTier,
AnalyzerIntegrity, AnalyzerStructure, AnalyzerCrossSkill,
AnalyzerMetadata,
} {
if !r.Has(id) {
t.Errorf("default registry missing analyzer %q", id)
@@ -19,8 +20,8 @@ func TestDefaultRegistry_ScopePartition(t *testing.T) {
if len(r.FileAnalyzers()) != 2 {
t.Errorf("file analyzers = %d, want 2 (static + dataflow)", len(r.FileAnalyzers()))
}
if len(r.SkillAnalyzers()) != 4 {
t.Errorf("skill analyzers = %d, want 4 (markdownLink + structure + integrity + tier)", len(r.SkillAnalyzers()))
if len(r.SkillAnalyzers()) != 5 {
t.Errorf("skill analyzers = %d, want 5 (markdownLink + structure + integrity + tier + metadata)", len(r.SkillAnalyzers()))
}
if len(r.BundleAnalyzers()) != 1 {
t.Errorf("bundle analyzers = %d, want 1 (cross-skill)", len(r.BundleAnalyzers()))
+1
View File
@@ -36,6 +36,7 @@ type AnalyzeContext struct {
// --- Skill-scope fields (populated after Walk completes) ---
SkillPath string
SourceURL string // git clone URL from .skillshare-meta.json (empty for local)
MDFiles []mdFileInfo
FileCache map[string][]byte
TierProfile TierProfile
+2
View File
@@ -68,6 +68,7 @@ const (
AnalyzerIntegrity = "integrity"
AnalyzerStructure = "structure"
AnalyzerCrossSkill = "cross-skill"
AnalyzerMetadata = "metadata"
)
// Category classifies what a finding is about.
@@ -80,6 +81,7 @@ const (
CategoryIntegrity = "integrity"
CategoryStructure = "structure"
CategoryRisk = "risk"
CategoryTrust = "trust"
)
// categoryForPattern maps a rule pattern name to a broad category.