Files
skillshare/internal/audit/python_inert_test.go
T
Willie 037c7943cf fix(audit): report credential paths in Python comments and docstrings as HIGH (#819)
Credential-access rules match whole lines, so a Python docstring that
named ~/.aws/credentials in a `cat` example counted as a CRITICAL read and
blocked the mcp-sentinel skill at the default threshold (#817).

In .py files, a built-in CRITICAL credential-access match on a comment-only
line, or inside a string that is a whole statement such as a docstring, is
now HIGH: it still shows, and still blocks under a HIGH threshold. A small
tokenizer decides this and keeps CRITICAL wherever it cannot be sure the
line never runs: argument, assigned and bracketed strings, f-strings and
their replacement fields, code before a trailing comment, files that read
__doc__ or use bare CR line breaks, and explicit severity overrides.

A lowered finding carries severityContext "python-inert" (JSON, SARIF,
`audit rules --format json`), and the dashboard, CLI text output, both
audit TUIs, install/update warnings and the markdown report say it was
lowered from CRITICAL, so it is not mistaken for a changed rule.

Closes #817
2026-11-09 23:30:37 +08:00

128 lines
6.1 KiB
Go

package audit
import (
"os"
"path/filepath"
"testing"
)
func credentialRules(t testing.TB, rules []rule) []rule {
t.Helper()
var selected []rule
for _, r := range rules {
if r.Pattern == "credential-access" {
selected = append(selected, r)
}
}
if len(selected) == 0 {
t.Fatal("missing credential-access rules")
}
return selected
}
func TestPythonInertCredentialSeverity(t *testing.T) {
builtin, err := loadBuiltinRules()
if err != nil {
t.Fatal(err)
}
rules := credentialRules(t, builtin)
const cmd = "cat ~/.aws/credentials"
for _, tc := range []struct{ name, file, content, severity string }{
{"function docstring", "hook.py", "def path_matches(pattern, text):\n \"\"\"Match a pattern.\n\n Shell commands such as " + cmd + " start mid-text.\n \"\"\"\n return True", SeverityHigh},
{"module docstring", "hook.py", "#!/usr/bin/env python3\n\"\"\"Blocks " + cmd + ".\"\"\"", SeverityHigh},
{"raw single-quoted docstring", "hook.py", "def f():\n r'''" + cmd + "''' # example", SeverityHigh},
{"escaped quote inside docstring", "hook.py", "\"\"\"a \\\"\"\" still open\n" + cmd + "\n\"\"\"", SeverityHigh},
{"single-quoted docstring", "hook.py", "def f():\n \"Blocks " + cmd + ".\"", SeverityHigh},
{"docstring after plain f-strings", "hook.py", "print(f\"{x!r:>{w}} {{ok}}\")\ndef f():\n \"\"\"" + cmd + "\"\"\"", SeverityHigh},
{"CRLF docstring", "hook.py", "def f():\r\n \"\"\"Example:\r\n " + cmd + "\r\n \"\"\"\r\n", SeverityHigh},
{"comment", "hook.PY", "x = 1\n # e.g. " + cmd, SeverityHigh},
{"comment inside call", "hook.py", "run(\n # " + cmd + "\n args,\n)", SeverityHigh},
{"call", "hook.py", "os.system(\"" + cmd + "\")", SeverityCritical},
{"trailing comment after code", "hook.py", "x = 1 # " + cmd, SeverityCritical},
{"string argument", "hook.py", "subprocess.run(\"\"\"" + cmd + "\"\"\", shell=True)", SeverityCritical},
{"string argument on its own line", "hook.py", "subprocess.run(\n \"\"\"" + cmd + "\"\"\",\n shell=True,\n)", SeverityCritical},
{"assigned string", "hook.py", "CMD = \"\"\"\n" + cmd + "\n\"\"\"", SeverityCritical},
{"comment line inside assigned string", "hook.py", "CMD = \"\"\"\n# " + cmd + "\n\"\"\"", SeverityCritical},
{"continued assignment", "hook.py", "CMD = \\\n \"\"\"" + cmd + "\"\"\"", SeverityCritical},
{"string used after close", "hook.py", "\"\"\"\n" + cmd + "\n\"\"\".split()", SeverityCritical},
{"f-string evaluates", "hook.py", "f\"\"\"{os.system('" + cmd + "')}\"\"\"", SeverityCritical},
{"unclosed string", "hook.py", "\"\"\"\n" + cmd, SeverityCritical},
{"backslash-continued string", "hook.py", "x = '\\\n# '; os.system('" + cmd + "')", SeverityCritical},
{"multi-line f-string field", "hook.py", "c = f\"{\n'''" + cmd + "'''\n}\"", SeverityCritical},
{"bare CR line break", "hook.py", "import os\n# example\ros.system('" + cmd + "')\n", SeverityCritical},
{"nested quote in f-string field", "hook.py", "f\"\"\"{os.system(\n \"\"\"#\"\"\"\n \"; " + cmd + "\"\n)}\"\"\"", SeverityCritical},
{"comment in f-string field", "hook.py", "f\"\"\"{os.system(\n# } \"\"\"#\n\"; " + cmd + "\"\n)}\"\"\"", SeverityCritical},
{"backslash before f-string field", "hook.py", "f\"\"\"\\{os.system(\n\"\"\"#\"\"\"\n\"; " + cmd + "\"\n)}\"\"\"", SeverityCritical},
{"docstring read back", "hook.py", "\"\"\"" + cmd + "\"\"\"\nimport os; os.system(__doc__)", SeverityCritical},
{"shell comment keeps severity", "hook.sh", "# " + cmd, SeverityCritical},
} {
t.Run(tc.name, func(t *testing.T) {
wantContext := ""
if tc.severity == SeverityHigh {
wantContext = SeverityContextPythonInert
}
check := func(path string, findings []Finding) {
t.Helper()
if len(findings) != 1 || findings[0].RuleID != "credential-access-aws-credentials" || findings[0].Severity != tc.severity || findings[0].SeverityContext != wantContext {
t.Fatalf("%s: want one %s aws finding with context %q, got %+v", path, tc.severity, wantContext, findings)
}
}
check("content", ScanContentWithRules([]byte(tc.content), tc.file, rules))
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, tc.file), []byte(tc.content), 0600); err != nil {
t.Fatal(err)
}
result, err := ScanSkillWithRules(dir, rules)
if err != nil {
t.Fatal(err)
}
check("skill", result.Findings)
result, err = scanFileImpl(filepath.Join(dir, tc.file), rules, nil)
if err != nil {
t.Fatal(err)
}
check("file", result.Findings)
})
}
}
func TestPythonInertCredentialOverrides(t *testing.T) {
base, err := parseBuiltinRulesYAML()
if err != nil {
t.Fatal(err)
}
content := []byte("# cat ~/.aws/credentials")
for _, tc := range []struct{ name, overlay, severity string }{
{"partial inherits context", "- id: credential-access-aws-credentials\n files: ['*.py']", SeverityHigh},
{"rule severity", "- id: credential-access-aws-credentials\n severity: CRITICAL", SeverityCritical},
{"pattern severity", "- pattern: credential-access\n severity: CRITICAL", SeverityCritical},
{"custom regex", "- id: credential-access-aws-credentials\n severity: CRITICAL\n pattern: credential-access\n regex: 'cat ~/\\.aws/credentials'", SeverityCritical},
} {
t.Run(tc.name, func(t *testing.T) {
overlay, err := parseRulesYAML([]byte("rules:\n" + tc.overlay))
if err != nil {
t.Fatal(err)
}
rules, err := compileRules(mergeYAMLRules(base, overlay))
if err != nil {
t.Fatal(err)
}
found := ScanContentWithRules(content, "hook.py", credentialRules(t, rules))
if len(found) != 1 || found[0].Severity != tc.severity {
t.Fatalf("want %s, got %+v", tc.severity, found)
}
for layer, layers := range map[string][2][]yamlRule{"global": {overlay, nil}, "project": {nil, overlay}} {
listed, err := buildCompiledList(base, layers[0], layers[1])
if err != nil {
t.Fatal(err)
}
for _, r := range listed {
if r.ID == "credential-access-aws-credentials" && (r.SeverityContext != "") != (tc.severity == SeverityHigh) {
t.Fatalf("%s rule list context %q disagrees with scanned severity %s", layer, r.SeverityContext, tc.severity)
}
}
}
})
}
}