mirror of
https://github.com/runkids/skillshare.git
synced 2026-11-19 02:17:09 +00:00
Credential-access rules match whole lines, so a Python docstring that named ~/.aws/credentials in a `cat` example counted as a CRITICAL read and blocked the mcp-sentinel skill at the default threshold (#817). In .py files, a built-in CRITICAL credential-access match on a comment-only line, or inside a string that is a whole statement such as a docstring, is now HIGH: it still shows, and still blocks under a HIGH threshold. A small tokenizer decides this and keeps CRITICAL wherever it cannot be sure the line never runs: argument, assigned and bracketed strings, f-strings and their replacement fields, code before a trailing comment, files that read __doc__ or use bare CR line breaks, and explicit severity overrides. A lowered finding carries severityContext "python-inert" (JSON, SARIF, `audit rules --format json`), and the dashboard, CLI text output, both audit TUIs, install/update warnings and the markdown report say it was lowered from CRITICAL, so it is not mistaken for a changed rule. Closes #817
128 lines
6.1 KiB
Go
128 lines
6.1 KiB
Go
package audit
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
)
|
|
|
|
func credentialRules(t testing.TB, rules []rule) []rule {
|
|
t.Helper()
|
|
var selected []rule
|
|
for _, r := range rules {
|
|
if r.Pattern == "credential-access" {
|
|
selected = append(selected, r)
|
|
}
|
|
}
|
|
if len(selected) == 0 {
|
|
t.Fatal("missing credential-access rules")
|
|
}
|
|
return selected
|
|
}
|
|
|
|
func TestPythonInertCredentialSeverity(t *testing.T) {
|
|
builtin, err := loadBuiltinRules()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rules := credentialRules(t, builtin)
|
|
const cmd = "cat ~/.aws/credentials"
|
|
for _, tc := range []struct{ name, file, content, severity string }{
|
|
{"function docstring", "hook.py", "def path_matches(pattern, text):\n \"\"\"Match a pattern.\n\n Shell commands such as " + cmd + " start mid-text.\n \"\"\"\n return True", SeverityHigh},
|
|
{"module docstring", "hook.py", "#!/usr/bin/env python3\n\"\"\"Blocks " + cmd + ".\"\"\"", SeverityHigh},
|
|
{"raw single-quoted docstring", "hook.py", "def f():\n r'''" + cmd + "''' # example", SeverityHigh},
|
|
{"escaped quote inside docstring", "hook.py", "\"\"\"a \\\"\"\" still open\n" + cmd + "\n\"\"\"", SeverityHigh},
|
|
{"single-quoted docstring", "hook.py", "def f():\n \"Blocks " + cmd + ".\"", SeverityHigh},
|
|
{"docstring after plain f-strings", "hook.py", "print(f\"{x!r:>{w}} {{ok}}\")\ndef f():\n \"\"\"" + cmd + "\"\"\"", SeverityHigh},
|
|
{"CRLF docstring", "hook.py", "def f():\r\n \"\"\"Example:\r\n " + cmd + "\r\n \"\"\"\r\n", SeverityHigh},
|
|
{"comment", "hook.PY", "x = 1\n # e.g. " + cmd, SeverityHigh},
|
|
{"comment inside call", "hook.py", "run(\n # " + cmd + "\n args,\n)", SeverityHigh},
|
|
{"call", "hook.py", "os.system(\"" + cmd + "\")", SeverityCritical},
|
|
{"trailing comment after code", "hook.py", "x = 1 # " + cmd, SeverityCritical},
|
|
{"string argument", "hook.py", "subprocess.run(\"\"\"" + cmd + "\"\"\", shell=True)", SeverityCritical},
|
|
{"string argument on its own line", "hook.py", "subprocess.run(\n \"\"\"" + cmd + "\"\"\",\n shell=True,\n)", SeverityCritical},
|
|
{"assigned string", "hook.py", "CMD = \"\"\"\n" + cmd + "\n\"\"\"", SeverityCritical},
|
|
{"comment line inside assigned string", "hook.py", "CMD = \"\"\"\n# " + cmd + "\n\"\"\"", SeverityCritical},
|
|
{"continued assignment", "hook.py", "CMD = \\\n \"\"\"" + cmd + "\"\"\"", SeverityCritical},
|
|
{"string used after close", "hook.py", "\"\"\"\n" + cmd + "\n\"\"\".split()", SeverityCritical},
|
|
{"f-string evaluates", "hook.py", "f\"\"\"{os.system('" + cmd + "')}\"\"\"", SeverityCritical},
|
|
{"unclosed string", "hook.py", "\"\"\"\n" + cmd, SeverityCritical},
|
|
{"backslash-continued string", "hook.py", "x = '\\\n# '; os.system('" + cmd + "')", SeverityCritical},
|
|
{"multi-line f-string field", "hook.py", "c = f\"{\n'''" + cmd + "'''\n}\"", SeverityCritical},
|
|
{"bare CR line break", "hook.py", "import os\n# example\ros.system('" + cmd + "')\n", SeverityCritical},
|
|
{"nested quote in f-string field", "hook.py", "f\"\"\"{os.system(\n \"\"\"#\"\"\"\n \"; " + cmd + "\"\n)}\"\"\"", SeverityCritical},
|
|
{"comment in f-string field", "hook.py", "f\"\"\"{os.system(\n# } \"\"\"#\n\"; " + cmd + "\"\n)}\"\"\"", SeverityCritical},
|
|
{"backslash before f-string field", "hook.py", "f\"\"\"\\{os.system(\n\"\"\"#\"\"\"\n\"; " + cmd + "\"\n)}\"\"\"", SeverityCritical},
|
|
{"docstring read back", "hook.py", "\"\"\"" + cmd + "\"\"\"\nimport os; os.system(__doc__)", SeverityCritical},
|
|
{"shell comment keeps severity", "hook.sh", "# " + cmd, SeverityCritical},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
wantContext := ""
|
|
if tc.severity == SeverityHigh {
|
|
wantContext = SeverityContextPythonInert
|
|
}
|
|
check := func(path string, findings []Finding) {
|
|
t.Helper()
|
|
if len(findings) != 1 || findings[0].RuleID != "credential-access-aws-credentials" || findings[0].Severity != tc.severity || findings[0].SeverityContext != wantContext {
|
|
t.Fatalf("%s: want one %s aws finding with context %q, got %+v", path, tc.severity, wantContext, findings)
|
|
}
|
|
}
|
|
check("content", ScanContentWithRules([]byte(tc.content), tc.file, rules))
|
|
dir := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(dir, tc.file), []byte(tc.content), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
result, err := ScanSkillWithRules(dir, rules)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
check("skill", result.Findings)
|
|
result, err = scanFileImpl(filepath.Join(dir, tc.file), rules, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
check("file", result.Findings)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPythonInertCredentialOverrides(t *testing.T) {
|
|
base, err := parseBuiltinRulesYAML()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
content := []byte("# cat ~/.aws/credentials")
|
|
for _, tc := range []struct{ name, overlay, severity string }{
|
|
{"partial inherits context", "- id: credential-access-aws-credentials\n files: ['*.py']", SeverityHigh},
|
|
{"rule severity", "- id: credential-access-aws-credentials\n severity: CRITICAL", SeverityCritical},
|
|
{"pattern severity", "- pattern: credential-access\n severity: CRITICAL", SeverityCritical},
|
|
{"custom regex", "- id: credential-access-aws-credentials\n severity: CRITICAL\n pattern: credential-access\n regex: 'cat ~/\\.aws/credentials'", SeverityCritical},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
overlay, err := parseRulesYAML([]byte("rules:\n" + tc.overlay))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rules, err := compileRules(mergeYAMLRules(base, overlay))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
found := ScanContentWithRules(content, "hook.py", credentialRules(t, rules))
|
|
if len(found) != 1 || found[0].Severity != tc.severity {
|
|
t.Fatalf("want %s, got %+v", tc.severity, found)
|
|
}
|
|
for layer, layers := range map[string][2][]yamlRule{"global": {overlay, nil}, "project": {nil, overlay}} {
|
|
listed, err := buildCompiledList(base, layers[0], layers[1])
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, r := range listed {
|
|
if r.ID == "credential-access-aws-credentials" && (r.SeverityContext != "") != (tc.severity == SeverityHigh) {
|
|
t.Fatalf("%s rule list context %q disagrees with scanned severity %s", layer, r.SeverityContext, tc.severity)
|
|
}
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|