From 8d4eacbba6f5cc6530159db6618e27ff7ea9c58d Mon Sep 17 00:00:00 2001 From: Wayner Barrios Date: Mon, 27 Jul 2026 15:33:27 -0400 Subject: [PATCH] Harden security review contracts --- skills/security-review/SKILL.md | 315 ++++++++++++++++---------------- tests/security-review.test.mjs | 60 ++++++ 2 files changed, 214 insertions(+), 161 deletions(-) create mode 100644 tests/security-review.test.mjs diff --git a/skills/security-review/SKILL.md b/skills/security-review/SKILL.md index 129591d..01a1548 100644 --- a/skills/security-review/SKILL.md +++ b/skills/security-review/SKILL.md @@ -6,188 +6,181 @@ license: MIT (modified; see UPSTREAMS.json) # Security Review -You are a senior security engineer conducting a focused security review of the changes on a git branch. The goal is to identify HIGH-CONFIDENCE security vulnerabilities that could have real exploitation potential, while ruthlessly avoiding false positives. +Review one frozen change set as a senior security engineer. Report only high-confidence vulnerabilities introduced by that change set, but make incomplete coverage visible instead of turning missing analysis into a clean result. -## When to use this skill +This is not a general code review. Use `code-review` for general correctness or convention review. -Trigger on phrases like "security review", "security audit", "check for vulnerabilities", "vuln scan on this branch", or when the user asks to review pending changes specifically for security implications. +## Workflow -This is **not** a general code review skill. Use the `code-review` skill for general PR review. Use this one only when the user is asking specifically about security. +Track scope discovery, category analysis, filtering, exploit validation, recovery, and output in a todo list. Do not run commands to reproduce vulnerabilities; inspect code and repository evidence only. -## Gather context +### 1. Freeze the scope -Before any analysis, run these commands to understand the change set: +Resolve review mode in this order. Explicit requested scope takes precedence over every inferred scope: -```bash -git status -git diff --name-only origin/HEAD... -git log --no-decorate origin/HEAD... -git diff --merge-base origin/HEAD +1. Explicit requested paths, commit range, or revision. +2. An explicit PR URL or number. +3. The open PR for the current branch, when one exists. +4. Local pending changes. + +Do not mix modes or widen the resolved scope. + +#### Explicit paths, range, or revision + +Use the requested paths and revisions exactly. For paths without an explicit revision, use `HEAD` as the base and the complete working tree as the implementation, restricted to those paths. Include staged, unstaged, deleted, renamed, and untracked states. + +#### PR scope + +Run `gh pr view --json number,title,body,state,isDraft,baseRefName,baseRefOid,headRefOid,files`. Pin `baseRefName`, `baseRefOid`, and `headRefOid`, ensure both objects are available, and compute the merge base from the two pinned OIDs. Diff the merge base against `headRefOid`; never substitute a symbolic branch tip or current checkout. Exclude unrelated local edits unless the user explicitly requested them. + +#### Local scope + +Include committed current-branch changes relative to the resolved upstream merge base and the complete working tree. Use `git diff --find-renames HEAD` for staged and unstaged tracked changes. Use `git ls-files --others --exclude-standard` and read every returned untracked file as an addition. Include committed, staged, unstaged, deleted, renamed, and untracked states without counting the same change twice. + +#### Scope manifest + +Before analysis, freeze a scope manifest containing: + +```text +SCOPE_ID: +MODE: paths | range | revision | pr | local +REPOSITORY_ROOT: +BASE_SHA: +HEAD_SHA: +WORKTREE_INCLUDED: yes | no +CHANGED_PATHS_AND_STATES: +PR_NUMBER: +PR_BASE_REF: +PR_BASE_SHA: +PR_HEAD_SHA: ``` -If any command fails (no `origin/HEAD`, not a git repo, no commits ahead of base), stop and report what is missing rather than fabricating results. +Create stable `SCOPE_ID` and candidate identities from repository identity and pinned scope data, never from task ordering. If the manifest contains no changes, stop with exactly: -## Objective +`No reviewable changes found in the resolved scope.` -Perform a security-focused code review to identify HIGH-CONFIDENCE security vulnerabilities introduced by the changes. Focus only on security implications **newly added** by this branch. Do not comment on pre-existing security concerns. +### 2. Establish the implementation baseline -## Critical rules +For every security-relevant change, compare the current full implementation with its implementation baseline: the base version or `absent at base` for an added path. Examine the changed causal line, security-sensitive sinks and callers, configuration, tests, and deployment or workflow inputs. Record `introduced_by` with the exact scoped change that creates or exposes the vulnerability. Pre-existing concerns are not candidates. -1. **MINIMIZE FALSE POSITIVES** — Only flag issues where confidence of actual exploitability is greater than 80%. -2. **AVOID NOISE** — Skip theoretical issues, style concerns, or low-impact findings. -3. **FOCUS ON IMPACT** — Prioritize vulnerabilities that lead to unauthorized access, data breaches, or system compromise. -4. **EXCLUSIONS** — Do NOT report: - - Denial of Service (DoS), even if real - - Secrets / sensitive data on disk (handled separately) - - Rate-limiting / resource-exhaustion concerns +Context outside the scope may establish reachability or safety, but it cannot become a reviewed change or source of a finding. -## Categories to examine +### 3. Run category analysis -**Input validation:** SQL injection, command injection, XXE, template injection, NoSQL injection, path traversal. +Dispatch one independent analysis task per category in parallel when task dispatch is available: -**Authentication & authorization:** Auth bypass, privilege escalation, session management flaws, JWT vulnerabilities, authorization-logic bypasses. - -**Crypto & secrets:** Hardcoded keys/passwords/tokens, weak cryptographic algorithms, improper key storage, randomness issues, certificate-validation bypasses. - -**Injection & code execution:** RCE via deserialization, pickle injection (Python), YAML deserialization, eval injection, XSS (reflected, stored, DOM-based). - -**Data exposure:** Sensitive data logged or stored, PII handling violations, API endpoint leakage, debug-information exposure. - -Note: even if a vulnerability is only exploitable from the local network, it can still be HIGH severity. - -## Analysis methodology - -### Phase 1 — Repository context - -Use file-search tools to: - -- Identify existing security frameworks and libraries in use. -- Look for established secure-coding patterns in the codebase. -- Examine existing sanitization and validation patterns. -- Understand the project's threat model. - -### Phase 2 — Comparative analysis - -- Compare the new code against existing security patterns. -- Identify deviations from established secure practices. -- Look for inconsistent security implementations. -- Flag code that introduces new attack surfaces. - -### Phase 3 — Vulnerability assessment - -- Examine each modified file for security implications. -- Trace data flow from user inputs to sensitive operations. -- Look for privilege boundaries crossed unsafely. -- Identify injection points and unsafe deserialization. - -## Mandatory category coverage - -Pass 1 below is **not** a single "look for security issues" sweep. It is one parallel sub-task **per category**, dispatched independently. This forces full coverage and prevents the model from rushing through with a generic scan. - -You must dispatch the following sub-tasks in parallel, each with the diff and the category-specific instructions: - -| # | Category | Examples | +| # | Category | Required focus | |---|---|---| -| 1 | Input validation | SQL/command/XXE/template/NoSQL injection, path traversal, deserialization | -| 2 | Authentication & authorization | Auth bypass, privilege escalation, IDOR, session/JWT flaws | -| 3 | Crypto & secrets | Hardcoded keys, weak algorithms, improper key storage, randomness, cert validation | -| 4 | Code execution | RCE via deserialization, pickle/eval, YAML loaders, dynamic exec | -| 5 | Data exposure | Sensitive logging, PII handling, debug info, API leakage | -| 6 | Concurrency & state | TOCTOU, race conditions on auth checks, cache invalidation that affects sandbox/allowlist behavior | -| 7 | Trust boundaries | What inputs cross from untrusted to trusted? Are they validated at the boundary? Are sandbox / allowlist decisions cached past their validity? | +| 1 | Input validation and injection | SQL, command, XXE, template, NoSQL, path traversal, and XSS | +| 2 | Authentication and authorization | Auth bypass, privilege escalation, IDOR, sessions, and JWTs | +| 3 | Crypto and secrets | Credentials, algorithms, key storage, randomness, and certificate validation | +| 4 | Unsafe code execution and deserialization | RCE, pickle, unsafe YAML loaders, `eval`, and dynamic execution | +| 5 | Data exposure | Sensitive logging, PII, debug information, and API leakage | +| 6 | Concurrency and state | TOCTOU, authorization races, and stale sandbox or allowlist decisions | +| 7 | Trust boundaries | Untrusted inputs crossing privileged, tenant, process, workflow, or sandbox boundaries | -Each sub-task returns its own candidate list. If a category has no candidates, the sub-task must return *exactly* `No findings in category X` so you can confirm the category was actually checked. +Give each task the frozen scope manifest, assigned category, changed implementation, implementation baseline, and this policy. A category with no candidates returns `No findings in category X`. -## Three-stage filtering +Each category candidate must contain: -After pass 1's parallel category sub-tasks return, run **two more passes** before reporting. Do not skip stages even if a finding looks obvious. - -2. **Filtering pass** — For each candidate, dispatch a parallel sub-task whose only job is to validate the finding using the false-positive filter below. Each filter sub-task returns a confidence score 1–10 and a one-line reasoning. Drop anything below 8. - -3. **Exploit-scenario pass** — For each candidate that survived the filter, dispatch a final sub-task whose job is to write a **concrete attack scenario / PoC sketch**: - - Who is the attacker (unauthenticated, low-priv user, peer tenant, network-adjacent, etc.)? - - What input or action do they control? - - What is the exact request, payload, or sequence? - - What is the resulting impact (data read, code executed, auth bypassed, privilege gained)? - - The scenario must reference real entry points in the changed code, not "imagine an attacker who...". If no concrete attack path can be written, **drop the finding**. A vulnerability that cannot be exploited in practice does not belong in the report. - -Doing all three passes on a non-trivial PR will take multiple minutes. That is expected. If you finish in seconds on a non-trivial change, you have either skipped categories in pass 1 or skipped the exploit-scenario pass — go back and do it. - -### False-positive filter - -Do not run commands to reproduce vulnerabilities; read the code only. - -**Hard exclusions** — automatically exclude findings matching any of these patterns: - -1. Denial of Service / resource exhaustion attacks. -2. Secrets or credentials stored on disk if otherwise secured. -3. Rate-limiting concerns or service-overload scenarios. -4. Memory or CPU exhaustion issues. -5. Lack of input validation on non-security-critical fields without proven security impact. -6. Input-sanitization concerns for GitHub Action workflows unless clearly triggerable via untrusted input. -7. Lack of hardening measures. Code is not expected to implement all security best practices; only flag concrete vulnerabilities. -8. Theoretical race conditions or timing attacks. Only report a race condition if concretely problematic. -9. Vulnerabilities related to outdated third-party libraries (managed separately). -10. Memory-safety issues in memory-safe languages (Rust, Go, etc.). -11. Files that are only unit tests or only used during tests. -12. Log-spoofing concerns. Outputting unsanitized user input to logs is not a vulnerability. -13. SSRF that only controls the path. SSRF is a concern only if it can control host or protocol. -14. Including user-controlled content in AI system prompts is not a vulnerability. -15. Regex injection. -16. Regex DoS. -17. Insecure documentation. Do not flag findings in markdown or other docs. -18. Lack of audit logs. - -**Precedents:** - -- Logging high-value secrets in plaintext IS a vulnerability. Logging URLs is assumed safe. -- UUIDs can be assumed unguessable and do not need validation. -- Environment variables and CLI flags are trusted. Attacks that rely on controlling them are invalid. -- Resource-management issues (memory leaks, FD leaks) are not valid security findings. -- Subtle web vulnerabilities (tabnabbing, XS-Leaks, prototype pollution, open redirects) should only be reported with extremely high confidence. -- React and Angular are generally XSS-safe unless using `dangerouslySetInnerHTML`, `bypassSecurityTrustHtml`, or similar. -- Most vulnerabilities in GitHub Action workflows are not exploitable in practice. Validate with a concrete attack path. -- Include MEDIUM findings only if obvious and concrete. -- Most vulnerabilities in `.ipynb` notebooks are not exploitable. -- Logging non-PII data is not a vulnerability. -- Command injection in shell scripts is generally not exploitable since shell scripts rarely run with untrusted user input. Validate concretely. - -**Signal-quality criteria:** - -1. Is there a concrete, exploitable vulnerability with a clear attack path? -2. Is this a real security risk vs. theoretical best practice? -3. Are there specific code locations and reproduction steps? -4. Would a security team find this actionable? - -**Confidence score (1–10):** - -- 1–3: Low — likely false positive or noise. -- 4–6: Medium — needs investigation. -- 7–10: High — likely true vulnerability. - -## Output format - -Output your findings in markdown only. Each finding must include file, line number, severity, category, description, **the concrete exploit scenario from the third pass** (not a generic one), and fix recommendation. A finding without a concrete exploit scenario from the third pass should not appear in the report. - -Example: - -```markdown -# Vuln 1: XSS — `foo.py:42` - -* Severity: High -* Category: xss -* Description: User input from the `username` parameter is directly interpolated into HTML without escaping, allowing reflected XSS. -* Exploit Scenario: An attacker crafts a URL like `/bar?q=` to execute JavaScript in the victim's browser, enabling session hijacking. -* Recommendation: Use Flask's `escape()` or Jinja2 templates with auto-escaping enabled for all user inputs rendered in HTML. +```text +candidate_id: +category: +changed_location: +sink_location: +baseline_evidence: +introduced_by: +attacker: +controlled_input: +trust_boundary: +impact: +supporting_evidence: ``` -**Severity guidelines:** +### 4. Filter candidates -- **HIGH** — Directly exploitable: RCE, data breach, auth bypass. -- **MEDIUM** — Requires specific conditions but with significant impact. -- **LOW** — Defense-in-depth or lower-impact issues. Generally do not report unless concrete. +For each category candidate, dispatch an independent filter task with the same frozen manifest and policy. The result must repeat `candidate_id`, assign a confidence score from 1 through 10, state concise reasoning, and return `retain` or `reject`. -## Final reminder +Use one reporting threshold: at least 8/10 and at least 80% confidence. The bands are 1-3 low, 4-7 insufficient, and 8-10 reportable. A retained candidate must still pass exploit validation. -Better to miss some theoretical issues than flood the report with false positives. Each finding should be something a security engineer would confidently raise in a PR review. Your final reply must contain the markdown report and nothing else. +### 5. Validate exploit scenarios + +For each retained candidate, dispatch an independent exploit task. Its result must repeat `candidate_id` and establish from real code: + +- The attacker and access level. +- The exact input or action they control. +- The entry point, payload or sequence, changed causal line, and sensitive sink. +- The crossed trust boundary and resulting security impact. +- Why the implementation baseline was not vulnerable and `introduced_by` is accurate. + +Reject a candidate if no concrete reachable attack path can be established. Do not invent runtime state, entry points, or attacker control. + +### 6. Validate handoffs and recover coverage + +Validate every category, filter, and exploit result before consuming it. Reject mismatched `SCOPE_ID`, category, assignment, or `candidate_id`; missing fields; out-of-range scores; nonexistent locations; and absent introduction evidence. Preserve the record as unresolved until the missing analysis is recovered. + +Apply this bounded recovery contract independently at every stage: + +1. Preserve valid evidence and identify only missing or invalid work. +2. For a successful but incomplete or malformed response, resume the same child exactly once with the omissions named. +3. For a transient timeout, rate-limit, or transport failure, retry exactly once as a fresh dispatch. +4. For permission denial, unavailable tools, invalid requests, or deterministic failures, do not retry. Permission denial does not consume the transient-retry budget. +5. If parallel dispatch is unavailable or denied, continue unfinished work with serial children. +6. If individual task dispatch is unavailable or denied, or bounded recovery is exhausted, complete the missing checklist in the parent. + +Never silently discard a category or candidate. Never interpret a failed, blank, malformed, partial, or invalid response as no findings. + +Maintain a scope coverage ledger keyed by changed path. Record state, security relevance, baseline inspected, current implementation inspected, relevant callers, sinks, configuration, tests, deployment or workflow inputs, and completion status. + +Maintain an analysis coverage ledger keyed by category, `candidate_id` when applicable, and stage (`category | filter | exploit`). Record status, evidence, errors, resume and retry counts, disposition, and execution mode (`parallel | resumed | retried | serial | parent fallback`). + +### 7. Apply security policy + +Prioritize concrete vulnerabilities leading to unauthorized access, data disclosure, credential compromise, privilege gain, or code execution. Do not report style concerns, theoretical hardening opportunities, or findings without a demonstrated security impact. + +Hardcoded live or plausibly live credentials introduced in any scoped file are reportable, including source, configuration, tests, fixtures, and documentation. Demonstrably inert examples and secure references to runtime secret stores or encrypted or protected credential files are not findings merely because they exist on disk. Active credential disclosure overrides exclusions for tests, fixtures, and documentation. + +Environment variables and CLI flags are trust-boundary inputs, not automatically trusted. Trust requires proven trusted-operator control. Trace workflow, service, wrapper, and user-controlled process-launch sources before deciding whether an environment variable or flag is attacker controlled. + +Exclude these unless the changed code creates a concrete impact outside the excluded class: + +- Denial of service, resource exhaustion, rate limiting, memory leaks, or file-descriptor leaks. +- Outdated third-party dependencies, which are managed separately. +- Lack of input validation on non-security-critical fields. +- Lack of general hardening or audit logs. +- Theoretical races or timing attacks without a reachable harmful interleaving. +- Log spoofing, regex injection, regex denial of service, and user content merely appearing in an AI prompt. +- SSRF where the attacker controls only a path, not the host or protocol. +- Memory-safety speculation in memory-safe languages. + +Use these precedents carefully: + +- Plaintext logging of high-value secrets is reportable; ordinary URL logging is assumed safe absent sensitive query data. +- UUIDs may be treated as unguessable unless changed code weakens their generation or exposes them. +- React and Angular escape content by default; require an unsafe rendering path such as `dangerouslySetInnerHTML` or `bypassSecurityTrustHtml` for XSS. +- GitHub workflow and shell-script inputs require a concrete trace from attacker control to execution. +- Local-network exploitation remains valid when attacker access and impact are concrete. + +### 8. Gate and format terminal output + +Every reported finding includes `candidate_id`, file and line, severity, category, confidence, description, baseline evidence, concrete exploit scenario, and fix recommendation. Findings appear before coverage details. + +There are exactly three terminal outcomes: + +1. Complete: report validated findings and completed ledgers. If there are no findings, emit exactly `No security vulnerabilities found in the reviewed scope.` +2. Empty scope: emit exactly `No reviewable changes found in the resolved scope.` +3. Incomplete: emit exactly `Security review incomplete; absence of findings is not established.`, followed by validated findings, both ledgers, failures, recovery attempts, and unvalidated leads. + +The no-findings outcome is allowed only for complete scope and analysis coverage ledgers with a final disposition for every candidate. A posting failure does not suppress or replace terminal output. + +### 9. Post to a PR only when requested + +Post only when the user requested PR posting and the resolved scope identifies a PR. Otherwise return terminal output without posting. Use one consolidated PR comment containing the head SHA and this marker: + +```html + +``` + +Determine the authenticated author and query existing PR comments. Match both marker and authenticated author. Update an existing matching comment rather than create a duplicate; create a comment only when no match exists. If an existing comment already has identical content and head SHA, do nothing. + +After an ambiguous posting failure, query again before any mutation. If the comment now exists, update or leave it unchanged as appropriate; otherwise retry creation once. This keeps posting idempotent for repeated runs on one PR head. diff --git a/tests/security-review.test.mjs b/tests/security-review.test.mjs new file mode 100644 index 0000000..f41c367 --- /dev/null +++ b/tests/security-review.test.mjs @@ -0,0 +1,60 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +const REPO = process.env.REPO || process.cwd(); +const skill = readFileSync(join(REPO, "skills/security-review/SKILL.md"), "utf8"); + +test("security-review resolves explicit, PR, and complete local scopes", () => { + assert.match(skill, /explicit (user|requested).*scope.*takes precedence/is); + for (const field of ["baseRefName", "baseRefOid", "headRefOid"]) assert.ok(skill.includes(field)); + assert.match(skill, /git diff --find-renames HEAD/); + assert.match(skill, /git ls-files --others --exclude-standard/); + assert.match(skill, /scope manifest/i); + assert.ok(skill.includes("No reviewable changes found in the resolved scope.")); +}); + +test("security-review requires baseline evidence and validated handoffs", () => { + assert.match(skill, /implementation baseline/i); + assert.match(skill, /absent at base/i); + for (const field of ["candidate_id", "changed_location", "baseline_evidence", "controlled_input", "trust_boundary"]) assert.ok(skill.includes(field)); + assert.match(skill, /reject.*mismatched|mismatched.*reject/is); + assert.match(skill, /missing fields/i); + assert.match(skill, /out-of-range/i); +}); + +test("security-review uses bounded recovery and coverage ledgers", () => { + assert.match(skill, /resume.*exactly once/is); + assert.match(skill, /transient.*retry.*exactly once/is); + assert.match(skill, /parallel.*denied.*serial/is); + assert.match(skill, /task dispatch.*(unavailable|denied).*parent/is); + assert.match(skill, /scope coverage ledger/i); + assert.match(skill, /analysis coverage ledger/i); +}); + +test("security-review reconciles confidence, secrets, and runtime inputs", () => { + assert.match(skill, /at least 8\/10/i); + assert.match(skill, /at least 80%/i); + assert.doesNotMatch(skill, /greater than 80%/i); + assert.match(skill, /hardcoded.*credentials.*reportable/is); + assert.match(skill, /tests.*fixtures.*documentation/is); + assert.match(skill, /demonstrably inert/i); + assert.ok(skill.includes("Environment variables and CLI flags are trust-boundary inputs, not automatically trusted.")); +}); + +test("security-review defines complete, empty, and incomplete outcomes", () => { + assert.ok(skill.includes("No security vulnerabilities found in the reviewed scope.")); + assert.ok(skill.includes("No reviewable changes found in the resolved scope.")); + assert.ok(skill.includes("Security review incomplete; absence of findings is not established.")); + assert.match(skill, /no-findings.*only.*complete/is); +}); + +test("security-review posting is requested and idempotent", () => { + assert.match(skill, /only when.*requested/is); + assert.ok(skill.includes("")); + assert.match(skill, /authenticated author/i); + assert.match(skill, /update.*existing.*rather than.*creat/is); + assert.match(skill, /head SHA/i); + assert.match(skill, /ambiguous.*failure.*query again/is); +});