fix(test): harden red-team assertions and mutation checks (#856)

Reject invalid audit output and CLI errors before accepting absent findings, including integrity and traversal checks. Require completed mutation runs to fail the single and batch update gates rather than accepting unrelated failures.

Build gate mutations with a private Go overlay and binary so failures cannot leave source or the shared binary weakened. Add regression coverage for assertions, signal checks, and source/binary isolation to every red-team Make target.
This commit is contained in:
Willie
2026-11-11 21:40:44 +08:00
committed by GitHub
parent 828ba93f27
commit eb3ca528fc
7 changed files with 190 additions and 94 deletions
+9 -3
View File
@@ -11,6 +11,7 @@ help:
@echo " make test-docker # docker offline sandbox (build + unit + integration)"
@echo " make test-docker-online # docker online install/update tests"
@echo " make test-redteam # red team supply-chain security tests"
@echo " make test-redteam-harness # regression tests for red team assertions and signal checks"
@echo " make test-redteam-signal # verify red team test fails on intentional mutation"
@echo " make test-redteam-rules-signal # verify red team test fails when critical builtin rules are disabled"
@echo " make playground # start playground + enter shell (one step)"
@@ -71,13 +72,18 @@ test-docker:
test-docker-online:
./scripts/test_docker_online.sh
test-redteam: build
.PHONY: test-redteam-harness
test-redteam-harness:
bash ./scripts/test_redteam_harness.sh
test-redteam: build test-redteam-harness
./scripts/red_team_test.sh
test-redteam-signal:
test-redteam-signal: test-redteam-harness
./scripts/test_redteam_signal.sh
test-redteam-rules-signal:
test-redteam-rules-signal: test-redteam-harness
./scripts/test_redteam_rules_signal.sh
playground:
+27 -8
View File
@@ -147,17 +147,36 @@ assert_finding() {
fi
}
# Assert audit JSON output does NOT contain a finding with given pattern
# Usage: assert_no_finding <label> <json_output> <pattern>
assert_no_finding() {
# Assert a successful audit has no findings matching a jq predicate.
# Null findings is the CLI's valid representation of an empty findings slice.
assert_no_findings_matching() {
local label="$1"
local json="$2"
local pattern="$3"
if echo "$json" | jq -e ".results[0].findings[] | select(.pattern == \"$pattern\")" >/dev/null 2>&1; then
fail "$label — should NOT have $pattern finding"
else
pass "$label"
local predicate="$3"
if [ "$SS_EXIT" -ne 0 ]; then
fail "$label — audit exited $SS_EXIT"
return
fi
if ! jq -se '
length == 1 and (.[0] |
(.results | type == "array" and length > 0) and
(.results[0] | type == "object" and has("findings")) and
(.results[0].findings | . == null or type == "array"))
' <<< "$json" >/dev/null 2>&1; then
fail "$label — invalid audit JSON or missing findings"
return
fi
if jq -e "[.results[0].findings[]? | select($predicate)] | length == 0" <<< "$json" >/dev/null 2>&1; then
pass "$label"
else
fail "$label — unexpected finding or invalid finding data"
fi
}
# Assert audit JSON output does NOT contain a finding with given pattern.
# Usage: assert_no_finding <label> <json_output> <pattern>
assert_no_finding() {
assert_no_findings_matching "$1" "$2" ".pattern == \"$3\""
}
# ── Git repo helpers ───────────────────────────────────────────────
+2 -7
View File
@@ -17,13 +17,8 @@ write_store_meta "$SOURCE_DIR" "integrity-skill" \
"{\"source\":\"test/integrity\",\"type\":\"local\",\"installed_at\":\"2026-01-01T00:00:00Z\",\"file_hashes\":{\"SKILL.md\":\"sha256:$SKILL_HASH\"}}"
ss_capture audit integrity-skill --format json
HAS_INTEGRITY=false
echo "$SS_OUTPUT" | jq -e '.results[0].findings[] | select(.pattern | startswith("content-"))' >/dev/null 2>&1 && HAS_INTEGRITY=true
if [ "$HAS_INTEGRITY" = false ]; then
pass "TC-25: clean skill with correct hashes passes"
else
fail "TC-25: unexpected integrity finding"
fi
assert_no_findings_matching "TC-25: clean skill with correct hashes passes" "$SS_OUTPUT" \
'.pattern | startswith("content-")'
# ── TC-26: Tamper file → content-tampered (MEDIUM) ────────────
+2 -8
View File
@@ -17,14 +17,8 @@ write_store_meta "$SOURCE_DIR" "traversal-skill" \
"{\"source\":\"test/traversal\",\"type\":\"local\",\"installed_at\":\"2026-01-01T00:00:00Z\",\"file_hashes\":{\"SKILL.md\":\"sha256:$TRAVERSAL_HASH\",\"../../../secret.txt\":\"sha256:0000\",\"../../secret.txt\":\"sha256:0000\",\"/etc/passwd\":\"sha256:0000\"}}"
ss_capture audit traversal-skill --format json
if echo "$SS_OUTPUT" | jq -e '
.results[0].findings[] |
select(.file == "../../../secret.txt" or .file == "../../secret.txt" or .file == "/etc/passwd")
' >/dev/null 2>&1; then
fail "TC-31: traversal/absolute keys should not produce findings"
else
pass "TC-31: traversal/absolute hash keys ignored"
fi
assert_no_findings_matching "TC-31: traversal/absolute hash keys ignored" "$SS_OUTPUT" \
'.file == "../../../secret.txt" or .file == "../../secret.txt" or .file == "/etc/passwd"'
# -- TC-32: symlink to external file is surfaced -----------------
+108
View File
@@ -0,0 +1,108 @@
#!/bin/bash
# Regression tests for red-team assertions and mutation-run isolation.
# Run inside the devcontainer: bash scripts/test_redteam_harness.sh
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "$SCRIPT_DIR/red_team/_helpers.sh"
check_absence() {
local label="$1" json="$2" exit_code="$3" expected="$4"
TESTS_PASSED=0
TESTS_FAILED=0
SS_EXIT="$exit_code"
assert_no_finding "$label" "$json" "prompt-injection" >/dev/null
if [ "$TESTS_FAILED" -ne "$expected" ]; then
echo "FAIL: $label — expected $expected failures, got $TESTS_FAILED"
exit 1
fi
}
check_absence "empty findings" '{"results":[{"findings":[]}]}' 0 0
check_absence "null findings" '{"results":[{"findings":null}]}' 0 0
check_absence "finding present" '{"results":[{"findings":[{"pattern":"prompt-injection"}]}]}' 0 1
check_absence "empty output" '' 0 1
check_absence "malformed JSON" 'not JSON' 0 1
check_absence "missing result" '{}' 0 1
check_absence "empty results" '{"results":[]}' 0 1
check_absence "multiple documents" '{"results":[{"findings":[]}]} {}' 0 1
check_absence "missing findings" '{"results":[{}]}' 0 1
check_absence "invalid findings" '{"results":[{"findings":"broken"}]}' 0 1
check_absence "CLI failure" '{"results":[{"findings":[]}]}' 2 1
echo "PASS: negative assertions reject invalid output and CLI failures"
# Use a disposable project with fake build/test commands, never mutate real source.
TEST_ROOT="$(mktemp -d)"
trap 'rm -rf "$TEST_ROOT"' EXIT
mkdir -p "$TEST_ROOT/project/scripts" "$TEST_ROOT/project/internal/install" "$TEST_ROOT/project/bin" "$TEST_ROOT/tools"
cp "$SCRIPT_DIR/test_redteam_signal.sh" "$SCRIPT_DIR/test_redteam_rules_signal.sh" "$TEST_ROOT/project/scripts/"
printf '%s\n' 'if !scanResult.HasSeverityAtOrAbove(threshold) {' > "$TEST_ROOT/project/internal/install/install_audit.go"
printf '%s\n' 'original binary' > "$TEST_ROOT/project/bin/skillshare"
cp "$TEST_ROOT/project/internal/install/install_audit.go" "$TEST_ROOT/original.go"
cp "$TEST_ROOT/project/bin/skillshare" "$TEST_ROOT/original.bin"
cat > "$TEST_ROOT/tools/go" <<'EOF'
#!/bin/bash
set -euo pipefail
printf '%s\n' "$*" >> "$BUILD_LOG"
while [ "$#" -gt 0 ]; do
if [ "$1" = -o ]; then
printf '%s\n' 'test binary' > "$2"
break
fi
shift
done
EOF
cat > "$TEST_ROOT/tools/make" <<'EOF'
#!/bin/bash
exit 0
EOF
cat > "$TEST_ROOT/project/scripts/red_team_test.sh" <<'EOF'
#!/bin/bash
set -euo pipefail
if [ ! -f "$RUN_MARKER" ]; then
touch "$RUN_MARKER"
exit 0
fi
case "$RUN_MODE" in
expected)
printf '\033[31mFAIL\033[0m: TC-21a: update exits non-zero\n'
echo 'FAIL: TC-29: update --all exits non-zero with malicious repo'
echo ' Red Team Test Results'
exit 1
;;
unrelated) echo 'tool missing'; exit 2 ;;
partial) echo 'FAIL: TC-21a: update exits non-zero'; exit 1 ;;
unexpected-pass) exit 0 ;;
esac
EOF
# Fixture permissions are private to this disposable test directory.
chmod +x "$TEST_ROOT/tools/go" "$TEST_ROOT/tools/make" "$TEST_ROOT/project/scripts/red_team_test.sh" "$TEST_ROOT/project/bin/skillshare"
for script in test_redteam_signal.sh test_redteam_rules_signal.sh; do
for mode in expected unrelated partial unexpected-pass; do
marker="$TEST_ROOT/$script-$mode.marker"
if [ "$script" = test_redteam_rules_signal.sh ]; then
touch "$marker"
fi
status=0
PATH="$TEST_ROOT/tools:$PATH" BUILD_LOG="$TEST_ROOT/build.log" \
RUN_MARKER="$marker" RUN_MODE="$mode" \
bash "$TEST_ROOT/project/scripts/$script" > "$TEST_ROOT/$mode.log" 2>&1 || status=$?
if [ "$mode" = expected ]; then
if [ "$status" -ne 0 ]; then
cat "$TEST_ROOT/$mode.log"
echo 'FAIL: expected mutation signal rejected'
exit 1
fi
elif [ "$status" -eq 0 ]; then
echo "FAIL: $mode incorrectly accepted as mutation signal"
exit 1
fi
cmp "$TEST_ROOT/original.go" "$TEST_ROOT/project/internal/install/install_audit.go"
cmp "$TEST_ROOT/original.bin" "$TEST_ROOT/project/bin/skillshare"
done
done
echo 'PASS: mutation checks require gate failures and leave source/shared binary unchanged'
+8 -11
View File
@@ -62,19 +62,16 @@ printf "%s\n" "$MUTATED_OUTPUT"
SANITIZED_OUTPUT="$(printf "%s\n" "$MUTATED_OUTPUT" | perl -pe 's/\e\[[0-9;]*m//g')"
if [ "$MUTATED_EXIT" -eq 0 ]; then
echo "ERROR: rules-disabled run unexpectedly passed; signal check failed"
if [ "$MUTATED_EXIT" -ne 1 ]; then
echo "ERROR: expected completed test failure (exit=1), got exit=$MUTATED_EXIT"
exit 1
fi
if ! printf "%s\n" "$SANITIZED_OUTPUT" | grep -qF "FAIL: TC-21a"; then
echo "ERROR: expected TC-21a failure was not observed"
exit 1
fi
if ! printf "%s\n" "$SANITIZED_OUTPUT" | grep -qF "FAIL: TC-29"; then
echo "ERROR: expected TC-29 failure was not observed"
exit 1
fi
for expected in 'FAIL: TC-21a:' 'FAIL: TC-29:' 'Red Team Test Results'; do
if ! grep -qF "$expected" <<< "$SANITIZED_OUTPUT"; then
echo "ERROR: expected mutation signal not observed: $expected"
exit 1
fi
done
echo "==> [3/3] PASS: rules-level signal verified (baseline pass + rules-disabled fail)"
+34 -57
View File
@@ -2,81 +2,58 @@
# Verify red_team_test.sh has real detection signal.
#
# Strategy:
# 1) Run baseline red team test (must pass).
# 2) Introduce a temporary mutation that bypasses HIGH/CRITICAL rollback gate.
# 3) Rebuild binary and run red team test again (must fail).
# 4) Restore original source and rebuild.
# 1) Build a private baseline binary and run red team (must pass).
# 2) Build with a Go overlay that bypasses the HIGH/CRITICAL gate.
# 3) Require the mutated run to fail the single and batch update gate tests.
# Source and bin/skillshare are never modified, even on failure or interruption.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
GATE_GO="$PROJECT_ROOT/internal/install/install_audit.go"
BACKUP_FILE=""
BACKUP_READY=false
RESTORED=false
backup_source() {
BACKUP_FILE="$(mktemp "${TMPDIR:-/tmp}/install_audit.go.backup.XXXXXX")"
cp "$GATE_GO" "$BACKUP_FILE"
BACKUP_READY=true
}
restore_source() {
if [ "$RESTORED" = false ] && [ "$BACKUP_READY" = true ] && [ -f "$BACKUP_FILE" ]; then
cp "$BACKUP_FILE" "$GATE_GO"
RESTORED=true
fi
if [ -n "$BACKUP_FILE" ]; then
rm -f "$BACKUP_FILE"
fi
}
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/redteam-signal.XXXXXX")"
TEST_BIN="$TMP_DIR/skillshare"
cleanup() {
restore_source
rm -rf "$TMP_DIR"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
echo "==> [1/4] Baseline: running make test-redteam (should pass)"
(cd "$PROJECT_ROOT" && make test-redteam)
echo "==> [1/3] Baseline: build private binary and run red team (should pass)"
(cd "$PROJECT_ROOT" && go build -o "$TEST_BIN" ./cmd/skillshare)
SKILLSHARE_TEST_BINARY="$TEST_BIN" "$SCRIPT_DIR/red_team_test.sh"
echo "==> [2/4] Injecting mutation: bypass rollback on HIGH/CRITICAL findings"
backup_source
# install and update both decide through AuditGate.Run; skip its threshold check.
perl -0pi -e 's@if !scanResult\.HasSeverityAtOrAbove\(threshold\) \{@if true { // MUTATION: bypass rollback gate@g' "$GATE_GO"
if command -v rg >/dev/null 2>&1; then
MUTATION_COUNT="$(rg -c "MUTATION: bypass rollback gate" "$GATE_GO" || true)"
else
MUTATION_COUNT="$(grep -c "MUTATION: bypass rollback gate" "$GATE_GO" || true)"
fi
MUTATION_COUNT="${MUTATION_COUNT:-0}"
if ! [[ "$MUTATION_COUNT" =~ ^[0-9]+$ ]]; then
echo "ERROR: mutation count is not numeric: $MUTATION_COUNT"
exit 2
fi
echo "==> [2/3] Build mutation: bypass HIGH/CRITICAL gate through a Go overlay"
# Keep the mutation in a private copy, not the working tree.
perl -0pe 's@if !scanResult\.HasSeverityAtOrAbove\(threshold\) \{@if true { // MUTATION: bypass rollback gate@g' \
"$GATE_GO" > "$TMP_DIR/install_audit.go"
MUTATION_COUNT="$(grep -c 'MUTATION: bypass rollback gate' "$TMP_DIR/install_audit.go" || true)"
if [ "$MUTATION_COUNT" -ne 1 ]; then
echo "ERROR: expected to inject 1 mutation point, got $MUTATION_COUNT"
exit 2
fi
jq -n --arg original "$GATE_GO" --arg mutated "$TMP_DIR/install_audit.go" \
'{Replace: {($original): $mutated}}' > "$TMP_DIR/overlay.json"
(cd "$PROJECT_ROOT" && go build -overlay "$TMP_DIR/overlay.json" -o "$TEST_BIN" ./cmd/skillshare)
echo "==> [3/4] Rebuild mutated binary and run red team (should fail)"
(cd "$PROJECT_ROOT" && go build -o bin/skillshare ./cmd/skillshare)
echo "==> [3/3] Run mutated red team (must fail the single and batch update gates)"
MUTATED_EXIT=0
SKILLSHARE_TEST_BINARY="$TEST_BIN" "$SCRIPT_DIR/red_team_test.sh" > "$TMP_DIR/output.log" 2>&1 || MUTATED_EXIT=$?
cat "$TMP_DIR/output.log"
perl -pe 's/\e\[[0-9;]*m//g' "$TMP_DIR/output.log" > "$TMP_DIR/plain.log"
set +e
(cd "$PROJECT_ROOT" && ./scripts/red_team_test.sh)
MUTATED_EXIT=$?
set -e
if [ "$MUTATED_EXIT" -eq 0 ]; then
echo "ERROR: mutated run unexpectedly passed; red team signal check failed"
if [ "$MUTATED_EXIT" -ne 1 ]; then
echo "ERROR: expected completed test failure (exit=1), got exit=$MUTATED_EXIT"
exit 1
fi
echo "PASS: mutated run failed as expected (exit=$MUTATED_EXIT)"
for expected in 'FAIL: TC-21a:' 'FAIL: TC-29:' 'Red Team Test Results'; do
if ! grep -qF "$expected" "$TMP_DIR/plain.log"; then
echo "ERROR: expected mutation signal not observed: $expected"
exit 1
fi
done
echo "==> [4/4] Restoring original source and rebuilding"
restore_source
(cd "$PROJECT_ROOT" && go build -o bin/skillshare ./cmd/skillshare)
echo "PASS: test-redteam signal verified (baseline pass + mutation fail)"
echo "PASS: test-redteam signal verified (baseline pass + expected gate failures)"