mirror of
https://github.com/runkids/skillshare.git
synced 2026-11-18 18:04:17 +00:00
fix(test): harden red-team assertions and mutation checks (#856)
Reject invalid audit output and CLI errors before accepting absent findings, including integrity and traversal checks. Require completed mutation runs to fail the single and batch update gates rather than accepting unrelated failures. Build gate mutations with a private Go overlay and binary so failures cannot leave source or the shared binary weakened. Add regression coverage for assertions, signal checks, and source/binary isolation to every red-team Make target.
This commit is contained in:
@@ -11,6 +11,7 @@ help:
|
||||
@echo " make test-docker # docker offline sandbox (build + unit + integration)"
|
||||
@echo " make test-docker-online # docker online install/update tests"
|
||||
@echo " make test-redteam # red team supply-chain security tests"
|
||||
@echo " make test-redteam-harness # regression tests for red team assertions and signal checks"
|
||||
@echo " make test-redteam-signal # verify red team test fails on intentional mutation"
|
||||
@echo " make test-redteam-rules-signal # verify red team test fails when critical builtin rules are disabled"
|
||||
@echo " make playground # start playground + enter shell (one step)"
|
||||
@@ -71,13 +72,18 @@ test-docker:
|
||||
test-docker-online:
|
||||
./scripts/test_docker_online.sh
|
||||
|
||||
test-redteam: build
|
||||
.PHONY: test-redteam-harness
|
||||
|
||||
test-redteam-harness:
|
||||
bash ./scripts/test_redteam_harness.sh
|
||||
|
||||
test-redteam: build test-redteam-harness
|
||||
./scripts/red_team_test.sh
|
||||
|
||||
test-redteam-signal:
|
||||
test-redteam-signal: test-redteam-harness
|
||||
./scripts/test_redteam_signal.sh
|
||||
|
||||
test-redteam-rules-signal:
|
||||
test-redteam-rules-signal: test-redteam-harness
|
||||
./scripts/test_redteam_rules_signal.sh
|
||||
|
||||
playground:
|
||||
|
||||
@@ -147,17 +147,36 @@ assert_finding() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Assert audit JSON output does NOT contain a finding with given pattern
|
||||
# Usage: assert_no_finding <label> <json_output> <pattern>
|
||||
assert_no_finding() {
|
||||
# Assert a successful audit has no findings matching a jq predicate.
|
||||
# Null findings is the CLI's valid representation of an empty findings slice.
|
||||
assert_no_findings_matching() {
|
||||
local label="$1"
|
||||
local json="$2"
|
||||
local pattern="$3"
|
||||
if echo "$json" | jq -e ".results[0].findings[] | select(.pattern == \"$pattern\")" >/dev/null 2>&1; then
|
||||
fail "$label — should NOT have $pattern finding"
|
||||
else
|
||||
pass "$label"
|
||||
local predicate="$3"
|
||||
if [ "$SS_EXIT" -ne 0 ]; then
|
||||
fail "$label — audit exited $SS_EXIT"
|
||||
return
|
||||
fi
|
||||
if ! jq -se '
|
||||
length == 1 and (.[0] |
|
||||
(.results | type == "array" and length > 0) and
|
||||
(.results[0] | type == "object" and has("findings")) and
|
||||
(.results[0].findings | . == null or type == "array"))
|
||||
' <<< "$json" >/dev/null 2>&1; then
|
||||
fail "$label — invalid audit JSON or missing findings"
|
||||
return
|
||||
fi
|
||||
if jq -e "[.results[0].findings[]? | select($predicate)] | length == 0" <<< "$json" >/dev/null 2>&1; then
|
||||
pass "$label"
|
||||
else
|
||||
fail "$label — unexpected finding or invalid finding data"
|
||||
fi
|
||||
}
|
||||
|
||||
# Assert audit JSON output does NOT contain a finding with given pattern.
|
||||
# Usage: assert_no_finding <label> <json_output> <pattern>
|
||||
assert_no_finding() {
|
||||
assert_no_findings_matching "$1" "$2" ".pattern == \"$3\""
|
||||
}
|
||||
|
||||
# ── Git repo helpers ───────────────────────────────────────────────
|
||||
|
||||
@@ -17,13 +17,8 @@ write_store_meta "$SOURCE_DIR" "integrity-skill" \
|
||||
"{\"source\":\"test/integrity\",\"type\":\"local\",\"installed_at\":\"2026-01-01T00:00:00Z\",\"file_hashes\":{\"SKILL.md\":\"sha256:$SKILL_HASH\"}}"
|
||||
|
||||
ss_capture audit integrity-skill --format json
|
||||
HAS_INTEGRITY=false
|
||||
echo "$SS_OUTPUT" | jq -e '.results[0].findings[] | select(.pattern | startswith("content-"))' >/dev/null 2>&1 && HAS_INTEGRITY=true
|
||||
if [ "$HAS_INTEGRITY" = false ]; then
|
||||
pass "TC-25: clean skill with correct hashes passes"
|
||||
else
|
||||
fail "TC-25: unexpected integrity finding"
|
||||
fi
|
||||
assert_no_findings_matching "TC-25: clean skill with correct hashes passes" "$SS_OUTPUT" \
|
||||
'.pattern | startswith("content-")'
|
||||
|
||||
# ── TC-26: Tamper file → content-tampered (MEDIUM) ────────────
|
||||
|
||||
|
||||
@@ -17,14 +17,8 @@ write_store_meta "$SOURCE_DIR" "traversal-skill" \
|
||||
"{\"source\":\"test/traversal\",\"type\":\"local\",\"installed_at\":\"2026-01-01T00:00:00Z\",\"file_hashes\":{\"SKILL.md\":\"sha256:$TRAVERSAL_HASH\",\"../../../secret.txt\":\"sha256:0000\",\"../../secret.txt\":\"sha256:0000\",\"/etc/passwd\":\"sha256:0000\"}}"
|
||||
|
||||
ss_capture audit traversal-skill --format json
|
||||
if echo "$SS_OUTPUT" | jq -e '
|
||||
.results[0].findings[] |
|
||||
select(.file == "../../../secret.txt" or .file == "../../secret.txt" or .file == "/etc/passwd")
|
||||
' >/dev/null 2>&1; then
|
||||
fail "TC-31: traversal/absolute keys should not produce findings"
|
||||
else
|
||||
pass "TC-31: traversal/absolute hash keys ignored"
|
||||
fi
|
||||
assert_no_findings_matching "TC-31: traversal/absolute hash keys ignored" "$SS_OUTPUT" \
|
||||
'.file == "../../../secret.txt" or .file == "../../secret.txt" or .file == "/etc/passwd"'
|
||||
|
||||
# -- TC-32: symlink to external file is surfaced -----------------
|
||||
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/bin/bash
|
||||
# Regression tests for red-team assertions and mutation-run isolation.
|
||||
# Run inside the devcontainer: bash scripts/test_redteam_harness.sh
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "$SCRIPT_DIR/red_team/_helpers.sh"
|
||||
|
||||
check_absence() {
|
||||
local label="$1" json="$2" exit_code="$3" expected="$4"
|
||||
TESTS_PASSED=0
|
||||
TESTS_FAILED=0
|
||||
SS_EXIT="$exit_code"
|
||||
assert_no_finding "$label" "$json" "prompt-injection" >/dev/null
|
||||
if [ "$TESTS_FAILED" -ne "$expected" ]; then
|
||||
echo "FAIL: $label — expected $expected failures, got $TESTS_FAILED"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
check_absence "empty findings" '{"results":[{"findings":[]}]}' 0 0
|
||||
check_absence "null findings" '{"results":[{"findings":null}]}' 0 0
|
||||
check_absence "finding present" '{"results":[{"findings":[{"pattern":"prompt-injection"}]}]}' 0 1
|
||||
check_absence "empty output" '' 0 1
|
||||
check_absence "malformed JSON" 'not JSON' 0 1
|
||||
check_absence "missing result" '{}' 0 1
|
||||
check_absence "empty results" '{"results":[]}' 0 1
|
||||
check_absence "multiple documents" '{"results":[{"findings":[]}]} {}' 0 1
|
||||
check_absence "missing findings" '{"results":[{}]}' 0 1
|
||||
check_absence "invalid findings" '{"results":[{"findings":"broken"}]}' 0 1
|
||||
check_absence "CLI failure" '{"results":[{"findings":[]}]}' 2 1
|
||||
|
||||
echo "PASS: negative assertions reject invalid output and CLI failures"
|
||||
|
||||
# Use a disposable project with fake build/test commands, never mutate real source.
|
||||
TEST_ROOT="$(mktemp -d)"
|
||||
trap 'rm -rf "$TEST_ROOT"' EXIT
|
||||
mkdir -p "$TEST_ROOT/project/scripts" "$TEST_ROOT/project/internal/install" "$TEST_ROOT/project/bin" "$TEST_ROOT/tools"
|
||||
cp "$SCRIPT_DIR/test_redteam_signal.sh" "$SCRIPT_DIR/test_redteam_rules_signal.sh" "$TEST_ROOT/project/scripts/"
|
||||
printf '%s\n' 'if !scanResult.HasSeverityAtOrAbove(threshold) {' > "$TEST_ROOT/project/internal/install/install_audit.go"
|
||||
printf '%s\n' 'original binary' > "$TEST_ROOT/project/bin/skillshare"
|
||||
cp "$TEST_ROOT/project/internal/install/install_audit.go" "$TEST_ROOT/original.go"
|
||||
cp "$TEST_ROOT/project/bin/skillshare" "$TEST_ROOT/original.bin"
|
||||
|
||||
cat > "$TEST_ROOT/tools/go" <<'EOF'
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
printf '%s\n' "$*" >> "$BUILD_LOG"
|
||||
while [ "$#" -gt 0 ]; do
|
||||
if [ "$1" = -o ]; then
|
||||
printf '%s\n' 'test binary' > "$2"
|
||||
break
|
||||
fi
|
||||
shift
|
||||
done
|
||||
EOF
|
||||
cat > "$TEST_ROOT/tools/make" <<'EOF'
|
||||
#!/bin/bash
|
||||
exit 0
|
||||
EOF
|
||||
cat > "$TEST_ROOT/project/scripts/red_team_test.sh" <<'EOF'
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
if [ ! -f "$RUN_MARKER" ]; then
|
||||
touch "$RUN_MARKER"
|
||||
exit 0
|
||||
fi
|
||||
case "$RUN_MODE" in
|
||||
expected)
|
||||
printf '\033[31mFAIL\033[0m: TC-21a: update exits non-zero\n'
|
||||
echo 'FAIL: TC-29: update --all exits non-zero with malicious repo'
|
||||
echo ' Red Team Test Results'
|
||||
exit 1
|
||||
;;
|
||||
unrelated) echo 'tool missing'; exit 2 ;;
|
||||
partial) echo 'FAIL: TC-21a: update exits non-zero'; exit 1 ;;
|
||||
unexpected-pass) exit 0 ;;
|
||||
esac
|
||||
EOF
|
||||
# Fixture permissions are private to this disposable test directory.
|
||||
chmod +x "$TEST_ROOT/tools/go" "$TEST_ROOT/tools/make" "$TEST_ROOT/project/scripts/red_team_test.sh" "$TEST_ROOT/project/bin/skillshare"
|
||||
|
||||
for script in test_redteam_signal.sh test_redteam_rules_signal.sh; do
|
||||
for mode in expected unrelated partial unexpected-pass; do
|
||||
marker="$TEST_ROOT/$script-$mode.marker"
|
||||
if [ "$script" = test_redteam_rules_signal.sh ]; then
|
||||
touch "$marker"
|
||||
fi
|
||||
status=0
|
||||
PATH="$TEST_ROOT/tools:$PATH" BUILD_LOG="$TEST_ROOT/build.log" \
|
||||
RUN_MARKER="$marker" RUN_MODE="$mode" \
|
||||
bash "$TEST_ROOT/project/scripts/$script" > "$TEST_ROOT/$mode.log" 2>&1 || status=$?
|
||||
if [ "$mode" = expected ]; then
|
||||
if [ "$status" -ne 0 ]; then
|
||||
cat "$TEST_ROOT/$mode.log"
|
||||
echo 'FAIL: expected mutation signal rejected'
|
||||
exit 1
|
||||
fi
|
||||
elif [ "$status" -eq 0 ]; then
|
||||
echo "FAIL: $mode incorrectly accepted as mutation signal"
|
||||
exit 1
|
||||
fi
|
||||
cmp "$TEST_ROOT/original.go" "$TEST_ROOT/project/internal/install/install_audit.go"
|
||||
cmp "$TEST_ROOT/original.bin" "$TEST_ROOT/project/bin/skillshare"
|
||||
done
|
||||
done
|
||||
|
||||
echo 'PASS: mutation checks require gate failures and leave source/shared binary unchanged'
|
||||
@@ -62,19 +62,16 @@ printf "%s\n" "$MUTATED_OUTPUT"
|
||||
|
||||
SANITIZED_OUTPUT="$(printf "%s\n" "$MUTATED_OUTPUT" | perl -pe 's/\e\[[0-9;]*m//g')"
|
||||
|
||||
if [ "$MUTATED_EXIT" -eq 0 ]; then
|
||||
echo "ERROR: rules-disabled run unexpectedly passed; signal check failed"
|
||||
if [ "$MUTATED_EXIT" -ne 1 ]; then
|
||||
echo "ERROR: expected completed test failure (exit=1), got exit=$MUTATED_EXIT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! printf "%s\n" "$SANITIZED_OUTPUT" | grep -qF "FAIL: TC-21a"; then
|
||||
echo "ERROR: expected TC-21a failure was not observed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! printf "%s\n" "$SANITIZED_OUTPUT" | grep -qF "FAIL: TC-29"; then
|
||||
echo "ERROR: expected TC-29 failure was not observed"
|
||||
exit 1
|
||||
fi
|
||||
for expected in 'FAIL: TC-21a:' 'FAIL: TC-29:' 'Red Team Test Results'; do
|
||||
if ! grep -qF "$expected" <<< "$SANITIZED_OUTPUT"; then
|
||||
echo "ERROR: expected mutation signal not observed: $expected"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "==> [3/3] PASS: rules-level signal verified (baseline pass + rules-disabled fail)"
|
||||
|
||||
@@ -2,81 +2,58 @@
|
||||
# Verify red_team_test.sh has real detection signal.
|
||||
#
|
||||
# Strategy:
|
||||
# 1) Run baseline red team test (must pass).
|
||||
# 2) Introduce a temporary mutation that bypasses HIGH/CRITICAL rollback gate.
|
||||
# 3) Rebuild binary and run red team test again (must fail).
|
||||
# 4) Restore original source and rebuild.
|
||||
# 1) Build a private baseline binary and run red team (must pass).
|
||||
# 2) Build with a Go overlay that bypasses the HIGH/CRITICAL gate.
|
||||
# 3) Require the mutated run to fail the single and batch update gate tests.
|
||||
# Source and bin/skillshare are never modified, even on failure or interruption.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
|
||||
GATE_GO="$PROJECT_ROOT/internal/install/install_audit.go"
|
||||
BACKUP_FILE=""
|
||||
BACKUP_READY=false
|
||||
RESTORED=false
|
||||
|
||||
backup_source() {
|
||||
BACKUP_FILE="$(mktemp "${TMPDIR:-/tmp}/install_audit.go.backup.XXXXXX")"
|
||||
cp "$GATE_GO" "$BACKUP_FILE"
|
||||
BACKUP_READY=true
|
||||
}
|
||||
|
||||
restore_source() {
|
||||
if [ "$RESTORED" = false ] && [ "$BACKUP_READY" = true ] && [ -f "$BACKUP_FILE" ]; then
|
||||
cp "$BACKUP_FILE" "$GATE_GO"
|
||||
RESTORED=true
|
||||
fi
|
||||
if [ -n "$BACKUP_FILE" ]; then
|
||||
rm -f "$BACKUP_FILE"
|
||||
fi
|
||||
}
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/redteam-signal.XXXXXX")"
|
||||
TEST_BIN="$TMP_DIR/skillshare"
|
||||
|
||||
cleanup() {
|
||||
restore_source
|
||||
rm -rf "$TMP_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
echo "==> [1/4] Baseline: running make test-redteam (should pass)"
|
||||
(cd "$PROJECT_ROOT" && make test-redteam)
|
||||
echo "==> [1/3] Baseline: build private binary and run red team (should pass)"
|
||||
(cd "$PROJECT_ROOT" && go build -o "$TEST_BIN" ./cmd/skillshare)
|
||||
SKILLSHARE_TEST_BINARY="$TEST_BIN" "$SCRIPT_DIR/red_team_test.sh"
|
||||
|
||||
echo "==> [2/4] Injecting mutation: bypass rollback on HIGH/CRITICAL findings"
|
||||
backup_source
|
||||
|
||||
# install and update both decide through AuditGate.Run; skip its threshold check.
|
||||
perl -0pi -e 's@if !scanResult\.HasSeverityAtOrAbove\(threshold\) \{@if true { // MUTATION: bypass rollback gate@g' "$GATE_GO"
|
||||
|
||||
if command -v rg >/dev/null 2>&1; then
|
||||
MUTATION_COUNT="$(rg -c "MUTATION: bypass rollback gate" "$GATE_GO" || true)"
|
||||
else
|
||||
MUTATION_COUNT="$(grep -c "MUTATION: bypass rollback gate" "$GATE_GO" || true)"
|
||||
fi
|
||||
MUTATION_COUNT="${MUTATION_COUNT:-0}"
|
||||
if ! [[ "$MUTATION_COUNT" =~ ^[0-9]+$ ]]; then
|
||||
echo "ERROR: mutation count is not numeric: $MUTATION_COUNT"
|
||||
exit 2
|
||||
fi
|
||||
echo "==> [2/3] Build mutation: bypass HIGH/CRITICAL gate through a Go overlay"
|
||||
# Keep the mutation in a private copy, not the working tree.
|
||||
perl -0pe 's@if !scanResult\.HasSeverityAtOrAbove\(threshold\) \{@if true { // MUTATION: bypass rollback gate@g' \
|
||||
"$GATE_GO" > "$TMP_DIR/install_audit.go"
|
||||
MUTATION_COUNT="$(grep -c 'MUTATION: bypass rollback gate' "$TMP_DIR/install_audit.go" || true)"
|
||||
if [ "$MUTATION_COUNT" -ne 1 ]; then
|
||||
echo "ERROR: expected to inject 1 mutation point, got $MUTATION_COUNT"
|
||||
exit 2
|
||||
fi
|
||||
jq -n --arg original "$GATE_GO" --arg mutated "$TMP_DIR/install_audit.go" \
|
||||
'{Replace: {($original): $mutated}}' > "$TMP_DIR/overlay.json"
|
||||
(cd "$PROJECT_ROOT" && go build -overlay "$TMP_DIR/overlay.json" -o "$TEST_BIN" ./cmd/skillshare)
|
||||
|
||||
echo "==> [3/4] Rebuild mutated binary and run red team (should fail)"
|
||||
(cd "$PROJECT_ROOT" && go build -o bin/skillshare ./cmd/skillshare)
|
||||
echo "==> [3/3] Run mutated red team (must fail the single and batch update gates)"
|
||||
MUTATED_EXIT=0
|
||||
SKILLSHARE_TEST_BINARY="$TEST_BIN" "$SCRIPT_DIR/red_team_test.sh" > "$TMP_DIR/output.log" 2>&1 || MUTATED_EXIT=$?
|
||||
cat "$TMP_DIR/output.log"
|
||||
perl -pe 's/\e\[[0-9;]*m//g' "$TMP_DIR/output.log" > "$TMP_DIR/plain.log"
|
||||
|
||||
set +e
|
||||
(cd "$PROJECT_ROOT" && ./scripts/red_team_test.sh)
|
||||
MUTATED_EXIT=$?
|
||||
set -e
|
||||
|
||||
if [ "$MUTATED_EXIT" -eq 0 ]; then
|
||||
echo "ERROR: mutated run unexpectedly passed; red team signal check failed"
|
||||
if [ "$MUTATED_EXIT" -ne 1 ]; then
|
||||
echo "ERROR: expected completed test failure (exit=1), got exit=$MUTATED_EXIT"
|
||||
exit 1
|
||||
fi
|
||||
echo "PASS: mutated run failed as expected (exit=$MUTATED_EXIT)"
|
||||
for expected in 'FAIL: TC-21a:' 'FAIL: TC-29:' 'Red Team Test Results'; do
|
||||
if ! grep -qF "$expected" "$TMP_DIR/plain.log"; then
|
||||
echo "ERROR: expected mutation signal not observed: $expected"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "==> [4/4] Restoring original source and rebuilding"
|
||||
restore_source
|
||||
(cd "$PROJECT_ROOT" && go build -o bin/skillshare ./cmd/skillshare)
|
||||
|
||||
echo "PASS: test-redteam signal verified (baseline pass + mutation fail)"
|
||||
echo "PASS: test-redteam signal verified (baseline pass + expected gate failures)"
|
||||
|
||||
Reference in New Issue
Block a user