Files
skillshare/scripts/test_redteam_rules_signal.sh
T
Willie eb3ca528fc fix(test): harden red-team assertions and mutation checks (#856)
Reject invalid audit output and CLI errors before accepting absent findings, including integrity and traversal checks. Require completed mutation runs to fail the single and batch update gates rather than accepting unrelated failures.

Build gate mutations with a private Go overlay and binary so failures cannot leave source or the shared binary weakened. Add regression coverage for assertions, signal checks, and source/binary isolation to every red-team Make target.
2026-11-11 21:40:44 +08:00

78 lines
2.0 KiB
Bash
Executable File

#!/bin/bash
# Verify red_team_test.sh has rule-level detection signal.
#
# Strategy:
# 1) Run baseline red team test (must pass).
# 2) Re-run red team with critical builtin prompt-injection rules disabled
# via a temporary audit-rules.yaml injected by a wrapper binary.
# 3) Assert the run fails and includes expected gate-related failures.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
REAL_BIN="$PROJECT_ROOT/bin/skillshare"
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/redteam-rules-signal.XXXXXX")"
WRAPPER_BIN="$TMP_DIR/skillshare-wrapper"
cleanup() {
rm -rf "$TMP_DIR"
}
trap cleanup EXIT
echo "==> [1/3] Baseline: running make test-redteam (should pass)"
(cd "$PROJECT_ROOT" && make test-redteam)
if [ ! -x "$REAL_BIN" ]; then
echo "ERROR: expected binary at $REAL_BIN after baseline run"
exit 2
fi
cat >"$WRAPPER_BIN" <<'WRAP_EOF'
#!/bin/bash
set -euo pipefail
if [ -n "${SKILLSHARE_CONFIG:-}" ]; then
RULES_FILE="$(dirname "$SKILLSHARE_CONFIG")/audit-rules.yaml"
cat >"$RULES_FILE" <<'RULES_EOF'
rules:
- id: prompt-injection-0
enabled: false
- id: prompt-injection-1
enabled: false
RULES_EOF
fi
exec "$SKILLSHARE_REAL_BINARY" "$@"
WRAP_EOF
chmod +x "$WRAPPER_BIN"
echo "==> [2/3] Run red team with builtin prompt-injection rules disabled (should fail)"
set +e
MUTATED_OUTPUT="$(
cd "$PROJECT_ROOT" && \
SKILLSHARE_REAL_BINARY="$REAL_BIN" \
SKILLSHARE_TEST_BINARY="$WRAPPER_BIN" \
./scripts/red_team_test.sh 2>&1
)"
MUTATED_EXIT=$?
set -e
printf "%s\n" "$MUTATED_OUTPUT"
SANITIZED_OUTPUT="$(printf "%s\n" "$MUTATED_OUTPUT" | perl -pe 's/\e\[[0-9;]*m//g')"
if [ "$MUTATED_EXIT" -ne 1 ]; then
echo "ERROR: expected completed test failure (exit=1), got exit=$MUTATED_EXIT"
exit 1
fi
for expected in 'FAIL: TC-21a:' 'FAIL: TC-29:' 'Red Team Test Results'; do
if ! grep -qF "$expected" <<< "$SANITIZED_OUTPUT"; then
echo "ERROR: expected mutation signal not observed: $expected"
exit 1
fi
done
echo "==> [3/3] PASS: rules-level signal verified (baseline pass + rules-disabled fail)"