From eb3ca528fcac200524e51e83ae67c10822b3791a Mon Sep 17 00:00:00 2001 From: Willie Date: Wed, 11 Nov 2026 21:40:44 +0800 Subject: [PATCH] fix(test): harden red-team assertions and mutation checks (#856) Reject invalid audit output and CLI errors before accepting absent findings, including integrity and traversal checks. Require completed mutation runs to fail the single and batch update gates rather than accepting unrelated failures. Build gate mutations with a private Go overlay and binary so failures cannot leave source or the shared binary weakened. Add regression coverage for assertions, signal checks, and source/binary isolation to every red-team Make target. --- Makefile | 12 ++- scripts/red_team/_helpers.sh | 35 +++++++-- scripts/red_team/phase3_integrity.sh | 9 +-- scripts/red_team/phase5_advanced.sh | 10 +-- scripts/test_redteam_harness.sh | 108 +++++++++++++++++++++++++++ scripts/test_redteam_rules_signal.sh | 19 ++--- scripts/test_redteam_signal.sh | 91 +++++++++------------- 7 files changed, 190 insertions(+), 94 deletions(-) create mode 100644 scripts/test_redteam_harness.sh diff --git a/Makefile b/Makefile index f89b9d735..8b3017171 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,7 @@ help: @echo " make test-docker # docker offline sandbox (build + unit + integration)" @echo " make test-docker-online # docker online install/update tests" @echo " make test-redteam # red team supply-chain security tests" + @echo " make test-redteam-harness # regression tests for red team assertions and signal checks" @echo " make test-redteam-signal # verify red team test fails on intentional mutation" @echo " make test-redteam-rules-signal # verify red team test fails when critical builtin rules are disabled" @echo " make playground # start playground + enter shell (one step)" @@ -71,13 +72,18 @@ test-docker: test-docker-online: ./scripts/test_docker_online.sh -test-redteam: build +.PHONY: test-redteam-harness + +test-redteam-harness: + bash ./scripts/test_redteam_harness.sh + +test-redteam: build test-redteam-harness ./scripts/red_team_test.sh -test-redteam-signal: +test-redteam-signal: test-redteam-harness ./scripts/test_redteam_signal.sh -test-redteam-rules-signal: +test-redteam-rules-signal: test-redteam-harness ./scripts/test_redteam_rules_signal.sh playground: diff --git a/scripts/red_team/_helpers.sh b/scripts/red_team/_helpers.sh index f5aed0e3a..bdf62daeb 100644 --- a/scripts/red_team/_helpers.sh +++ b/scripts/red_team/_helpers.sh @@ -147,17 +147,36 @@ assert_finding() { fi } -# Assert audit JSON output does NOT contain a finding with given pattern -# Usage: assert_no_finding