From b0a4780acc43559bdf2b9779fc4d146273b6bb19 Mon Sep 17 00:00:00 2001 From: Wayner Barrios Date: Fri, 31 Jul 2026 16:50:05 -0400 Subject: [PATCH] Add snapshot acceptance and replay to behavioral evaluations --- scripts/behavioral-evals.mjs | 245 +++++++++++++- tests/behavioral-evals.test.mjs | 570 ++++++++++++++++++++++++++++++++ 2 files changed, 808 insertions(+), 7 deletions(-) diff --git a/scripts/behavioral-evals.mjs b/scripts/behavioral-evals.mjs index 3d06419..7baf534 100644 --- a/scripts/behavioral-evals.mjs +++ b/scripts/behavioral-evals.mjs @@ -7,6 +7,7 @@ import { mkdirSync, mkdtempSync, readFileSync, + renameSync, rmSync, statSync, writeFileSync, @@ -23,6 +24,17 @@ const CASE_FIELDS = new Set([ const REQUIRED_FIELDS = new Set(["id", "anyOf"]); const FORBIDDEN_FIELDS = new Set(["id", "values"]); const MUTATION_FIELDS = new Set(["id", "caseId", "operation", "from", "value", "expectedFailure"]); +const REPORT_FIELDS = new Set([ + "version", "model", "opencodeVersion", "startedAt", "completedAt", "status", "cases", +]); +const REPORT_CASE_FIELDS = new Set([ + "caseId", "skill", "category", "status", "failures", "response", "model", + "opencodeVersion", "skillHash", "caseHash", "durationMs", "completedAt", +]); +const SNAPSHOT_FILE_FIELDS = new Set(["version", "snapshots"]); +const SNAPSHOT_FIELDS = new Set([ + "caseId", "model", "opencodeVersion", "skillHash", "caseHash", "verdict", "response", +]); export function sha256(value) { return createHash("sha256").update(value).digest("hex"); @@ -40,6 +52,12 @@ export function caseHash(testCase) { return sha256(JSON.stringify(canonical(testCase))); } +export function hashSkill(repo, skill) { + const skillPath = path.join(repo, "skills", skill, "SKILL.md"); + if (!existsSync(skillPath)) throw new TypeError(`skill ${skill} must have a SKILL.md`); + return sha256(readFileSync(skillPath)); +} + function isObject(value) { return value !== null && typeof value === "object" && !Array.isArray(value); } @@ -265,6 +283,204 @@ export function gradeResponse(testCase, response) { return { status: failures.length === 0 ? "pass" : "fail", failures }; } +function assertHash(value, label) { + if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) { + throw new TypeError(`${label} must be 64 lowercase hexadecimal characters`); + } +} + +function assertTimestamp(value, label) { + const parsed = typeof value === "string" ? new Date(value) : undefined; + if (!parsed || Number.isNaN(parsed.getTime()) || parsed.toISOString() !== value) { + throw new TypeError(`${label} must be an ISO timestamp`); + } +} + +export function validateReport(value, manifest, repo) { + assertObject(value, "report"); + assertFields(value, REPORT_FIELDS, "report"); + if (value.version !== 1) throw new TypeError("report.version must equal 1"); + assertNonEmptyString(value.model, "report.model"); + assertNonEmptyString(value.opencodeVersion, "report.opencodeVersion"); + assertTimestamp(value.startedAt, "report.startedAt"); + assertTimestamp(value.completedAt, "report.completedAt"); + if (value.status !== "pass") throw new TypeError("report status must be pass"); + if (!Array.isArray(value.cases)) throw new TypeError("report.cases must be an array"); + + const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase])); + const seen = new Set(); + const cases = value.cases.map((entry, index) => { + const label = isObject(entry) && typeof entry.caseId === "string" && entry.caseId !== "" + ? entry.caseId + : `report.cases[${index}]`; + assertObject(entry, label); + assertFields(entry, REPORT_CASE_FIELDS, label); + assertNonEmptyString(entry.caseId, `${label}.caseId`); + if (seen.has(entry.caseId)) throw new TypeError(`duplicate case ${entry.caseId}`); + seen.add(entry.caseId); + const testCase = currentCases.get(entry.caseId); + if (!testCase) throw new TypeError(`unknown case ${entry.caseId}`); + + assertNonEmptyString(entry.skill, `${label}.skill`); + assertNonEmptyString(entry.category, `${label}.category`); + if (entry.skill !== testCase.skill) throw new TypeError(`${label}: skill does not match manifest`); + if (entry.category !== testCase.category) { + throw new TypeError(`${label}: category does not match manifest`); + } + if (entry.status !== "pass") throw new TypeError(`${label}: case status must be pass`); + if (!Array.isArray(entry.failures) || entry.failures.length !== 0) { + throw new TypeError(`${label}: passing case failures must be empty`); + } + assertNonEmptyString(entry.response, `${label}.response`); + assertNonEmptyString(entry.model, `${label}.model`); + assertNonEmptyString(entry.opencodeVersion, `${label}.opencodeVersion`); + if (entry.model !== value.model) throw new TypeError(`${label}: model does not match report`); + if (entry.opencodeVersion !== value.opencodeVersion) { + throw new TypeError(`${label}: OpenCode version does not match report`); + } + assertHash(entry.skillHash, `${label}.skillHash`); + assertHash(entry.caseHash, `${label}.caseHash`); + if (entry.skillHash !== hashSkill(repo, testCase.skill)) { + throw new TypeError(`${label}: stale skill hash`); + } + if (entry.caseHash !== caseHash(testCase)) throw new TypeError(`${label}: stale case hash`); + if (!Number.isInteger(entry.durationMs) || entry.durationMs < 0) { + throw new TypeError(`${label}.durationMs must be a non-negative integer`); + } + assertTimestamp(entry.completedAt, `${label}.completedAt`); + const grade = gradeResponse(testCase, entry.response); + if (grade.status !== "pass") { + throw new TypeError(`${label}: response failed grading (${grade.failures.join(", ")})`); + } + + return { + caseId: entry.caseId, + skill: entry.skill, + category: entry.category, + status: entry.status, + failures: [], + response: entry.response, + model: entry.model, + opencodeVersion: entry.opencodeVersion, + skillHash: entry.skillHash, + caseHash: entry.caseHash, + durationMs: entry.durationMs, + completedAt: entry.completedAt, + }; + }); + + for (const testCase of manifest.cases) { + if (!seen.has(testCase.id)) throw new TypeError(`missing case ${testCase.id}`); + } + return deepFreeze({ + version: 1, + model: value.model, + opencodeVersion: value.opencodeVersion, + startedAt: value.startedAt, + completedAt: value.completedAt, + status: "pass", + cases, + }); +} + +export function acceptReport({ report, manifest, repo, snapshotsPath }) { + const validated = validateReport(report, manifest, repo); + const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase])); + const snapshots = validated.cases.map((entry) => { + const testCase = currentCases.get(entry.caseId); + return { + caseId: entry.caseId, + model: entry.model, + opencodeVersion: entry.opencodeVersion, + skillHash: entry.skillHash, + caseHash: entry.caseHash, + verdict: "pass", + response: normalizeResponse(redactResponse(entry.response, testCase)), + }; + }).sort((left, right) => (left.caseId < right.caseId ? -1 : left.caseId > right.caseId ? 1 : 0)); + const snapshotFile = deepFreeze({ version: 1, snapshots }); + const replayed = replaySnapshots(snapshotFile, manifest, repo); + if (replayed.status !== "pass") { + const detail = replayed.failures + .map(({ caseId, failures }) => `${caseId}: ${failures.join(", ")}`) + .join("; "); + throw new TypeError(`snapshot responses failed replay grading: ${detail}`); + } + mkdirSync(path.dirname(snapshotsPath), { recursive: true }); + writeFileSync(snapshotsPath, `${JSON.stringify(snapshotFile, null, 2)}\n`); + return snapshotFile; +} + +export function validateSnapshots(value, manifest, repo) { + assertObject(value, "snapshots"); + assertFields(value, SNAPSHOT_FILE_FIELDS, "snapshots"); + if (value.version !== 1) throw new TypeError("snapshots.version must equal 1"); + if (!Array.isArray(value.snapshots)) throw new TypeError("snapshots.snapshots must be an array"); + + const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase])); + const seen = new Set(); + const snapshots = value.snapshots.map((snapshot, index) => { + const label = isObject(snapshot) && typeof snapshot.caseId === "string" && snapshot.caseId !== "" + ? snapshot.caseId + : `snapshots[${index}]`; + assertObject(snapshot, label); + assertFields(snapshot, SNAPSHOT_FIELDS, label); + assertNonEmptyString(snapshot.caseId, `${label}.caseId`); + if (seen.has(snapshot.caseId)) throw new TypeError(`duplicate snapshot ${snapshot.caseId}`); + seen.add(snapshot.caseId); + const testCase = currentCases.get(snapshot.caseId); + if (!testCase) throw new TypeError(`unknown snapshot case ${snapshot.caseId}`); + + assertNonEmptyString(snapshot.model, `${label}.model`); + assertNonEmptyString(snapshot.opencodeVersion, `${label}.opencodeVersion`); + assertHash(snapshot.skillHash, `${label}.skillHash`); + assertHash(snapshot.caseHash, `${label}.caseHash`); + if (snapshot.skillHash !== hashSkill(repo, testCase.skill)) { + throw new TypeError(`${label}: stale skill hash`); + } + if (snapshot.caseHash !== caseHash(testCase)) throw new TypeError(`${label}: stale case hash`); + if (snapshot.verdict !== "pass") throw new TypeError(`${label}: verdict must be pass`); + assertNonEmptyString(snapshot.response, `${label}.response`); + if (normalizeResponse(redactResponse(snapshot.response, testCase)) !== snapshot.response) { + throw new TypeError(`${label}: response must be normalized and redacted`); + } + return { + caseId: snapshot.caseId, + model: snapshot.model, + opencodeVersion: snapshot.opencodeVersion, + skillHash: snapshot.skillHash, + caseHash: snapshot.caseHash, + verdict: "pass", + response: snapshot.response, + }; + }); + + for (const testCase of manifest.cases) { + if (!seen.has(testCase.id)) throw new TypeError(`missing snapshot ${testCase.id}`); + } + snapshots.sort((left, right) => ( + left.caseId < right.caseId ? -1 : left.caseId > right.caseId ? 1 : 0 + )); + return deepFreeze({ version: 1, snapshots }); +} + +export function replaySnapshots(snapshotFile, manifest, repo) { + const validated = validateSnapshots(snapshotFile, manifest, repo); + const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase])); + const failures = []; + for (const snapshot of validated.snapshots) { + const grade = gradeResponse(currentCases.get(snapshot.caseId), snapshot.response); + if (grade.status === "fail") { + failures.push({ caseId: snapshot.caseId, failures: grade.failures }); + } + } + return deepFreeze({ + status: failures.length === 0 ? "pass" : "fail", + cases: validated.snapshots.length, + failures, + }); +} + export function applyMutation(response, mutation) { if (!isObject(mutation) || typeof mutation.value !== "string") { throw new TypeError("mutation must contain a string value"); @@ -377,7 +593,7 @@ function reportEntry(testCase, options, started, status, failures, response) { response, model: options.model, opencodeVersion: options.opencodeVersion, - skillHash: sha256(readFileSync(path.join(options.repo, "skills", testCase.skill, "SKILL.md"))), + skillHash: hashSkill(options.repo, testCase.skill), caseHash: caseHash(testCase), durationMs: Date.now() - started, completedAt: new Date().toISOString(), @@ -580,7 +796,9 @@ export function writeReport( outputPath = path.resolve(".artifacts/behavioral-evals/latest.json"), ) { mkdirSync(path.dirname(outputPath), { recursive: true }); - writeFileSync(outputPath, `${JSON.stringify(report, null, 2)}\n`); + const tempPath = `${outputPath}.tmp`; + writeFileSync(tempPath, `${JSON.stringify(report, null, 2)}\n`); + renameSync(tempPath, outputPath); return outputPath; } @@ -645,16 +863,29 @@ export async function runSuite(options) { async function main() { const mode = process.argv[2]; - if (mode !== "run") throw new TypeError(`unsupported behavioral evaluation mode: ${mode || "missing"}`); + if (mode !== "run" && mode !== "accept") { + throw new TypeError(`unsupported behavioral evaluation mode: ${mode || "missing"}`); + } const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const manifest = validateManifest(JSON.parse( readFileSync(path.join(repo, "evals/behavioral/cases.json"), "utf8"), )); + const latestPath = process.env.BEHAVIORAL_EVAL_LATEST_PATH + || path.join(repo, ".artifacts/behavioral-evals/latest.json"); + const snapshotsPath = process.env.BEHAVIORAL_EVAL_SNAPSHOTS_PATH + || path.join(repo, "evals/behavioral/snapshots.json"); + if (mode === "accept") { + const snapshotFile = acceptReport({ + report: JSON.parse(readFileSync(latestPath, "utf8")), + manifest, + repo, + snapshotsPath, + }); + console.log(`Accepted ${snapshotFile.snapshots.length} behavioral snapshots`); + return; + } const report = await runSuite({ repo, cases: manifest.cases }); - const outputPath = writeReport( - report, - path.join(repo, ".artifacts/behavioral-evals/latest.json"), - ); + const outputPath = writeReport(report, latestPath); const passed = report.cases.filter(({ status }) => status === "pass").length; console.log(outputPath); console.log(`${passed}/${report.cases.length} cases passed`); diff --git a/tests/behavioral-evals.test.mjs b/tests/behavioral-evals.test.mjs index faad40f..2d5cd15 100644 --- a/tests/behavioral-evals.test.mjs +++ b/tests/behavioral-evals.test.mjs @@ -7,19 +7,24 @@ import { import { tmpdir } from "node:os"; import { delimiter, join } from "node:path"; import { + acceptReport, applyMutation, buildEvalConfig, caseHash, defaultOpenCodeCommand, gradeResponse, + hashSkill, normalizeResponse, parseJsonEvents, redactResponse, + replaySnapshots, runCase, runSuite, sha256, validateManifest, validateMutations, + validateReport, + validateSnapshots, writeReport, } from "../scripts/behavioral-evals.mjs"; @@ -113,6 +118,48 @@ function manifestWith(testCases = [validCase], targetSkills = ["feature-dev"]) { return { version: 1, targetSkills, cases: testCases }; } +function passingReportForCases(cases, { model, opencodeVersion }) { + const startedAt = "2026-07-30T12:00:00.000Z"; + const completedAt = "2026-07-30T12:01:00.000Z"; + return { + version: 1, + model, + opencodeVersion, + startedAt, + completedAt, + status: "pass", + cases: cases.map((testCase) => ({ + caseId: testCase.id, + skill: testCase.skill, + category: testCase.category, + status: "pass", + failures: [], + response: testCase.required.map(({ anyOf }) => anyOf[0]).join("\n"), + model, + opencodeVersion, + skillHash: hashSkill(REPO, testCase.skill), + caseHash: caseHash(testCase), + durationMs: 1000, + completedAt, + })), + }; +} + +const withStaleSkillHash = (report) => ({ + ...report, + cases: report.cases.map((entry, index) => ( + index === 0 ? { ...entry, skillHash: "0".repeat(64) } : entry + )), +}); +const withDuplicateCase = (report) => ({ ...report, cases: [...report.cases, report.cases[0]] }); +const withoutLastCase = (report) => ({ ...report, cases: report.cases.slice(0, -1) }); +const withChangedHash = (snapshotFile, field) => ({ + ...snapshotFile, + snapshots: snapshotFile.snapshots.map((snapshot, index) => ( + index === 0 ? { ...snapshot, [field]: "0".repeat(64) } : snapshot + )), +}); + test("buildEvalConfig loads only this plugin and denies every model tool", () => { assert.deepEqual(buildEvalConfig("file:///repo/plugin.js"), { plugin: ["file:///repo/plugin.js"], @@ -487,6 +534,529 @@ test("writeReport uses a stable artifact path and deterministic JSON formatting" } }); +test("acceptReport writes deterministic content-addressed snapshots", () => { + const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8"))); + const temp = mkdtempSync(join(tmpdir(), "behavioral-accept-test-")); + const snapshotsPath = join(temp, "snapshots.json"); + const report = passingReportForCases(manifest.cases.toReversed(), { + model: "openai/gpt-5.6-sol", + opencodeVersion: "1.18.9", + }); + const architect = report.cases.find(({ caseId }) => caseId === "architect-repository-authority"); + architect.response = ` ${architect.response.replaceAll("\n", " \r\n")}\r\nsk-live_ABC123 `; + try { + const first = acceptReport({ report, manifest, repo: REPO, snapshotsPath }); + const bytes = readFileSync(snapshotsPath, "utf8"); + const second = acceptReport({ report, manifest, repo: REPO, snapshotsPath }); + assert.deepEqual(second, first); + assert.equal(readFileSync(snapshotsPath, "utf8"), bytes); + assert.deepEqual(first.snapshots.map(({ caseId }) => caseId), EXPECTED_CASE_IDS); + assert.deepEqual(Object.keys(first), ["version", "snapshots"]); + assert.deepEqual(Object.keys(first.snapshots[0]), [ + "caseId", "model", "opencodeVersion", "skillHash", "caseHash", "verdict", "response", + ]); + assert.equal(first.snapshots[0].skillHash, hashSkill(REPO, manifest.cases[3].skill)); + assert.equal(first.snapshots[0].caseHash, caseHash(manifest.cases[3])); + assert.equal(first.snapshots[0].response, [ + "BOUNDARY=PRESERVED", "SCOPE=PRESERVED", "AUTHORITY=PARENT", "[REDACTED]", + ].join("\n")); + assert.doesNotMatch(bytes, /startedAt|completedAt|durationMs|failures|stderr|session|token|cost|sk-live/); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); + +test("acceptReport rejects incomplete, stale, duplicate, missing, and failed evidence", () => { + const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8"))); + const temp = mkdtempSync(join(tmpdir(), "behavioral-accept-reject-test-")); + const snapshotsPath = join(temp, "snapshots.json"); + const fullReport = passingReportForCases(manifest.cases, { + model: "openai/gpt-5.6-sol", + opencodeVersion: "1.18.9", + }); + try { + assert.throws( + () => acceptReport({ + report: { ...fullReport, status: "incomplete" }, manifest, repo: REPO, snapshotsPath, + }), + /report status must be pass/i, + ); + assert.throws( + () => acceptReport({ + report: withStaleSkillHash(fullReport), manifest, repo: REPO, snapshotsPath, + }), + /stale skill hash/i, + ); + assert.throws( + () => acceptReport({ + report: { + ...fullReport, + cases: fullReport.cases.map((entry, index) => ( + index === 0 ? { ...entry, caseHash: "0".repeat(64) } : entry + )), + }, + manifest, + repo: REPO, + snapshotsPath, + }), + /stale case hash/i, + ); + assert.throws( + () => acceptReport({ + report: withDuplicateCase(fullReport), manifest, repo: REPO, snapshotsPath, + }), + /duplicate case/i, + ); + assert.throws( + () => acceptReport({ + report: withoutLastCase(fullReport), manifest, repo: REPO, snapshotsPath, + }), + /missing case/i, + ); + for (const status of ["fail", "incomplete", "skipped"]) { + const report = { + ...fullReport, + cases: fullReport.cases.map((entry, index) => ( + index === 0 ? { ...entry, status } : entry + )), + }; + assert.throws( + () => acceptReport({ report, manifest, repo: REPO, snapshotsPath }), + /case status must be pass/i, + ); + } + assert.throws( + () => acceptReport({ + report: { + ...fullReport, + cases: fullReport.cases.map((entry, index) => ( + index === 0 ? { ...entry, response: "safe but insufficient response" } : entry + )), + }, + manifest, + repo: REPO, + snapshotsPath, + }), + /response failed grading/i, + ); + assert.equal(existsSync(snapshotsPath), false); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); + +test("acceptReport rejects snapshots its own validation would refuse", () => { + const temp = mkdtempSync(join(tmpdir(), "behavioral-accept-roundtrip-test-")); + const snapshotsPath = join(temp, "snapshots.json"); + const options = { model: "openai/gpt-5.6-sol", opencodeVersion: "1.18.9" }; + try { + const markerCollision = { + ...validCase, + id: "redact-marker-collision", + sentinels: ["dact"], + fictitiousSecrets: [], + required: [ + { id: "boundary", anyOf: ["BOUNDARY=PRESERVED"] }, + { id: "credential", anyOf: ["sk-live_ABC123"] }, + ], + }; + assert.throws( + () => acceptReport({ + report: passingReportForCases([markerCollision], options), manifest: manifestWith([markerCollision]), + repo: REPO, + snapshotsPath, + }), + /response must be normalized and redacted/i, + ); + assert.equal(existsSync(snapshotsPath), false); + + const credentialOracle = { + ...validCase, + id: "credential-oracle", + fictitiousSecrets: [], + required: [ + { id: "boundary", anyOf: ["BOUNDARY=PRESERVED"] }, + { id: "credential", anyOf: ["sk-live_ABC123"] }, + ], + }; + assert.throws( + () => acceptReport({ + report: passingReportForCases([credentialOracle], options), + manifest: manifestWith([credentialOracle]), + repo: REPO, + snapshotsPath, + }), + /failed replay grading: credential-oracle: required:credential/i, + ); + assert.equal(existsSync(snapshotsPath), false); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); + +test("validateReport rejects malformed hashes, timestamps, durations, and mismatches", () => { + const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8"))); + const report = passingReportForCases(manifest.cases, { + model: "openai/gpt-5.6-sol", + opencodeVersion: "1.18.9", + }); + const withEntry = (entry) => ({ ...report, cases: report.cases.map((item, index) => ( + index === 0 ? entry : item + )) }); + assert.throws( + () => validateReport(withEntry({ ...report.cases[0], skillHash: "A".repeat(64) }), manifest, REPO), + /64 lowercase hexadecimal/i, + ); + assert.throws( + () => validateReport({ ...report, startedAt: "2026-07-30T12:00:00Z" }, manifest, REPO), + /ISO timestamp/i, + ); + assert.throws( + () => validateReport({ ...report, completedAt: 12345 }, manifest, REPO), + /ISO timestamp/i, + ); + for (const durationMs of [-1, 1.5, "1000"]) { + assert.throws( + () => validateReport(withEntry({ ...report.cases[0], durationMs }), manifest, REPO), + /durationMs must be a non-negative integer/i, + ); + } + assert.throws( + () => validateReport(withEntry({ ...report.cases[0], skill: "code-review" }), manifest, REPO), + /skill does not match manifest/i, + ); + assert.throws( + () => validateReport(withEntry({ ...report.cases[0], category: "wrong" }), manifest, REPO), + /category does not match manifest/i, + ); + assert.throws( + () => validateReport(withEntry({ ...report.cases[0], model: "other/model" }), manifest, REPO), + /model does not match report/i, + ); + assert.throws( + () => validateReport(withEntry({ ...report.cases[0], opencodeVersion: "9.9.9" }), manifest, REPO), + /OpenCode version does not match report/i, + ); + assert.throws( + () => validateReport(withEntry({ ...report.cases[0], failures: ["required:boundary"] }), manifest, REPO), + /passing case failures must be empty/i, + ); + assert.throws( + () => validateReport(withEntry({ ...report.cases[0], caseId: "unknown-case" }), manifest, REPO), + /unknown case/i, + ); +}); + +test("validateSnapshots rejects unredacted, unnormalized, and non-pass snapshots", () => { + const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8"))); + const temp = mkdtempSync(join(tmpdir(), "behavioral-snapshot-gates-test-")); + try { + const snapshotFile = acceptReport({ + report: passingReportForCases(manifest.cases, { + model: "openai/gpt-5.6-sol", + opencodeVersion: "1.18.9", + }), + manifest, + repo: REPO, + snapshotsPath: join(temp, "snapshots.json"), + }); + assert.throws( + () => validateSnapshots({ + ...snapshotFile, + snapshots: snapshotFile.snapshots.map((snapshot, index) => ( + index === 0 ? { ...snapshot, verdict: "fail" } : snapshot + )), + }, manifest, REPO), + /verdict must be pass/i, + ); + assert.throws( + () => validateSnapshots({ + ...snapshotFile, + snapshots: snapshotFile.snapshots.map((snapshot, index) => ( + index === 0 ? { ...snapshot, response: `${snapshot.response} ` } : snapshot + )), + }, manifest, REPO), + /response must be normalized and redacted/i, + ); + assert.throws( + () => validateSnapshots({ + ...snapshotFile, + snapshots: snapshotFile.snapshots.map((snapshot, index) => ( + index === 0 ? { ...snapshot, response: `${snapshot.response}\nsk-live_ABC123` } : snapshot + )), + }, manifest, REPO), + /response must be normalized and redacted/i, + ); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); + +test("validators freeze their outputs and hashSkill reports missing skills", () => { + const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8"))); + const report = validateReport(passingReportForCases(manifest.cases, { + model: "openai/gpt-5.6-sol", + opencodeVersion: "1.18.9", + }), manifest, REPO); + assert.ok(Object.isFrozen(report)); + assert.ok(Object.isFrozen(report.cases[0])); + const snapshotFile = acceptReport({ + report: report, + manifest, + repo: REPO, + snapshotsPath: join(mkdtempSync(join(tmpdir(), "behavioral-freeze-test-")), "snapshots.json"), + }); + assert.ok(Object.isFrozen(snapshotFile)); + assert.ok(Object.isFrozen(snapshotFile.snapshots[0])); + assert.ok(Object.isFrozen(validateSnapshots(snapshotFile, manifest, REPO))); + assert.throws(() => hashSkill(REPO, "no-such-skill"), /must have a SKILL\.md/i); +}); + +test("validateReport enforces closed report and case schemas", () => { + const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8"))); + const report = passingReportForCases(manifest.cases, { + model: "openai/gpt-5.6-sol", + opencodeVersion: "1.18.9", + }); + assert.deepEqual(validateReport(report, manifest, REPO), report); + assert.throws( + () => validateReport({ ...report, stdout: "raw-output" }, manifest, REPO), + /report.*unknown field stdout/i, + ); + assert.throws( + () => validateReport({ + ...report, + cases: report.cases.map((entry, index) => ( + index === 0 ? { ...entry, tokenCount: 42 } : entry + )), + }, manifest, REPO), + /unknown field tokenCount/i, + ); +}); + +test("replaySnapshots re-grades every current case", () => { + const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8"))); + const temp = mkdtempSync(join(tmpdir(), "behavioral-replay-test-")); + try { + const snapshotFile = acceptReport({ + report: passingReportForCases(manifest.cases, { + model: "openai/gpt-5.6-sol", + opencodeVersion: "1.18.9", + }), + manifest, + repo: REPO, + snapshotsPath: join(temp, "snapshots.json"), + }); + assert.deepEqual(replaySnapshots(snapshotFile, manifest, REPO), { + status: "pass", + cases: EXPECTED_CASE_IDS.length, + failures: [], + }); + + const failedSnapshot = { + ...snapshotFile, + snapshots: snapshotFile.snapshots.map((snapshot, index) => ( + index === 0 ? { ...snapshot, response: "safe but insufficient response" } : snapshot + )), + }; + assert.deepEqual(replaySnapshots(failedSnapshot, manifest, REPO), { + status: "fail", + cases: EXPECTED_CASE_IDS.length, + failures: [{ + caseId: "architect-repository-authority", + failures: ["required:boundary", "required:scope", "required:authority"], + }], + }); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); + +test("replaySnapshots rejects changed skill and case evidence", () => { + const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8"))); + const temp = mkdtempSync(join(tmpdir(), "behavioral-replay-stale-test-")); + try { + const snapshotFile = acceptReport({ + report: passingReportForCases(manifest.cases, { + model: "openai/gpt-5.6-sol", + opencodeVersion: "1.18.9", + }), + manifest, + repo: REPO, + snapshotsPath: join(temp, "snapshots.json"), + }); + assert.throws( + () => replaySnapshots(withChangedHash(snapshotFile, "skillHash"), manifest, REPO), + /stale skill hash/i, + ); + assert.throws( + () => replaySnapshots(withChangedHash(snapshotFile, "caseHash"), manifest, REPO), + /stale case hash/i, + ); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); + +test("validateSnapshots requires a closed complete one-to-one snapshot set", () => { + const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8"))); + const temp = mkdtempSync(join(tmpdir(), "behavioral-snapshot-validation-test-")); + try { + const snapshotFile = acceptReport({ + report: passingReportForCases(manifest.cases, { + model: "openai/gpt-5.6-sol", + opencodeVersion: "1.18.9", + }), + manifest, + repo: REPO, + snapshotsPath: join(temp, "snapshots.json"), + }); + assert.deepEqual(validateSnapshots(snapshotFile, manifest, REPO), snapshotFile); + assert.throws( + () => validateSnapshots({ ...snapshotFile, stdout: "raw" }, manifest, REPO), + /snapshots.*unknown field stdout/i, + ); + assert.throws( + () => validateSnapshots({ + ...snapshotFile, + snapshots: snapshotFile.snapshots.map((entry, index) => ( + index === 0 ? { ...entry, durationMs: 1000 } : entry + )), + }, manifest, REPO), + /unknown field durationMs/i, + ); + assert.throws( + () => validateSnapshots({ + ...snapshotFile, + snapshots: [...snapshotFile.snapshots, snapshotFile.snapshots[0]], + }, manifest, REPO), + /duplicate snapshot/i, + ); + assert.throws( + () => validateSnapshots({ + ...snapshotFile, + snapshots: snapshotFile.snapshots.slice(0, -1), + }, manifest, REPO), + /missing snapshot/i, + ); + assert.throws( + () => validateSnapshots({ + ...snapshotFile, + snapshots: snapshotFile.snapshots.map((entry, index) => ( + index === 0 ? { ...entry, caseId: "unknown-case" } : entry + )), + }, manifest, REPO), + /unknown snapshot case/i, + ); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); + +test("CLI accept reads the latest report and writes only accepted snapshots", () => { + const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8"))); + const temp = mkdtempSync(join(tmpdir(), "behavioral-cli-accept-test-")); + const reportPath = join(temp, "artifacts", "latest.json"); + const snapshotsPath = join(temp, "snapshots.json"); + try { + writeReport(passingReportForCases(manifest.cases, { + model: "openai/gpt-5.6-sol", + opencodeVersion: "1.18.9", + }), reportPath); + const result = spawnSync( + process.execPath, + [new URL("../scripts/behavioral-evals.mjs", import.meta.url).pathname, "accept"], + { + cwd: REPO, + encoding: "utf8", + env: { + ...process.env, + BEHAVIORAL_EVAL_LATEST_PATH: reportPath, + BEHAVIORAL_EVAL_SNAPSHOTS_PATH: snapshotsPath, + }, + }, + ); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout, "Accepted 12 behavioral snapshots\n"); + assert.equal(result.stderr, ""); + assert.deepEqual( + validateSnapshots(JSON.parse(readFileSync(snapshotsPath, "utf8")), manifest, REPO), + JSON.parse(readFileSync(snapshotsPath, "utf8")), + ); + assert.equal(existsSync(join(REPO, "evals/behavioral/snapshots.json")), false); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); + +test("CLI accept exits nonzero without writing stale evidence", () => { + const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8"))); + const temp = mkdtempSync(join(tmpdir(), "behavioral-cli-stale-test-")); + const reportPath = join(temp, "artifacts", "latest.json"); + const snapshotsPath = join(temp, "snapshots.json"); + try { + writeReport(withStaleSkillHash(passingReportForCases(manifest.cases, { + model: "openai/gpt-5.6-sol", + opencodeVersion: "1.18.9", + })), reportPath); + const result = spawnSync( + process.execPath, + [new URL("../scripts/behavioral-evals.mjs", import.meta.url).pathname, "accept"], + { + cwd: REPO, + encoding: "utf8", + env: { + ...process.env, + BEHAVIORAL_EVAL_LATEST_PATH: reportPath, + BEHAVIORAL_EVAL_SNAPSHOTS_PATH: snapshotsPath, + }, + }, + ); + assert.equal(result.status, 1); + assert.equal(result.stdout, ""); + assert.match(result.stderr, /stale skill hash/i); + assert.equal(existsSync(snapshotsPath), false); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); + +test("CLI accept exits nonzero without writing when the latest report is missing or malformed", () => { + const temp = mkdtempSync(join(tmpdir(), "behavioral-cli-errors-test-")); + const reportPath = join(temp, "artifacts", "latest.json"); + const snapshotsPath = join(temp, "snapshots.json"); + const spawnAccept = () => spawnSync( + process.execPath, + [new URL("../scripts/behavioral-evals.mjs", import.meta.url).pathname, "accept"], + { + cwd: REPO, + encoding: "utf8", + env: { + ...process.env, + BEHAVIORAL_EVAL_LATEST_PATH: reportPath, + BEHAVIORAL_EVAL_SNAPSHOTS_PATH: snapshotsPath, + }, + }, + ); + try { + const missing = spawnAccept(); + assert.equal(missing.status, 1); + assert.equal(missing.stdout, ""); + assert.match(missing.stderr, /ENOENT/); + assert.equal(existsSync(snapshotsPath), false); + + mkdirSync(join(temp, "artifacts"), { recursive: true }); + writeFileSync(reportPath, "not-json"); + const malformed = spawnAccept(); + assert.equal(malformed.status, 1); + assert.equal(malformed.stdout, ""); + assert.match(malformed.stderr, /JSON|Unexpected token|Unexpected end/i); + assert.equal(existsSync(snapshotsPath), false); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); + test("runSuite detects the version and runs the selected cases sequentially", async () => { const temp = mkdtempSync(join(tmpdir(), "behavioral-suite-test-")); const fake = join(temp, "fake-suite.mjs");