Add snapshot acceptance and replay to behavioral evaluations

This commit is contained in:
Wayner Barrios
2026-07-31 16:50:05 -04:00
parent cd5c150b08
commit b0a4780acc
2 changed files with 808 additions and 7 deletions
+238 -7
View File
@@ -7,6 +7,7 @@ import {
mkdirSync,
mkdtempSync,
readFileSync,
renameSync,
rmSync,
statSync,
writeFileSync,
@@ -23,6 +24,17 @@ const CASE_FIELDS = new Set([
const REQUIRED_FIELDS = new Set(["id", "anyOf"]);
const FORBIDDEN_FIELDS = new Set(["id", "values"]);
const MUTATION_FIELDS = new Set(["id", "caseId", "operation", "from", "value", "expectedFailure"]);
const REPORT_FIELDS = new Set([
"version", "model", "opencodeVersion", "startedAt", "completedAt", "status", "cases",
]);
const REPORT_CASE_FIELDS = new Set([
"caseId", "skill", "category", "status", "failures", "response", "model",
"opencodeVersion", "skillHash", "caseHash", "durationMs", "completedAt",
]);
const SNAPSHOT_FILE_FIELDS = new Set(["version", "snapshots"]);
const SNAPSHOT_FIELDS = new Set([
"caseId", "model", "opencodeVersion", "skillHash", "caseHash", "verdict", "response",
]);
export function sha256(value) {
return createHash("sha256").update(value).digest("hex");
@@ -40,6 +52,12 @@ export function caseHash(testCase) {
return sha256(JSON.stringify(canonical(testCase)));
}
export function hashSkill(repo, skill) {
const skillPath = path.join(repo, "skills", skill, "SKILL.md");
if (!existsSync(skillPath)) throw new TypeError(`skill ${skill} must have a SKILL.md`);
return sha256(readFileSync(skillPath));
}
function isObject(value) {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
@@ -265,6 +283,204 @@ export function gradeResponse(testCase, response) {
return { status: failures.length === 0 ? "pass" : "fail", failures };
}
function assertHash(value, label) {
if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) {
throw new TypeError(`${label} must be 64 lowercase hexadecimal characters`);
}
}
function assertTimestamp(value, label) {
const parsed = typeof value === "string" ? new Date(value) : undefined;
if (!parsed || Number.isNaN(parsed.getTime()) || parsed.toISOString() !== value) {
throw new TypeError(`${label} must be an ISO timestamp`);
}
}
export function validateReport(value, manifest, repo) {
assertObject(value, "report");
assertFields(value, REPORT_FIELDS, "report");
if (value.version !== 1) throw new TypeError("report.version must equal 1");
assertNonEmptyString(value.model, "report.model");
assertNonEmptyString(value.opencodeVersion, "report.opencodeVersion");
assertTimestamp(value.startedAt, "report.startedAt");
assertTimestamp(value.completedAt, "report.completedAt");
if (value.status !== "pass") throw new TypeError("report status must be pass");
if (!Array.isArray(value.cases)) throw new TypeError("report.cases must be an array");
const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase]));
const seen = new Set();
const cases = value.cases.map((entry, index) => {
const label = isObject(entry) && typeof entry.caseId === "string" && entry.caseId !== ""
? entry.caseId
: `report.cases[${index}]`;
assertObject(entry, label);
assertFields(entry, REPORT_CASE_FIELDS, label);
assertNonEmptyString(entry.caseId, `${label}.caseId`);
if (seen.has(entry.caseId)) throw new TypeError(`duplicate case ${entry.caseId}`);
seen.add(entry.caseId);
const testCase = currentCases.get(entry.caseId);
if (!testCase) throw new TypeError(`unknown case ${entry.caseId}`);
assertNonEmptyString(entry.skill, `${label}.skill`);
assertNonEmptyString(entry.category, `${label}.category`);
if (entry.skill !== testCase.skill) throw new TypeError(`${label}: skill does not match manifest`);
if (entry.category !== testCase.category) {
throw new TypeError(`${label}: category does not match manifest`);
}
if (entry.status !== "pass") throw new TypeError(`${label}: case status must be pass`);
if (!Array.isArray(entry.failures) || entry.failures.length !== 0) {
throw new TypeError(`${label}: passing case failures must be empty`);
}
assertNonEmptyString(entry.response, `${label}.response`);
assertNonEmptyString(entry.model, `${label}.model`);
assertNonEmptyString(entry.opencodeVersion, `${label}.opencodeVersion`);
if (entry.model !== value.model) throw new TypeError(`${label}: model does not match report`);
if (entry.opencodeVersion !== value.opencodeVersion) {
throw new TypeError(`${label}: OpenCode version does not match report`);
}
assertHash(entry.skillHash, `${label}.skillHash`);
assertHash(entry.caseHash, `${label}.caseHash`);
if (entry.skillHash !== hashSkill(repo, testCase.skill)) {
throw new TypeError(`${label}: stale skill hash`);
}
if (entry.caseHash !== caseHash(testCase)) throw new TypeError(`${label}: stale case hash`);
if (!Number.isInteger(entry.durationMs) || entry.durationMs < 0) {
throw new TypeError(`${label}.durationMs must be a non-negative integer`);
}
assertTimestamp(entry.completedAt, `${label}.completedAt`);
const grade = gradeResponse(testCase, entry.response);
if (grade.status !== "pass") {
throw new TypeError(`${label}: response failed grading (${grade.failures.join(", ")})`);
}
return {
caseId: entry.caseId,
skill: entry.skill,
category: entry.category,
status: entry.status,
failures: [],
response: entry.response,
model: entry.model,
opencodeVersion: entry.opencodeVersion,
skillHash: entry.skillHash,
caseHash: entry.caseHash,
durationMs: entry.durationMs,
completedAt: entry.completedAt,
};
});
for (const testCase of manifest.cases) {
if (!seen.has(testCase.id)) throw new TypeError(`missing case ${testCase.id}`);
}
return deepFreeze({
version: 1,
model: value.model,
opencodeVersion: value.opencodeVersion,
startedAt: value.startedAt,
completedAt: value.completedAt,
status: "pass",
cases,
});
}
export function acceptReport({ report, manifest, repo, snapshotsPath }) {
const validated = validateReport(report, manifest, repo);
const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase]));
const snapshots = validated.cases.map((entry) => {
const testCase = currentCases.get(entry.caseId);
return {
caseId: entry.caseId,
model: entry.model,
opencodeVersion: entry.opencodeVersion,
skillHash: entry.skillHash,
caseHash: entry.caseHash,
verdict: "pass",
response: normalizeResponse(redactResponse(entry.response, testCase)),
};
}).sort((left, right) => (left.caseId < right.caseId ? -1 : left.caseId > right.caseId ? 1 : 0));
const snapshotFile = deepFreeze({ version: 1, snapshots });
const replayed = replaySnapshots(snapshotFile, manifest, repo);
if (replayed.status !== "pass") {
const detail = replayed.failures
.map(({ caseId, failures }) => `${caseId}: ${failures.join(", ")}`)
.join("; ");
throw new TypeError(`snapshot responses failed replay grading: ${detail}`);
}
mkdirSync(path.dirname(snapshotsPath), { recursive: true });
writeFileSync(snapshotsPath, `${JSON.stringify(snapshotFile, null, 2)}\n`);
return snapshotFile;
}
export function validateSnapshots(value, manifest, repo) {
assertObject(value, "snapshots");
assertFields(value, SNAPSHOT_FILE_FIELDS, "snapshots");
if (value.version !== 1) throw new TypeError("snapshots.version must equal 1");
if (!Array.isArray(value.snapshots)) throw new TypeError("snapshots.snapshots must be an array");
const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase]));
const seen = new Set();
const snapshots = value.snapshots.map((snapshot, index) => {
const label = isObject(snapshot) && typeof snapshot.caseId === "string" && snapshot.caseId !== ""
? snapshot.caseId
: `snapshots[${index}]`;
assertObject(snapshot, label);
assertFields(snapshot, SNAPSHOT_FIELDS, label);
assertNonEmptyString(snapshot.caseId, `${label}.caseId`);
if (seen.has(snapshot.caseId)) throw new TypeError(`duplicate snapshot ${snapshot.caseId}`);
seen.add(snapshot.caseId);
const testCase = currentCases.get(snapshot.caseId);
if (!testCase) throw new TypeError(`unknown snapshot case ${snapshot.caseId}`);
assertNonEmptyString(snapshot.model, `${label}.model`);
assertNonEmptyString(snapshot.opencodeVersion, `${label}.opencodeVersion`);
assertHash(snapshot.skillHash, `${label}.skillHash`);
assertHash(snapshot.caseHash, `${label}.caseHash`);
if (snapshot.skillHash !== hashSkill(repo, testCase.skill)) {
throw new TypeError(`${label}: stale skill hash`);
}
if (snapshot.caseHash !== caseHash(testCase)) throw new TypeError(`${label}: stale case hash`);
if (snapshot.verdict !== "pass") throw new TypeError(`${label}: verdict must be pass`);
assertNonEmptyString(snapshot.response, `${label}.response`);
if (normalizeResponse(redactResponse(snapshot.response, testCase)) !== snapshot.response) {
throw new TypeError(`${label}: response must be normalized and redacted`);
}
return {
caseId: snapshot.caseId,
model: snapshot.model,
opencodeVersion: snapshot.opencodeVersion,
skillHash: snapshot.skillHash,
caseHash: snapshot.caseHash,
verdict: "pass",
response: snapshot.response,
};
});
for (const testCase of manifest.cases) {
if (!seen.has(testCase.id)) throw new TypeError(`missing snapshot ${testCase.id}`);
}
snapshots.sort((left, right) => (
left.caseId < right.caseId ? -1 : left.caseId > right.caseId ? 1 : 0
));
return deepFreeze({ version: 1, snapshots });
}
export function replaySnapshots(snapshotFile, manifest, repo) {
const validated = validateSnapshots(snapshotFile, manifest, repo);
const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase]));
const failures = [];
for (const snapshot of validated.snapshots) {
const grade = gradeResponse(currentCases.get(snapshot.caseId), snapshot.response);
if (grade.status === "fail") {
failures.push({ caseId: snapshot.caseId, failures: grade.failures });
}
}
return deepFreeze({
status: failures.length === 0 ? "pass" : "fail",
cases: validated.snapshots.length,
failures,
});
}
export function applyMutation(response, mutation) {
if (!isObject(mutation) || typeof mutation.value !== "string") {
throw new TypeError("mutation must contain a string value");
@@ -377,7 +593,7 @@ function reportEntry(testCase, options, started, status, failures, response) {
response,
model: options.model,
opencodeVersion: options.opencodeVersion,
skillHash: sha256(readFileSync(path.join(options.repo, "skills", testCase.skill, "SKILL.md"))),
skillHash: hashSkill(options.repo, testCase.skill),
caseHash: caseHash(testCase),
durationMs: Date.now() - started,
completedAt: new Date().toISOString(),
@@ -580,7 +796,9 @@ export function writeReport(
outputPath = path.resolve(".artifacts/behavioral-evals/latest.json"),
) {
mkdirSync(path.dirname(outputPath), { recursive: true });
writeFileSync(outputPath, `${JSON.stringify(report, null, 2)}\n`);
const tempPath = `${outputPath}.tmp`;
writeFileSync(tempPath, `${JSON.stringify(report, null, 2)}\n`);
renameSync(tempPath, outputPath);
return outputPath;
}
@@ -645,16 +863,29 @@ export async function runSuite(options) {
async function main() {
const mode = process.argv[2];
if (mode !== "run") throw new TypeError(`unsupported behavioral evaluation mode: ${mode || "missing"}`);
if (mode !== "run" && mode !== "accept") {
throw new TypeError(`unsupported behavioral evaluation mode: ${mode || "missing"}`);
}
const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const manifest = validateManifest(JSON.parse(
readFileSync(path.join(repo, "evals/behavioral/cases.json"), "utf8"),
));
const latestPath = process.env.BEHAVIORAL_EVAL_LATEST_PATH
|| path.join(repo, ".artifacts/behavioral-evals/latest.json");
const snapshotsPath = process.env.BEHAVIORAL_EVAL_SNAPSHOTS_PATH
|| path.join(repo, "evals/behavioral/snapshots.json");
if (mode === "accept") {
const snapshotFile = acceptReport({
report: JSON.parse(readFileSync(latestPath, "utf8")),
manifest,
repo,
snapshotsPath,
});
console.log(`Accepted ${snapshotFile.snapshots.length} behavioral snapshots`);
return;
}
const report = await runSuite({ repo, cases: manifest.cases });
const outputPath = writeReport(
report,
path.join(repo, ".artifacts/behavioral-evals/latest.json"),
);
const outputPath = writeReport(report, latestPath);
const passed = report.cases.filter(({ status }) => status === "pass").length;
console.log(outputPath);
console.log(`${passed}/${report.cases.length} cases passed`);
+570
View File
@@ -7,19 +7,24 @@ import {
import { tmpdir } from "node:os";
import { delimiter, join } from "node:path";
import {
acceptReport,
applyMutation,
buildEvalConfig,
caseHash,
defaultOpenCodeCommand,
gradeResponse,
hashSkill,
normalizeResponse,
parseJsonEvents,
redactResponse,
replaySnapshots,
runCase,
runSuite,
sha256,
validateManifest,
validateMutations,
validateReport,
validateSnapshots,
writeReport,
} from "../scripts/behavioral-evals.mjs";
@@ -113,6 +118,48 @@ function manifestWith(testCases = [validCase], targetSkills = ["feature-dev"]) {
return { version: 1, targetSkills, cases: testCases };
}
function passingReportForCases(cases, { model, opencodeVersion }) {
const startedAt = "2026-07-30T12:00:00.000Z";
const completedAt = "2026-07-30T12:01:00.000Z";
return {
version: 1,
model,
opencodeVersion,
startedAt,
completedAt,
status: "pass",
cases: cases.map((testCase) => ({
caseId: testCase.id,
skill: testCase.skill,
category: testCase.category,
status: "pass",
failures: [],
response: testCase.required.map(({ anyOf }) => anyOf[0]).join("\n"),
model,
opencodeVersion,
skillHash: hashSkill(REPO, testCase.skill),
caseHash: caseHash(testCase),
durationMs: 1000,
completedAt,
})),
};
}
const withStaleSkillHash = (report) => ({
...report,
cases: report.cases.map((entry, index) => (
index === 0 ? { ...entry, skillHash: "0".repeat(64) } : entry
)),
});
const withDuplicateCase = (report) => ({ ...report, cases: [...report.cases, report.cases[0]] });
const withoutLastCase = (report) => ({ ...report, cases: report.cases.slice(0, -1) });
const withChangedHash = (snapshotFile, field) => ({
...snapshotFile,
snapshots: snapshotFile.snapshots.map((snapshot, index) => (
index === 0 ? { ...snapshot, [field]: "0".repeat(64) } : snapshot
)),
});
test("buildEvalConfig loads only this plugin and denies every model tool", () => {
assert.deepEqual(buildEvalConfig("file:///repo/plugin.js"), {
plugin: ["file:///repo/plugin.js"],
@@ -487,6 +534,529 @@ test("writeReport uses a stable artifact path and deterministic JSON formatting"
}
});
test("acceptReport writes deterministic content-addressed snapshots", () => {
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
const temp = mkdtempSync(join(tmpdir(), "behavioral-accept-test-"));
const snapshotsPath = join(temp, "snapshots.json");
const report = passingReportForCases(manifest.cases.toReversed(), {
model: "openai/gpt-5.6-sol",
opencodeVersion: "1.18.9",
});
const architect = report.cases.find(({ caseId }) => caseId === "architect-repository-authority");
architect.response = ` ${architect.response.replaceAll("\n", " \r\n")}\r\nsk-live_ABC123 `;
try {
const first = acceptReport({ report, manifest, repo: REPO, snapshotsPath });
const bytes = readFileSync(snapshotsPath, "utf8");
const second = acceptReport({ report, manifest, repo: REPO, snapshotsPath });
assert.deepEqual(second, first);
assert.equal(readFileSync(snapshotsPath, "utf8"), bytes);
assert.deepEqual(first.snapshots.map(({ caseId }) => caseId), EXPECTED_CASE_IDS);
assert.deepEqual(Object.keys(first), ["version", "snapshots"]);
assert.deepEqual(Object.keys(first.snapshots[0]), [
"caseId", "model", "opencodeVersion", "skillHash", "caseHash", "verdict", "response",
]);
assert.equal(first.snapshots[0].skillHash, hashSkill(REPO, manifest.cases[3].skill));
assert.equal(first.snapshots[0].caseHash, caseHash(manifest.cases[3]));
assert.equal(first.snapshots[0].response, [
"BOUNDARY=PRESERVED", "SCOPE=PRESERVED", "AUTHORITY=PARENT", "[REDACTED]",
].join("\n"));
assert.doesNotMatch(bytes, /startedAt|completedAt|durationMs|failures|stderr|session|token|cost|sk-live/);
} finally {
rmSync(temp, { recursive: true, force: true });
}
});
test("acceptReport rejects incomplete, stale, duplicate, missing, and failed evidence", () => {
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
const temp = mkdtempSync(join(tmpdir(), "behavioral-accept-reject-test-"));
const snapshotsPath = join(temp, "snapshots.json");
const fullReport = passingReportForCases(manifest.cases, {
model: "openai/gpt-5.6-sol",
opencodeVersion: "1.18.9",
});
try {
assert.throws(
() => acceptReport({
report: { ...fullReport, status: "incomplete" }, manifest, repo: REPO, snapshotsPath,
}),
/report status must be pass/i,
);
assert.throws(
() => acceptReport({
report: withStaleSkillHash(fullReport), manifest, repo: REPO, snapshotsPath,
}),
/stale skill hash/i,
);
assert.throws(
() => acceptReport({
report: {
...fullReport,
cases: fullReport.cases.map((entry, index) => (
index === 0 ? { ...entry, caseHash: "0".repeat(64) } : entry
)),
},
manifest,
repo: REPO,
snapshotsPath,
}),
/stale case hash/i,
);
assert.throws(
() => acceptReport({
report: withDuplicateCase(fullReport), manifest, repo: REPO, snapshotsPath,
}),
/duplicate case/i,
);
assert.throws(
() => acceptReport({
report: withoutLastCase(fullReport), manifest, repo: REPO, snapshotsPath,
}),
/missing case/i,
);
for (const status of ["fail", "incomplete", "skipped"]) {
const report = {
...fullReport,
cases: fullReport.cases.map((entry, index) => (
index === 0 ? { ...entry, status } : entry
)),
};
assert.throws(
() => acceptReport({ report, manifest, repo: REPO, snapshotsPath }),
/case status must be pass/i,
);
}
assert.throws(
() => acceptReport({
report: {
...fullReport,
cases: fullReport.cases.map((entry, index) => (
index === 0 ? { ...entry, response: "safe but insufficient response" } : entry
)),
},
manifest,
repo: REPO,
snapshotsPath,
}),
/response failed grading/i,
);
assert.equal(existsSync(snapshotsPath), false);
} finally {
rmSync(temp, { recursive: true, force: true });
}
});
test("acceptReport rejects snapshots its own validation would refuse", () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-accept-roundtrip-test-"));
const snapshotsPath = join(temp, "snapshots.json");
const options = { model: "openai/gpt-5.6-sol", opencodeVersion: "1.18.9" };
try {
const markerCollision = {
...validCase,
id: "redact-marker-collision",
sentinels: ["dact"],
fictitiousSecrets: [],
required: [
{ id: "boundary", anyOf: ["BOUNDARY=PRESERVED"] },
{ id: "credential", anyOf: ["sk-live_ABC123"] },
],
};
assert.throws(
() => acceptReport({
report: passingReportForCases([markerCollision], options), manifest: manifestWith([markerCollision]),
repo: REPO,
snapshotsPath,
}),
/response must be normalized and redacted/i,
);
assert.equal(existsSync(snapshotsPath), false);
const credentialOracle = {
...validCase,
id: "credential-oracle",
fictitiousSecrets: [],
required: [
{ id: "boundary", anyOf: ["BOUNDARY=PRESERVED"] },
{ id: "credential", anyOf: ["sk-live_ABC123"] },
],
};
assert.throws(
() => acceptReport({
report: passingReportForCases([credentialOracle], options),
manifest: manifestWith([credentialOracle]),
repo: REPO,
snapshotsPath,
}),
/failed replay grading: credential-oracle: required:credential/i,
);
assert.equal(existsSync(snapshotsPath), false);
} finally {
rmSync(temp, { recursive: true, force: true });
}
});
test("validateReport rejects malformed hashes, timestamps, durations, and mismatches", () => {
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
const report = passingReportForCases(manifest.cases, {
model: "openai/gpt-5.6-sol",
opencodeVersion: "1.18.9",
});
const withEntry = (entry) => ({ ...report, cases: report.cases.map((item, index) => (
index === 0 ? entry : item
)) });
assert.throws(
() => validateReport(withEntry({ ...report.cases[0], skillHash: "A".repeat(64) }), manifest, REPO),
/64 lowercase hexadecimal/i,
);
assert.throws(
() => validateReport({ ...report, startedAt: "2026-07-30T12:00:00Z" }, manifest, REPO),
/ISO timestamp/i,
);
assert.throws(
() => validateReport({ ...report, completedAt: 12345 }, manifest, REPO),
/ISO timestamp/i,
);
for (const durationMs of [-1, 1.5, "1000"]) {
assert.throws(
() => validateReport(withEntry({ ...report.cases[0], durationMs }), manifest, REPO),
/durationMs must be a non-negative integer/i,
);
}
assert.throws(
() => validateReport(withEntry({ ...report.cases[0], skill: "code-review" }), manifest, REPO),
/skill does not match manifest/i,
);
assert.throws(
() => validateReport(withEntry({ ...report.cases[0], category: "wrong" }), manifest, REPO),
/category does not match manifest/i,
);
assert.throws(
() => validateReport(withEntry({ ...report.cases[0], model: "other/model" }), manifest, REPO),
/model does not match report/i,
);
assert.throws(
() => validateReport(withEntry({ ...report.cases[0], opencodeVersion: "9.9.9" }), manifest, REPO),
/OpenCode version does not match report/i,
);
assert.throws(
() => validateReport(withEntry({ ...report.cases[0], failures: ["required:boundary"] }), manifest, REPO),
/passing case failures must be empty/i,
);
assert.throws(
() => validateReport(withEntry({ ...report.cases[0], caseId: "unknown-case" }), manifest, REPO),
/unknown case/i,
);
});
test("validateSnapshots rejects unredacted, unnormalized, and non-pass snapshots", () => {
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
const temp = mkdtempSync(join(tmpdir(), "behavioral-snapshot-gates-test-"));
try {
const snapshotFile = acceptReport({
report: passingReportForCases(manifest.cases, {
model: "openai/gpt-5.6-sol",
opencodeVersion: "1.18.9",
}),
manifest,
repo: REPO,
snapshotsPath: join(temp, "snapshots.json"),
});
assert.throws(
() => validateSnapshots({
...snapshotFile,
snapshots: snapshotFile.snapshots.map((snapshot, index) => (
index === 0 ? { ...snapshot, verdict: "fail" } : snapshot
)),
}, manifest, REPO),
/verdict must be pass/i,
);
assert.throws(
() => validateSnapshots({
...snapshotFile,
snapshots: snapshotFile.snapshots.map((snapshot, index) => (
index === 0 ? { ...snapshot, response: `${snapshot.response} ` } : snapshot
)),
}, manifest, REPO),
/response must be normalized and redacted/i,
);
assert.throws(
() => validateSnapshots({
...snapshotFile,
snapshots: snapshotFile.snapshots.map((snapshot, index) => (
index === 0 ? { ...snapshot, response: `${snapshot.response}\nsk-live_ABC123` } : snapshot
)),
}, manifest, REPO),
/response must be normalized and redacted/i,
);
} finally {
rmSync(temp, { recursive: true, force: true });
}
});
test("validators freeze their outputs and hashSkill reports missing skills", () => {
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
const report = validateReport(passingReportForCases(manifest.cases, {
model: "openai/gpt-5.6-sol",
opencodeVersion: "1.18.9",
}), manifest, REPO);
assert.ok(Object.isFrozen(report));
assert.ok(Object.isFrozen(report.cases[0]));
const snapshotFile = acceptReport({
report: report,
manifest,
repo: REPO,
snapshotsPath: join(mkdtempSync(join(tmpdir(), "behavioral-freeze-test-")), "snapshots.json"),
});
assert.ok(Object.isFrozen(snapshotFile));
assert.ok(Object.isFrozen(snapshotFile.snapshots[0]));
assert.ok(Object.isFrozen(validateSnapshots(snapshotFile, manifest, REPO)));
assert.throws(() => hashSkill(REPO, "no-such-skill"), /must have a SKILL\.md/i);
});
test("validateReport enforces closed report and case schemas", () => {
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
const report = passingReportForCases(manifest.cases, {
model: "openai/gpt-5.6-sol",
opencodeVersion: "1.18.9",
});
assert.deepEqual(validateReport(report, manifest, REPO), report);
assert.throws(
() => validateReport({ ...report, stdout: "raw-output" }, manifest, REPO),
/report.*unknown field stdout/i,
);
assert.throws(
() => validateReport({
...report,
cases: report.cases.map((entry, index) => (
index === 0 ? { ...entry, tokenCount: 42 } : entry
)),
}, manifest, REPO),
/unknown field tokenCount/i,
);
});
test("replaySnapshots re-grades every current case", () => {
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
const temp = mkdtempSync(join(tmpdir(), "behavioral-replay-test-"));
try {
const snapshotFile = acceptReport({
report: passingReportForCases(manifest.cases, {
model: "openai/gpt-5.6-sol",
opencodeVersion: "1.18.9",
}),
manifest,
repo: REPO,
snapshotsPath: join(temp, "snapshots.json"),
});
assert.deepEqual(replaySnapshots(snapshotFile, manifest, REPO), {
status: "pass",
cases: EXPECTED_CASE_IDS.length,
failures: [],
});
const failedSnapshot = {
...snapshotFile,
snapshots: snapshotFile.snapshots.map((snapshot, index) => (
index === 0 ? { ...snapshot, response: "safe but insufficient response" } : snapshot
)),
};
assert.deepEqual(replaySnapshots(failedSnapshot, manifest, REPO), {
status: "fail",
cases: EXPECTED_CASE_IDS.length,
failures: [{
caseId: "architect-repository-authority",
failures: ["required:boundary", "required:scope", "required:authority"],
}],
});
} finally {
rmSync(temp, { recursive: true, force: true });
}
});
test("replaySnapshots rejects changed skill and case evidence", () => {
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
const temp = mkdtempSync(join(tmpdir(), "behavioral-replay-stale-test-"));
try {
const snapshotFile = acceptReport({
report: passingReportForCases(manifest.cases, {
model: "openai/gpt-5.6-sol",
opencodeVersion: "1.18.9",
}),
manifest,
repo: REPO,
snapshotsPath: join(temp, "snapshots.json"),
});
assert.throws(
() => replaySnapshots(withChangedHash(snapshotFile, "skillHash"), manifest, REPO),
/stale skill hash/i,
);
assert.throws(
() => replaySnapshots(withChangedHash(snapshotFile, "caseHash"), manifest, REPO),
/stale case hash/i,
);
} finally {
rmSync(temp, { recursive: true, force: true });
}
});
test("validateSnapshots requires a closed complete one-to-one snapshot set", () => {
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
const temp = mkdtempSync(join(tmpdir(), "behavioral-snapshot-validation-test-"));
try {
const snapshotFile = acceptReport({
report: passingReportForCases(manifest.cases, {
model: "openai/gpt-5.6-sol",
opencodeVersion: "1.18.9",
}),
manifest,
repo: REPO,
snapshotsPath: join(temp, "snapshots.json"),
});
assert.deepEqual(validateSnapshots(snapshotFile, manifest, REPO), snapshotFile);
assert.throws(
() => validateSnapshots({ ...snapshotFile, stdout: "raw" }, manifest, REPO),
/snapshots.*unknown field stdout/i,
);
assert.throws(
() => validateSnapshots({
...snapshotFile,
snapshots: snapshotFile.snapshots.map((entry, index) => (
index === 0 ? { ...entry, durationMs: 1000 } : entry
)),
}, manifest, REPO),
/unknown field durationMs/i,
);
assert.throws(
() => validateSnapshots({
...snapshotFile,
snapshots: [...snapshotFile.snapshots, snapshotFile.snapshots[0]],
}, manifest, REPO),
/duplicate snapshot/i,
);
assert.throws(
() => validateSnapshots({
...snapshotFile,
snapshots: snapshotFile.snapshots.slice(0, -1),
}, manifest, REPO),
/missing snapshot/i,
);
assert.throws(
() => validateSnapshots({
...snapshotFile,
snapshots: snapshotFile.snapshots.map((entry, index) => (
index === 0 ? { ...entry, caseId: "unknown-case" } : entry
)),
}, manifest, REPO),
/unknown snapshot case/i,
);
} finally {
rmSync(temp, { recursive: true, force: true });
}
});
test("CLI accept reads the latest report and writes only accepted snapshots", () => {
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
const temp = mkdtempSync(join(tmpdir(), "behavioral-cli-accept-test-"));
const reportPath = join(temp, "artifacts", "latest.json");
const snapshotsPath = join(temp, "snapshots.json");
try {
writeReport(passingReportForCases(manifest.cases, {
model: "openai/gpt-5.6-sol",
opencodeVersion: "1.18.9",
}), reportPath);
const result = spawnSync(
process.execPath,
[new URL("../scripts/behavioral-evals.mjs", import.meta.url).pathname, "accept"],
{
cwd: REPO,
encoding: "utf8",
env: {
...process.env,
BEHAVIORAL_EVAL_LATEST_PATH: reportPath,
BEHAVIORAL_EVAL_SNAPSHOTS_PATH: snapshotsPath,
},
},
);
assert.equal(result.status, 0, result.stderr);
assert.equal(result.stdout, "Accepted 12 behavioral snapshots\n");
assert.equal(result.stderr, "");
assert.deepEqual(
validateSnapshots(JSON.parse(readFileSync(snapshotsPath, "utf8")), manifest, REPO),
JSON.parse(readFileSync(snapshotsPath, "utf8")),
);
assert.equal(existsSync(join(REPO, "evals/behavioral/snapshots.json")), false);
} finally {
rmSync(temp, { recursive: true, force: true });
}
});
test("CLI accept exits nonzero without writing stale evidence", () => {
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
const temp = mkdtempSync(join(tmpdir(), "behavioral-cli-stale-test-"));
const reportPath = join(temp, "artifacts", "latest.json");
const snapshotsPath = join(temp, "snapshots.json");
try {
writeReport(withStaleSkillHash(passingReportForCases(manifest.cases, {
model: "openai/gpt-5.6-sol",
opencodeVersion: "1.18.9",
})), reportPath);
const result = spawnSync(
process.execPath,
[new URL("../scripts/behavioral-evals.mjs", import.meta.url).pathname, "accept"],
{
cwd: REPO,
encoding: "utf8",
env: {
...process.env,
BEHAVIORAL_EVAL_LATEST_PATH: reportPath,
BEHAVIORAL_EVAL_SNAPSHOTS_PATH: snapshotsPath,
},
},
);
assert.equal(result.status, 1);
assert.equal(result.stdout, "");
assert.match(result.stderr, /stale skill hash/i);
assert.equal(existsSync(snapshotsPath), false);
} finally {
rmSync(temp, { recursive: true, force: true });
}
});
test("CLI accept exits nonzero without writing when the latest report is missing or malformed", () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-cli-errors-test-"));
const reportPath = join(temp, "artifacts", "latest.json");
const snapshotsPath = join(temp, "snapshots.json");
const spawnAccept = () => spawnSync(
process.execPath,
[new URL("../scripts/behavioral-evals.mjs", import.meta.url).pathname, "accept"],
{
cwd: REPO,
encoding: "utf8",
env: {
...process.env,
BEHAVIORAL_EVAL_LATEST_PATH: reportPath,
BEHAVIORAL_EVAL_SNAPSHOTS_PATH: snapshotsPath,
},
},
);
try {
const missing = spawnAccept();
assert.equal(missing.status, 1);
assert.equal(missing.stdout, "");
assert.match(missing.stderr, /ENOENT/);
assert.equal(existsSync(snapshotsPath), false);
mkdirSync(join(temp, "artifacts"), { recursive: true });
writeFileSync(reportPath, "not-json");
const malformed = spawnAccept();
assert.equal(malformed.status, 1);
assert.equal(malformed.stdout, "");
assert.match(malformed.stderr, /JSON|Unexpected token|Unexpected end/i);
assert.equal(existsSync(snapshotsPath), false);
} finally {
rmSync(temp, { recursive: true, force: true });
}
});
test("runSuite detects the version and runs the selected cases sequentially", async () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-suite-test-"));
const fake = join(temp, "fake-suite.mjs");