mirror of
https://github.com/waybarrios/opencode-power-pack.git
synced 2026-11-18 18:03:43 +00:00
Add snapshot acceptance and replay to behavioral evaluations
This commit is contained in:
@@ -7,6 +7,7 @@ import {
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
renameSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
@@ -23,6 +24,17 @@ const CASE_FIELDS = new Set([
|
||||
const REQUIRED_FIELDS = new Set(["id", "anyOf"]);
|
||||
const FORBIDDEN_FIELDS = new Set(["id", "values"]);
|
||||
const MUTATION_FIELDS = new Set(["id", "caseId", "operation", "from", "value", "expectedFailure"]);
|
||||
const REPORT_FIELDS = new Set([
|
||||
"version", "model", "opencodeVersion", "startedAt", "completedAt", "status", "cases",
|
||||
]);
|
||||
const REPORT_CASE_FIELDS = new Set([
|
||||
"caseId", "skill", "category", "status", "failures", "response", "model",
|
||||
"opencodeVersion", "skillHash", "caseHash", "durationMs", "completedAt",
|
||||
]);
|
||||
const SNAPSHOT_FILE_FIELDS = new Set(["version", "snapshots"]);
|
||||
const SNAPSHOT_FIELDS = new Set([
|
||||
"caseId", "model", "opencodeVersion", "skillHash", "caseHash", "verdict", "response",
|
||||
]);
|
||||
|
||||
export function sha256(value) {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
@@ -40,6 +52,12 @@ export function caseHash(testCase) {
|
||||
return sha256(JSON.stringify(canonical(testCase)));
|
||||
}
|
||||
|
||||
export function hashSkill(repo, skill) {
|
||||
const skillPath = path.join(repo, "skills", skill, "SKILL.md");
|
||||
if (!existsSync(skillPath)) throw new TypeError(`skill ${skill} must have a SKILL.md`);
|
||||
return sha256(readFileSync(skillPath));
|
||||
}
|
||||
|
||||
function isObject(value) {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
@@ -265,6 +283,204 @@ export function gradeResponse(testCase, response) {
|
||||
return { status: failures.length === 0 ? "pass" : "fail", failures };
|
||||
}
|
||||
|
||||
function assertHash(value, label) {
|
||||
if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) {
|
||||
throw new TypeError(`${label} must be 64 lowercase hexadecimal characters`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertTimestamp(value, label) {
|
||||
const parsed = typeof value === "string" ? new Date(value) : undefined;
|
||||
if (!parsed || Number.isNaN(parsed.getTime()) || parsed.toISOString() !== value) {
|
||||
throw new TypeError(`${label} must be an ISO timestamp`);
|
||||
}
|
||||
}
|
||||
|
||||
export function validateReport(value, manifest, repo) {
|
||||
assertObject(value, "report");
|
||||
assertFields(value, REPORT_FIELDS, "report");
|
||||
if (value.version !== 1) throw new TypeError("report.version must equal 1");
|
||||
assertNonEmptyString(value.model, "report.model");
|
||||
assertNonEmptyString(value.opencodeVersion, "report.opencodeVersion");
|
||||
assertTimestamp(value.startedAt, "report.startedAt");
|
||||
assertTimestamp(value.completedAt, "report.completedAt");
|
||||
if (value.status !== "pass") throw new TypeError("report status must be pass");
|
||||
if (!Array.isArray(value.cases)) throw new TypeError("report.cases must be an array");
|
||||
|
||||
const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase]));
|
||||
const seen = new Set();
|
||||
const cases = value.cases.map((entry, index) => {
|
||||
const label = isObject(entry) && typeof entry.caseId === "string" && entry.caseId !== ""
|
||||
? entry.caseId
|
||||
: `report.cases[${index}]`;
|
||||
assertObject(entry, label);
|
||||
assertFields(entry, REPORT_CASE_FIELDS, label);
|
||||
assertNonEmptyString(entry.caseId, `${label}.caseId`);
|
||||
if (seen.has(entry.caseId)) throw new TypeError(`duplicate case ${entry.caseId}`);
|
||||
seen.add(entry.caseId);
|
||||
const testCase = currentCases.get(entry.caseId);
|
||||
if (!testCase) throw new TypeError(`unknown case ${entry.caseId}`);
|
||||
|
||||
assertNonEmptyString(entry.skill, `${label}.skill`);
|
||||
assertNonEmptyString(entry.category, `${label}.category`);
|
||||
if (entry.skill !== testCase.skill) throw new TypeError(`${label}: skill does not match manifest`);
|
||||
if (entry.category !== testCase.category) {
|
||||
throw new TypeError(`${label}: category does not match manifest`);
|
||||
}
|
||||
if (entry.status !== "pass") throw new TypeError(`${label}: case status must be pass`);
|
||||
if (!Array.isArray(entry.failures) || entry.failures.length !== 0) {
|
||||
throw new TypeError(`${label}: passing case failures must be empty`);
|
||||
}
|
||||
assertNonEmptyString(entry.response, `${label}.response`);
|
||||
assertNonEmptyString(entry.model, `${label}.model`);
|
||||
assertNonEmptyString(entry.opencodeVersion, `${label}.opencodeVersion`);
|
||||
if (entry.model !== value.model) throw new TypeError(`${label}: model does not match report`);
|
||||
if (entry.opencodeVersion !== value.opencodeVersion) {
|
||||
throw new TypeError(`${label}: OpenCode version does not match report`);
|
||||
}
|
||||
assertHash(entry.skillHash, `${label}.skillHash`);
|
||||
assertHash(entry.caseHash, `${label}.caseHash`);
|
||||
if (entry.skillHash !== hashSkill(repo, testCase.skill)) {
|
||||
throw new TypeError(`${label}: stale skill hash`);
|
||||
}
|
||||
if (entry.caseHash !== caseHash(testCase)) throw new TypeError(`${label}: stale case hash`);
|
||||
if (!Number.isInteger(entry.durationMs) || entry.durationMs < 0) {
|
||||
throw new TypeError(`${label}.durationMs must be a non-negative integer`);
|
||||
}
|
||||
assertTimestamp(entry.completedAt, `${label}.completedAt`);
|
||||
const grade = gradeResponse(testCase, entry.response);
|
||||
if (grade.status !== "pass") {
|
||||
throw new TypeError(`${label}: response failed grading (${grade.failures.join(", ")})`);
|
||||
}
|
||||
|
||||
return {
|
||||
caseId: entry.caseId,
|
||||
skill: entry.skill,
|
||||
category: entry.category,
|
||||
status: entry.status,
|
||||
failures: [],
|
||||
response: entry.response,
|
||||
model: entry.model,
|
||||
opencodeVersion: entry.opencodeVersion,
|
||||
skillHash: entry.skillHash,
|
||||
caseHash: entry.caseHash,
|
||||
durationMs: entry.durationMs,
|
||||
completedAt: entry.completedAt,
|
||||
};
|
||||
});
|
||||
|
||||
for (const testCase of manifest.cases) {
|
||||
if (!seen.has(testCase.id)) throw new TypeError(`missing case ${testCase.id}`);
|
||||
}
|
||||
return deepFreeze({
|
||||
version: 1,
|
||||
model: value.model,
|
||||
opencodeVersion: value.opencodeVersion,
|
||||
startedAt: value.startedAt,
|
||||
completedAt: value.completedAt,
|
||||
status: "pass",
|
||||
cases,
|
||||
});
|
||||
}
|
||||
|
||||
export function acceptReport({ report, manifest, repo, snapshotsPath }) {
|
||||
const validated = validateReport(report, manifest, repo);
|
||||
const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase]));
|
||||
const snapshots = validated.cases.map((entry) => {
|
||||
const testCase = currentCases.get(entry.caseId);
|
||||
return {
|
||||
caseId: entry.caseId,
|
||||
model: entry.model,
|
||||
opencodeVersion: entry.opencodeVersion,
|
||||
skillHash: entry.skillHash,
|
||||
caseHash: entry.caseHash,
|
||||
verdict: "pass",
|
||||
response: normalizeResponse(redactResponse(entry.response, testCase)),
|
||||
};
|
||||
}).sort((left, right) => (left.caseId < right.caseId ? -1 : left.caseId > right.caseId ? 1 : 0));
|
||||
const snapshotFile = deepFreeze({ version: 1, snapshots });
|
||||
const replayed = replaySnapshots(snapshotFile, manifest, repo);
|
||||
if (replayed.status !== "pass") {
|
||||
const detail = replayed.failures
|
||||
.map(({ caseId, failures }) => `${caseId}: ${failures.join(", ")}`)
|
||||
.join("; ");
|
||||
throw new TypeError(`snapshot responses failed replay grading: ${detail}`);
|
||||
}
|
||||
mkdirSync(path.dirname(snapshotsPath), { recursive: true });
|
||||
writeFileSync(snapshotsPath, `${JSON.stringify(snapshotFile, null, 2)}\n`);
|
||||
return snapshotFile;
|
||||
}
|
||||
|
||||
export function validateSnapshots(value, manifest, repo) {
|
||||
assertObject(value, "snapshots");
|
||||
assertFields(value, SNAPSHOT_FILE_FIELDS, "snapshots");
|
||||
if (value.version !== 1) throw new TypeError("snapshots.version must equal 1");
|
||||
if (!Array.isArray(value.snapshots)) throw new TypeError("snapshots.snapshots must be an array");
|
||||
|
||||
const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase]));
|
||||
const seen = new Set();
|
||||
const snapshots = value.snapshots.map((snapshot, index) => {
|
||||
const label = isObject(snapshot) && typeof snapshot.caseId === "string" && snapshot.caseId !== ""
|
||||
? snapshot.caseId
|
||||
: `snapshots[${index}]`;
|
||||
assertObject(snapshot, label);
|
||||
assertFields(snapshot, SNAPSHOT_FIELDS, label);
|
||||
assertNonEmptyString(snapshot.caseId, `${label}.caseId`);
|
||||
if (seen.has(snapshot.caseId)) throw new TypeError(`duplicate snapshot ${snapshot.caseId}`);
|
||||
seen.add(snapshot.caseId);
|
||||
const testCase = currentCases.get(snapshot.caseId);
|
||||
if (!testCase) throw new TypeError(`unknown snapshot case ${snapshot.caseId}`);
|
||||
|
||||
assertNonEmptyString(snapshot.model, `${label}.model`);
|
||||
assertNonEmptyString(snapshot.opencodeVersion, `${label}.opencodeVersion`);
|
||||
assertHash(snapshot.skillHash, `${label}.skillHash`);
|
||||
assertHash(snapshot.caseHash, `${label}.caseHash`);
|
||||
if (snapshot.skillHash !== hashSkill(repo, testCase.skill)) {
|
||||
throw new TypeError(`${label}: stale skill hash`);
|
||||
}
|
||||
if (snapshot.caseHash !== caseHash(testCase)) throw new TypeError(`${label}: stale case hash`);
|
||||
if (snapshot.verdict !== "pass") throw new TypeError(`${label}: verdict must be pass`);
|
||||
assertNonEmptyString(snapshot.response, `${label}.response`);
|
||||
if (normalizeResponse(redactResponse(snapshot.response, testCase)) !== snapshot.response) {
|
||||
throw new TypeError(`${label}: response must be normalized and redacted`);
|
||||
}
|
||||
return {
|
||||
caseId: snapshot.caseId,
|
||||
model: snapshot.model,
|
||||
opencodeVersion: snapshot.opencodeVersion,
|
||||
skillHash: snapshot.skillHash,
|
||||
caseHash: snapshot.caseHash,
|
||||
verdict: "pass",
|
||||
response: snapshot.response,
|
||||
};
|
||||
});
|
||||
|
||||
for (const testCase of manifest.cases) {
|
||||
if (!seen.has(testCase.id)) throw new TypeError(`missing snapshot ${testCase.id}`);
|
||||
}
|
||||
snapshots.sort((left, right) => (
|
||||
left.caseId < right.caseId ? -1 : left.caseId > right.caseId ? 1 : 0
|
||||
));
|
||||
return deepFreeze({ version: 1, snapshots });
|
||||
}
|
||||
|
||||
export function replaySnapshots(snapshotFile, manifest, repo) {
|
||||
const validated = validateSnapshots(snapshotFile, manifest, repo);
|
||||
const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase]));
|
||||
const failures = [];
|
||||
for (const snapshot of validated.snapshots) {
|
||||
const grade = gradeResponse(currentCases.get(snapshot.caseId), snapshot.response);
|
||||
if (grade.status === "fail") {
|
||||
failures.push({ caseId: snapshot.caseId, failures: grade.failures });
|
||||
}
|
||||
}
|
||||
return deepFreeze({
|
||||
status: failures.length === 0 ? "pass" : "fail",
|
||||
cases: validated.snapshots.length,
|
||||
failures,
|
||||
});
|
||||
}
|
||||
|
||||
export function applyMutation(response, mutation) {
|
||||
if (!isObject(mutation) || typeof mutation.value !== "string") {
|
||||
throw new TypeError("mutation must contain a string value");
|
||||
@@ -377,7 +593,7 @@ function reportEntry(testCase, options, started, status, failures, response) {
|
||||
response,
|
||||
model: options.model,
|
||||
opencodeVersion: options.opencodeVersion,
|
||||
skillHash: sha256(readFileSync(path.join(options.repo, "skills", testCase.skill, "SKILL.md"))),
|
||||
skillHash: hashSkill(options.repo, testCase.skill),
|
||||
caseHash: caseHash(testCase),
|
||||
durationMs: Date.now() - started,
|
||||
completedAt: new Date().toISOString(),
|
||||
@@ -580,7 +796,9 @@ export function writeReport(
|
||||
outputPath = path.resolve(".artifacts/behavioral-evals/latest.json"),
|
||||
) {
|
||||
mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||
writeFileSync(outputPath, `${JSON.stringify(report, null, 2)}\n`);
|
||||
const tempPath = `${outputPath}.tmp`;
|
||||
writeFileSync(tempPath, `${JSON.stringify(report, null, 2)}\n`);
|
||||
renameSync(tempPath, outputPath);
|
||||
return outputPath;
|
||||
}
|
||||
|
||||
@@ -645,16 +863,29 @@ export async function runSuite(options) {
|
||||
|
||||
async function main() {
|
||||
const mode = process.argv[2];
|
||||
if (mode !== "run") throw new TypeError(`unsupported behavioral evaluation mode: ${mode || "missing"}`);
|
||||
if (mode !== "run" && mode !== "accept") {
|
||||
throw new TypeError(`unsupported behavioral evaluation mode: ${mode || "missing"}`);
|
||||
}
|
||||
const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const manifest = validateManifest(JSON.parse(
|
||||
readFileSync(path.join(repo, "evals/behavioral/cases.json"), "utf8"),
|
||||
));
|
||||
const latestPath = process.env.BEHAVIORAL_EVAL_LATEST_PATH
|
||||
|| path.join(repo, ".artifacts/behavioral-evals/latest.json");
|
||||
const snapshotsPath = process.env.BEHAVIORAL_EVAL_SNAPSHOTS_PATH
|
||||
|| path.join(repo, "evals/behavioral/snapshots.json");
|
||||
if (mode === "accept") {
|
||||
const snapshotFile = acceptReport({
|
||||
report: JSON.parse(readFileSync(latestPath, "utf8")),
|
||||
manifest,
|
||||
repo,
|
||||
snapshotsPath,
|
||||
});
|
||||
console.log(`Accepted ${snapshotFile.snapshots.length} behavioral snapshots`);
|
||||
return;
|
||||
}
|
||||
const report = await runSuite({ repo, cases: manifest.cases });
|
||||
const outputPath = writeReport(
|
||||
report,
|
||||
path.join(repo, ".artifacts/behavioral-evals/latest.json"),
|
||||
);
|
||||
const outputPath = writeReport(report, latestPath);
|
||||
const passed = report.cases.filter(({ status }) => status === "pass").length;
|
||||
console.log(outputPath);
|
||||
console.log(`${passed}/${report.cases.length} cases passed`);
|
||||
|
||||
@@ -7,19 +7,24 @@ import {
|
||||
import { tmpdir } from "node:os";
|
||||
import { delimiter, join } from "node:path";
|
||||
import {
|
||||
acceptReport,
|
||||
applyMutation,
|
||||
buildEvalConfig,
|
||||
caseHash,
|
||||
defaultOpenCodeCommand,
|
||||
gradeResponse,
|
||||
hashSkill,
|
||||
normalizeResponse,
|
||||
parseJsonEvents,
|
||||
redactResponse,
|
||||
replaySnapshots,
|
||||
runCase,
|
||||
runSuite,
|
||||
sha256,
|
||||
validateManifest,
|
||||
validateMutations,
|
||||
validateReport,
|
||||
validateSnapshots,
|
||||
writeReport,
|
||||
} from "../scripts/behavioral-evals.mjs";
|
||||
|
||||
@@ -113,6 +118,48 @@ function manifestWith(testCases = [validCase], targetSkills = ["feature-dev"]) {
|
||||
return { version: 1, targetSkills, cases: testCases };
|
||||
}
|
||||
|
||||
function passingReportForCases(cases, { model, opencodeVersion }) {
|
||||
const startedAt = "2026-07-30T12:00:00.000Z";
|
||||
const completedAt = "2026-07-30T12:01:00.000Z";
|
||||
return {
|
||||
version: 1,
|
||||
model,
|
||||
opencodeVersion,
|
||||
startedAt,
|
||||
completedAt,
|
||||
status: "pass",
|
||||
cases: cases.map((testCase) => ({
|
||||
caseId: testCase.id,
|
||||
skill: testCase.skill,
|
||||
category: testCase.category,
|
||||
status: "pass",
|
||||
failures: [],
|
||||
response: testCase.required.map(({ anyOf }) => anyOf[0]).join("\n"),
|
||||
model,
|
||||
opencodeVersion,
|
||||
skillHash: hashSkill(REPO, testCase.skill),
|
||||
caseHash: caseHash(testCase),
|
||||
durationMs: 1000,
|
||||
completedAt,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
const withStaleSkillHash = (report) => ({
|
||||
...report,
|
||||
cases: report.cases.map((entry, index) => (
|
||||
index === 0 ? { ...entry, skillHash: "0".repeat(64) } : entry
|
||||
)),
|
||||
});
|
||||
const withDuplicateCase = (report) => ({ ...report, cases: [...report.cases, report.cases[0]] });
|
||||
const withoutLastCase = (report) => ({ ...report, cases: report.cases.slice(0, -1) });
|
||||
const withChangedHash = (snapshotFile, field) => ({
|
||||
...snapshotFile,
|
||||
snapshots: snapshotFile.snapshots.map((snapshot, index) => (
|
||||
index === 0 ? { ...snapshot, [field]: "0".repeat(64) } : snapshot
|
||||
)),
|
||||
});
|
||||
|
||||
test("buildEvalConfig loads only this plugin and denies every model tool", () => {
|
||||
assert.deepEqual(buildEvalConfig("file:///repo/plugin.js"), {
|
||||
plugin: ["file:///repo/plugin.js"],
|
||||
@@ -487,6 +534,529 @@ test("writeReport uses a stable artifact path and deterministic JSON formatting"
|
||||
}
|
||||
});
|
||||
|
||||
test("acceptReport writes deterministic content-addressed snapshots", () => {
|
||||
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-accept-test-"));
|
||||
const snapshotsPath = join(temp, "snapshots.json");
|
||||
const report = passingReportForCases(manifest.cases.toReversed(), {
|
||||
model: "openai/gpt-5.6-sol",
|
||||
opencodeVersion: "1.18.9",
|
||||
});
|
||||
const architect = report.cases.find(({ caseId }) => caseId === "architect-repository-authority");
|
||||
architect.response = ` ${architect.response.replaceAll("\n", " \r\n")}\r\nsk-live_ABC123 `;
|
||||
try {
|
||||
const first = acceptReport({ report, manifest, repo: REPO, snapshotsPath });
|
||||
const bytes = readFileSync(snapshotsPath, "utf8");
|
||||
const second = acceptReport({ report, manifest, repo: REPO, snapshotsPath });
|
||||
assert.deepEqual(second, first);
|
||||
assert.equal(readFileSync(snapshotsPath, "utf8"), bytes);
|
||||
assert.deepEqual(first.snapshots.map(({ caseId }) => caseId), EXPECTED_CASE_IDS);
|
||||
assert.deepEqual(Object.keys(first), ["version", "snapshots"]);
|
||||
assert.deepEqual(Object.keys(first.snapshots[0]), [
|
||||
"caseId", "model", "opencodeVersion", "skillHash", "caseHash", "verdict", "response",
|
||||
]);
|
||||
assert.equal(first.snapshots[0].skillHash, hashSkill(REPO, manifest.cases[3].skill));
|
||||
assert.equal(first.snapshots[0].caseHash, caseHash(manifest.cases[3]));
|
||||
assert.equal(first.snapshots[0].response, [
|
||||
"BOUNDARY=PRESERVED", "SCOPE=PRESERVED", "AUTHORITY=PARENT", "[REDACTED]",
|
||||
].join("\n"));
|
||||
assert.doesNotMatch(bytes, /startedAt|completedAt|durationMs|failures|stderr|session|token|cost|sk-live/);
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("acceptReport rejects incomplete, stale, duplicate, missing, and failed evidence", () => {
|
||||
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-accept-reject-test-"));
|
||||
const snapshotsPath = join(temp, "snapshots.json");
|
||||
const fullReport = passingReportForCases(manifest.cases, {
|
||||
model: "openai/gpt-5.6-sol",
|
||||
opencodeVersion: "1.18.9",
|
||||
});
|
||||
try {
|
||||
assert.throws(
|
||||
() => acceptReport({
|
||||
report: { ...fullReport, status: "incomplete" }, manifest, repo: REPO, snapshotsPath,
|
||||
}),
|
||||
/report status must be pass/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => acceptReport({
|
||||
report: withStaleSkillHash(fullReport), manifest, repo: REPO, snapshotsPath,
|
||||
}),
|
||||
/stale skill hash/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => acceptReport({
|
||||
report: {
|
||||
...fullReport,
|
||||
cases: fullReport.cases.map((entry, index) => (
|
||||
index === 0 ? { ...entry, caseHash: "0".repeat(64) } : entry
|
||||
)),
|
||||
},
|
||||
manifest,
|
||||
repo: REPO,
|
||||
snapshotsPath,
|
||||
}),
|
||||
/stale case hash/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => acceptReport({
|
||||
report: withDuplicateCase(fullReport), manifest, repo: REPO, snapshotsPath,
|
||||
}),
|
||||
/duplicate case/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => acceptReport({
|
||||
report: withoutLastCase(fullReport), manifest, repo: REPO, snapshotsPath,
|
||||
}),
|
||||
/missing case/i,
|
||||
);
|
||||
for (const status of ["fail", "incomplete", "skipped"]) {
|
||||
const report = {
|
||||
...fullReport,
|
||||
cases: fullReport.cases.map((entry, index) => (
|
||||
index === 0 ? { ...entry, status } : entry
|
||||
)),
|
||||
};
|
||||
assert.throws(
|
||||
() => acceptReport({ report, manifest, repo: REPO, snapshotsPath }),
|
||||
/case status must be pass/i,
|
||||
);
|
||||
}
|
||||
assert.throws(
|
||||
() => acceptReport({
|
||||
report: {
|
||||
...fullReport,
|
||||
cases: fullReport.cases.map((entry, index) => (
|
||||
index === 0 ? { ...entry, response: "safe but insufficient response" } : entry
|
||||
)),
|
||||
},
|
||||
manifest,
|
||||
repo: REPO,
|
||||
snapshotsPath,
|
||||
}),
|
||||
/response failed grading/i,
|
||||
);
|
||||
assert.equal(existsSync(snapshotsPath), false);
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("acceptReport rejects snapshots its own validation would refuse", () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-accept-roundtrip-test-"));
|
||||
const snapshotsPath = join(temp, "snapshots.json");
|
||||
const options = { model: "openai/gpt-5.6-sol", opencodeVersion: "1.18.9" };
|
||||
try {
|
||||
const markerCollision = {
|
||||
...validCase,
|
||||
id: "redact-marker-collision",
|
||||
sentinels: ["dact"],
|
||||
fictitiousSecrets: [],
|
||||
required: [
|
||||
{ id: "boundary", anyOf: ["BOUNDARY=PRESERVED"] },
|
||||
{ id: "credential", anyOf: ["sk-live_ABC123"] },
|
||||
],
|
||||
};
|
||||
assert.throws(
|
||||
() => acceptReport({
|
||||
report: passingReportForCases([markerCollision], options), manifest: manifestWith([markerCollision]),
|
||||
repo: REPO,
|
||||
snapshotsPath,
|
||||
}),
|
||||
/response must be normalized and redacted/i,
|
||||
);
|
||||
assert.equal(existsSync(snapshotsPath), false);
|
||||
|
||||
const credentialOracle = {
|
||||
...validCase,
|
||||
id: "credential-oracle",
|
||||
fictitiousSecrets: [],
|
||||
required: [
|
||||
{ id: "boundary", anyOf: ["BOUNDARY=PRESERVED"] },
|
||||
{ id: "credential", anyOf: ["sk-live_ABC123"] },
|
||||
],
|
||||
};
|
||||
assert.throws(
|
||||
() => acceptReport({
|
||||
report: passingReportForCases([credentialOracle], options),
|
||||
manifest: manifestWith([credentialOracle]),
|
||||
repo: REPO,
|
||||
snapshotsPath,
|
||||
}),
|
||||
/failed replay grading: credential-oracle: required:credential/i,
|
||||
);
|
||||
assert.equal(existsSync(snapshotsPath), false);
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("validateReport rejects malformed hashes, timestamps, durations, and mismatches", () => {
|
||||
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
|
||||
const report = passingReportForCases(manifest.cases, {
|
||||
model: "openai/gpt-5.6-sol",
|
||||
opencodeVersion: "1.18.9",
|
||||
});
|
||||
const withEntry = (entry) => ({ ...report, cases: report.cases.map((item, index) => (
|
||||
index === 0 ? entry : item
|
||||
)) });
|
||||
assert.throws(
|
||||
() => validateReport(withEntry({ ...report.cases[0], skillHash: "A".repeat(64) }), manifest, REPO),
|
||||
/64 lowercase hexadecimal/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => validateReport({ ...report, startedAt: "2026-07-30T12:00:00Z" }, manifest, REPO),
|
||||
/ISO timestamp/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => validateReport({ ...report, completedAt: 12345 }, manifest, REPO),
|
||||
/ISO timestamp/i,
|
||||
);
|
||||
for (const durationMs of [-1, 1.5, "1000"]) {
|
||||
assert.throws(
|
||||
() => validateReport(withEntry({ ...report.cases[0], durationMs }), manifest, REPO),
|
||||
/durationMs must be a non-negative integer/i,
|
||||
);
|
||||
}
|
||||
assert.throws(
|
||||
() => validateReport(withEntry({ ...report.cases[0], skill: "code-review" }), manifest, REPO),
|
||||
/skill does not match manifest/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => validateReport(withEntry({ ...report.cases[0], category: "wrong" }), manifest, REPO),
|
||||
/category does not match manifest/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => validateReport(withEntry({ ...report.cases[0], model: "other/model" }), manifest, REPO),
|
||||
/model does not match report/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => validateReport(withEntry({ ...report.cases[0], opencodeVersion: "9.9.9" }), manifest, REPO),
|
||||
/OpenCode version does not match report/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => validateReport(withEntry({ ...report.cases[0], failures: ["required:boundary"] }), manifest, REPO),
|
||||
/passing case failures must be empty/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => validateReport(withEntry({ ...report.cases[0], caseId: "unknown-case" }), manifest, REPO),
|
||||
/unknown case/i,
|
||||
);
|
||||
});
|
||||
|
||||
test("validateSnapshots rejects unredacted, unnormalized, and non-pass snapshots", () => {
|
||||
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-snapshot-gates-test-"));
|
||||
try {
|
||||
const snapshotFile = acceptReport({
|
||||
report: passingReportForCases(manifest.cases, {
|
||||
model: "openai/gpt-5.6-sol",
|
||||
opencodeVersion: "1.18.9",
|
||||
}),
|
||||
manifest,
|
||||
repo: REPO,
|
||||
snapshotsPath: join(temp, "snapshots.json"),
|
||||
});
|
||||
assert.throws(
|
||||
() => validateSnapshots({
|
||||
...snapshotFile,
|
||||
snapshots: snapshotFile.snapshots.map((snapshot, index) => (
|
||||
index === 0 ? { ...snapshot, verdict: "fail" } : snapshot
|
||||
)),
|
||||
}, manifest, REPO),
|
||||
/verdict must be pass/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => validateSnapshots({
|
||||
...snapshotFile,
|
||||
snapshots: snapshotFile.snapshots.map((snapshot, index) => (
|
||||
index === 0 ? { ...snapshot, response: `${snapshot.response} ` } : snapshot
|
||||
)),
|
||||
}, manifest, REPO),
|
||||
/response must be normalized and redacted/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => validateSnapshots({
|
||||
...snapshotFile,
|
||||
snapshots: snapshotFile.snapshots.map((snapshot, index) => (
|
||||
index === 0 ? { ...snapshot, response: `${snapshot.response}\nsk-live_ABC123` } : snapshot
|
||||
)),
|
||||
}, manifest, REPO),
|
||||
/response must be normalized and redacted/i,
|
||||
);
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("validators freeze their outputs and hashSkill reports missing skills", () => {
|
||||
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
|
||||
const report = validateReport(passingReportForCases(manifest.cases, {
|
||||
model: "openai/gpt-5.6-sol",
|
||||
opencodeVersion: "1.18.9",
|
||||
}), manifest, REPO);
|
||||
assert.ok(Object.isFrozen(report));
|
||||
assert.ok(Object.isFrozen(report.cases[0]));
|
||||
const snapshotFile = acceptReport({
|
||||
report: report,
|
||||
manifest,
|
||||
repo: REPO,
|
||||
snapshotsPath: join(mkdtempSync(join(tmpdir(), "behavioral-freeze-test-")), "snapshots.json"),
|
||||
});
|
||||
assert.ok(Object.isFrozen(snapshotFile));
|
||||
assert.ok(Object.isFrozen(snapshotFile.snapshots[0]));
|
||||
assert.ok(Object.isFrozen(validateSnapshots(snapshotFile, manifest, REPO)));
|
||||
assert.throws(() => hashSkill(REPO, "no-such-skill"), /must have a SKILL\.md/i);
|
||||
});
|
||||
|
||||
test("validateReport enforces closed report and case schemas", () => {
|
||||
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
|
||||
const report = passingReportForCases(manifest.cases, {
|
||||
model: "openai/gpt-5.6-sol",
|
||||
opencodeVersion: "1.18.9",
|
||||
});
|
||||
assert.deepEqual(validateReport(report, manifest, REPO), report);
|
||||
assert.throws(
|
||||
() => validateReport({ ...report, stdout: "raw-output" }, manifest, REPO),
|
||||
/report.*unknown field stdout/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => validateReport({
|
||||
...report,
|
||||
cases: report.cases.map((entry, index) => (
|
||||
index === 0 ? { ...entry, tokenCount: 42 } : entry
|
||||
)),
|
||||
}, manifest, REPO),
|
||||
/unknown field tokenCount/i,
|
||||
);
|
||||
});
|
||||
|
||||
test("replaySnapshots re-grades every current case", () => {
|
||||
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-replay-test-"));
|
||||
try {
|
||||
const snapshotFile = acceptReport({
|
||||
report: passingReportForCases(manifest.cases, {
|
||||
model: "openai/gpt-5.6-sol",
|
||||
opencodeVersion: "1.18.9",
|
||||
}),
|
||||
manifest,
|
||||
repo: REPO,
|
||||
snapshotsPath: join(temp, "snapshots.json"),
|
||||
});
|
||||
assert.deepEqual(replaySnapshots(snapshotFile, manifest, REPO), {
|
||||
status: "pass",
|
||||
cases: EXPECTED_CASE_IDS.length,
|
||||
failures: [],
|
||||
});
|
||||
|
||||
const failedSnapshot = {
|
||||
...snapshotFile,
|
||||
snapshots: snapshotFile.snapshots.map((snapshot, index) => (
|
||||
index === 0 ? { ...snapshot, response: "safe but insufficient response" } : snapshot
|
||||
)),
|
||||
};
|
||||
assert.deepEqual(replaySnapshots(failedSnapshot, manifest, REPO), {
|
||||
status: "fail",
|
||||
cases: EXPECTED_CASE_IDS.length,
|
||||
failures: [{
|
||||
caseId: "architect-repository-authority",
|
||||
failures: ["required:boundary", "required:scope", "required:authority"],
|
||||
}],
|
||||
});
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("replaySnapshots rejects changed skill and case evidence", () => {
|
||||
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-replay-stale-test-"));
|
||||
try {
|
||||
const snapshotFile = acceptReport({
|
||||
report: passingReportForCases(manifest.cases, {
|
||||
model: "openai/gpt-5.6-sol",
|
||||
opencodeVersion: "1.18.9",
|
||||
}),
|
||||
manifest,
|
||||
repo: REPO,
|
||||
snapshotsPath: join(temp, "snapshots.json"),
|
||||
});
|
||||
assert.throws(
|
||||
() => replaySnapshots(withChangedHash(snapshotFile, "skillHash"), manifest, REPO),
|
||||
/stale skill hash/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => replaySnapshots(withChangedHash(snapshotFile, "caseHash"), manifest, REPO),
|
||||
/stale case hash/i,
|
||||
);
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("validateSnapshots requires a closed complete one-to-one snapshot set", () => {
|
||||
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-snapshot-validation-test-"));
|
||||
try {
|
||||
const snapshotFile = acceptReport({
|
||||
report: passingReportForCases(manifest.cases, {
|
||||
model: "openai/gpt-5.6-sol",
|
||||
opencodeVersion: "1.18.9",
|
||||
}),
|
||||
manifest,
|
||||
repo: REPO,
|
||||
snapshotsPath: join(temp, "snapshots.json"),
|
||||
});
|
||||
assert.deepEqual(validateSnapshots(snapshotFile, manifest, REPO), snapshotFile);
|
||||
assert.throws(
|
||||
() => validateSnapshots({ ...snapshotFile, stdout: "raw" }, manifest, REPO),
|
||||
/snapshots.*unknown field stdout/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => validateSnapshots({
|
||||
...snapshotFile,
|
||||
snapshots: snapshotFile.snapshots.map((entry, index) => (
|
||||
index === 0 ? { ...entry, durationMs: 1000 } : entry
|
||||
)),
|
||||
}, manifest, REPO),
|
||||
/unknown field durationMs/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => validateSnapshots({
|
||||
...snapshotFile,
|
||||
snapshots: [...snapshotFile.snapshots, snapshotFile.snapshots[0]],
|
||||
}, manifest, REPO),
|
||||
/duplicate snapshot/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => validateSnapshots({
|
||||
...snapshotFile,
|
||||
snapshots: snapshotFile.snapshots.slice(0, -1),
|
||||
}, manifest, REPO),
|
||||
/missing snapshot/i,
|
||||
);
|
||||
assert.throws(
|
||||
() => validateSnapshots({
|
||||
...snapshotFile,
|
||||
snapshots: snapshotFile.snapshots.map((entry, index) => (
|
||||
index === 0 ? { ...entry, caseId: "unknown-case" } : entry
|
||||
)),
|
||||
}, manifest, REPO),
|
||||
/unknown snapshot case/i,
|
||||
);
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("CLI accept reads the latest report and writes only accepted snapshots", () => {
|
||||
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-cli-accept-test-"));
|
||||
const reportPath = join(temp, "artifacts", "latest.json");
|
||||
const snapshotsPath = join(temp, "snapshots.json");
|
||||
try {
|
||||
writeReport(passingReportForCases(manifest.cases, {
|
||||
model: "openai/gpt-5.6-sol",
|
||||
opencodeVersion: "1.18.9",
|
||||
}), reportPath);
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[new URL("../scripts/behavioral-evals.mjs", import.meta.url).pathname, "accept"],
|
||||
{
|
||||
cwd: REPO,
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
BEHAVIORAL_EVAL_LATEST_PATH: reportPath,
|
||||
BEHAVIORAL_EVAL_SNAPSHOTS_PATH: snapshotsPath,
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.equal(result.stdout, "Accepted 12 behavioral snapshots\n");
|
||||
assert.equal(result.stderr, "");
|
||||
assert.deepEqual(
|
||||
validateSnapshots(JSON.parse(readFileSync(snapshotsPath, "utf8")), manifest, REPO),
|
||||
JSON.parse(readFileSync(snapshotsPath, "utf8")),
|
||||
);
|
||||
assert.equal(existsSync(join(REPO, "evals/behavioral/snapshots.json")), false);
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("CLI accept exits nonzero without writing stale evidence", () => {
|
||||
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-cli-stale-test-"));
|
||||
const reportPath = join(temp, "artifacts", "latest.json");
|
||||
const snapshotsPath = join(temp, "snapshots.json");
|
||||
try {
|
||||
writeReport(withStaleSkillHash(passingReportForCases(manifest.cases, {
|
||||
model: "openai/gpt-5.6-sol",
|
||||
opencodeVersion: "1.18.9",
|
||||
})), reportPath);
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[new URL("../scripts/behavioral-evals.mjs", import.meta.url).pathname, "accept"],
|
||||
{
|
||||
cwd: REPO,
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
BEHAVIORAL_EVAL_LATEST_PATH: reportPath,
|
||||
BEHAVIORAL_EVAL_SNAPSHOTS_PATH: snapshotsPath,
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.status, 1);
|
||||
assert.equal(result.stdout, "");
|
||||
assert.match(result.stderr, /stale skill hash/i);
|
||||
assert.equal(existsSync(snapshotsPath), false);
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("CLI accept exits nonzero without writing when the latest report is missing or malformed", () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-cli-errors-test-"));
|
||||
const reportPath = join(temp, "artifacts", "latest.json");
|
||||
const snapshotsPath = join(temp, "snapshots.json");
|
||||
const spawnAccept = () => spawnSync(
|
||||
process.execPath,
|
||||
[new URL("../scripts/behavioral-evals.mjs", import.meta.url).pathname, "accept"],
|
||||
{
|
||||
cwd: REPO,
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
BEHAVIORAL_EVAL_LATEST_PATH: reportPath,
|
||||
BEHAVIORAL_EVAL_SNAPSHOTS_PATH: snapshotsPath,
|
||||
},
|
||||
},
|
||||
);
|
||||
try {
|
||||
const missing = spawnAccept();
|
||||
assert.equal(missing.status, 1);
|
||||
assert.equal(missing.stdout, "");
|
||||
assert.match(missing.stderr, /ENOENT/);
|
||||
assert.equal(existsSync(snapshotsPath), false);
|
||||
|
||||
mkdirSync(join(temp, "artifacts"), { recursive: true });
|
||||
writeFileSync(reportPath, "not-json");
|
||||
const malformed = spawnAccept();
|
||||
assert.equal(malformed.status, 1);
|
||||
assert.equal(malformed.stdout, "");
|
||||
assert.match(malformed.stderr, /JSON|Unexpected token|Unexpected end/i);
|
||||
assert.equal(existsSync(snapshotsPath), false);
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("runSuite detects the version and runs the selected cases sequentially", async () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-suite-test-"));
|
||||
const fake = join(temp, "fake-suite.mjs");
|
||||
|
||||
Reference in New Issue
Block a user