mirror of
https://github.com/waybarrios/opencode-power-pack.git
synced 2026-11-18 18:03:43 +00:00
970 lines
35 KiB
JavaScript
970 lines
35 KiB
JavaScript
import { createHash, randomBytes } from "node:crypto";
|
|
import { spawn } from "node:child_process";
|
|
import {
|
|
chmodSync,
|
|
copyFileSync,
|
|
existsSync,
|
|
mkdirSync,
|
|
mkdtempSync,
|
|
readFileSync,
|
|
renameSync,
|
|
rmSync,
|
|
statSync,
|
|
writeFileSync,
|
|
} from "node:fs";
|
|
import { createServer } from "node:net";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
|
|
const MANIFEST_FIELDS = new Set(["version", "targetSkills", "cases"]);
|
|
const CASE_FIELDS = new Set([
|
|
"id", "skill", "category", "timeoutMs", "prompt", "sentinels",
|
|
"fictitiousSecrets", "required", "forbidden",
|
|
]);
|
|
const REQUIRED_FIELDS = new Set(["id", "anyOf"]);
|
|
const FORBIDDEN_FIELDS = new Set(["id", "values"]);
|
|
const MUTATION_FIELDS = new Set(["id", "caseId", "operation", "from", "value", "expectedFailure"]);
|
|
const REPORT_FIELDS = new Set([
|
|
"version", "model", "opencodeVersion", "startedAt", "completedAt", "status", "cases",
|
|
]);
|
|
const REPORT_CASE_FIELDS = new Set([
|
|
"caseId", "skill", "category", "status", "failures", "response", "model",
|
|
"opencodeVersion", "skillHash", "caseHash", "durationMs", "completedAt",
|
|
]);
|
|
const SNAPSHOT_FILE_FIELDS = new Set(["version", "snapshots"]);
|
|
const SNAPSHOT_FIELDS = new Set([
|
|
"caseId", "model", "opencodeVersion", "skillHash", "caseHash", "verdict", "response",
|
|
]);
|
|
|
|
export function sha256(value) {
|
|
return createHash("sha256").update(value).digest("hex");
|
|
}
|
|
|
|
function canonical(value) {
|
|
if (Array.isArray(value)) return value.map(canonical);
|
|
if (value && typeof value === "object") {
|
|
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])]));
|
|
}
|
|
return value;
|
|
}
|
|
|
|
export function caseHash(testCase) {
|
|
return sha256(JSON.stringify(canonical(testCase)));
|
|
}
|
|
|
|
export function hashSkill(repo, skill) {
|
|
const skillPath = path.join(repo, "skills", skill, "SKILL.md");
|
|
if (!existsSync(skillPath)) throw new TypeError(`skill ${skill} must have a SKILL.md`);
|
|
return sha256(readFileSync(skillPath));
|
|
}
|
|
|
|
function isObject(value) {
|
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
}
|
|
|
|
function assertObject(value, label) {
|
|
if (!isObject(value)) throw new TypeError(`${label} must be an object`);
|
|
}
|
|
|
|
function assertFields(value, fields, label) {
|
|
for (const field of Object.keys(value)) {
|
|
if (!fields.has(field)) throw new TypeError(`${label}: unknown field ${field}`);
|
|
}
|
|
for (const field of fields) {
|
|
if (!Object.hasOwn(value, field)) throw new TypeError(`${label}: missing field ${field}`);
|
|
}
|
|
}
|
|
|
|
function assertNonEmptyString(value, label) {
|
|
if (typeof value !== "string" || value.trim() === "") {
|
|
throw new TypeError(`${label} must be a non-empty string`);
|
|
}
|
|
}
|
|
|
|
function copyStringArray(value, label, allowEmpty = true) {
|
|
if (!Array.isArray(value) || (!allowEmpty && value.length === 0)) {
|
|
throw new TypeError(`${label} must be a non-empty array`);
|
|
}
|
|
return value.map((entry, index) => {
|
|
assertNonEmptyString(entry, `${label}[${index}]`);
|
|
return entry;
|
|
});
|
|
}
|
|
|
|
function copyOracles(value, fields, valuesField, label) {
|
|
if (!Array.isArray(value) || value.length === 0) {
|
|
throw new TypeError(`${label} must be a non-empty array`);
|
|
}
|
|
return value.map((oracle, index) => {
|
|
const oracleLabel = `${label}[${index}]`;
|
|
assertObject(oracle, oracleLabel);
|
|
assertFields(oracle, fields, oracleLabel);
|
|
assertNonEmptyString(oracle.id, `${oracleLabel}.id`);
|
|
return {
|
|
id: oracle.id,
|
|
[valuesField]: copyStringArray(oracle[valuesField], `${oracleLabel}.${valuesField}`, false),
|
|
};
|
|
});
|
|
}
|
|
|
|
function deepFreeze(value) {
|
|
if (value && typeof value === "object" && !Object.isFrozen(value)) {
|
|
for (const entry of Object.values(value)) deepFreeze(entry);
|
|
Object.freeze(value);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
export function validateManifest(value) {
|
|
assertObject(value, "manifest");
|
|
assertFields(value, MANIFEST_FIELDS, "manifest");
|
|
if (value.version !== 1) throw new TypeError("manifest.version must equal 1");
|
|
|
|
const targetSkills = copyStringArray(value.targetSkills, "manifest.targetSkills", false);
|
|
const targetSkillSet = new Set();
|
|
for (const skill of targetSkills) {
|
|
if (targetSkillSet.has(skill)) throw new TypeError(`duplicate target skill ${skill}`);
|
|
targetSkillSet.add(skill);
|
|
}
|
|
if (!Array.isArray(value.cases)) throw new TypeError("manifest.cases must be an array");
|
|
|
|
const caseIds = new Set();
|
|
const sentinels = new Set();
|
|
const coveredSkills = new Set();
|
|
const cases = value.cases.map((testCase, index) => {
|
|
const initialLabel = isObject(testCase) && typeof testCase.id === "string" && testCase.id !== ""
|
|
? testCase.id
|
|
: `case[${index}]`;
|
|
assertObject(testCase, initialLabel);
|
|
assertFields(testCase, CASE_FIELDS, initialLabel);
|
|
assertNonEmptyString(testCase.id, `${initialLabel}.id`);
|
|
assertNonEmptyString(testCase.skill, `${testCase.id}.skill`);
|
|
assertNonEmptyString(testCase.category, `${testCase.id}.category`);
|
|
assertNonEmptyString(testCase.prompt, `${testCase.id}.prompt`);
|
|
if (!Number.isInteger(testCase.timeoutMs)
|
|
|| testCase.timeoutMs < 10
|
|
|| testCase.timeoutMs > 300000) {
|
|
throw new TypeError(`${testCase.id}.timeoutMs must be an integer from 10 through 300000`);
|
|
}
|
|
if (caseIds.has(testCase.id)) throw new TypeError(`duplicate case id ${testCase.id}`);
|
|
if (!targetSkillSet.has(testCase.skill)) {
|
|
throw new TypeError(`${testCase.id}: case uses undeclared skill ${testCase.skill}`);
|
|
}
|
|
|
|
const caseSentinels = copyStringArray(testCase.sentinels, `${testCase.id}.sentinels`);
|
|
for (const sentinel of caseSentinels) {
|
|
if (sentinels.has(sentinel)) throw new TypeError(`duplicate sentinel ${sentinel}`);
|
|
sentinels.add(sentinel);
|
|
}
|
|
caseIds.add(testCase.id);
|
|
coveredSkills.add(testCase.skill);
|
|
|
|
return {
|
|
id: testCase.id,
|
|
skill: testCase.skill,
|
|
category: testCase.category,
|
|
timeoutMs: testCase.timeoutMs,
|
|
prompt: testCase.prompt,
|
|
sentinels: caseSentinels,
|
|
fictitiousSecrets: copyStringArray(
|
|
testCase.fictitiousSecrets,
|
|
`${testCase.id}.fictitiousSecrets`,
|
|
),
|
|
required: copyOracles(
|
|
testCase.required,
|
|
REQUIRED_FIELDS,
|
|
"anyOf",
|
|
`${testCase.id}.required`,
|
|
),
|
|
forbidden: copyOracles(
|
|
testCase.forbidden,
|
|
FORBIDDEN_FIELDS,
|
|
"values",
|
|
`${testCase.id}.forbidden`,
|
|
),
|
|
};
|
|
});
|
|
|
|
for (const skill of targetSkills) {
|
|
if (!coveredSkills.has(skill)) throw new TypeError(`target skill ${skill} is without a case`);
|
|
}
|
|
|
|
return deepFreeze({ version: 1, targetSkills, cases });
|
|
}
|
|
|
|
export function validateMutations(value, manifest) {
|
|
if (!Array.isArray(value)) throw new TypeError("mutations must be an array");
|
|
|
|
const cases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase]));
|
|
const mutationIds = new Set();
|
|
const mutations = value.map((mutation, index) => {
|
|
const label = isObject(mutation) && typeof mutation.id === "string" && mutation.id !== ""
|
|
? mutation.id
|
|
: `mutation[${index}]`;
|
|
assertObject(mutation, label);
|
|
for (const field of Object.keys(mutation)) {
|
|
if (!MUTATION_FIELDS.has(field)) throw new TypeError(`${label}: unknown field ${field}`);
|
|
}
|
|
for (const field of ["id", "caseId", "operation", "value", "expectedFailure"]) {
|
|
if (!Object.hasOwn(mutation, field)) throw new TypeError(`${label}: missing field ${field}`);
|
|
assertNonEmptyString(mutation[field], `${label}.${field}`);
|
|
}
|
|
if (mutationIds.has(mutation.id)) throw new TypeError(`duplicate mutation id ${mutation.id}`);
|
|
mutationIds.add(mutation.id);
|
|
|
|
if (mutation.operation === "append") {
|
|
if (Object.hasOwn(mutation, "from")) {
|
|
throw new TypeError(`${label}: append operation forbids from`);
|
|
}
|
|
} else if (mutation.operation === "replace") {
|
|
if (!Object.hasOwn(mutation, "from")) {
|
|
throw new TypeError(`${label}: replace operation requires from`);
|
|
}
|
|
assertNonEmptyString(mutation.from, `${label}.from`);
|
|
} else {
|
|
throw new TypeError(`${label}.operation must be append or replace`);
|
|
}
|
|
|
|
const testCase = cases.get(mutation.caseId);
|
|
if (!testCase) throw new TypeError(`${label}: unknown case ${mutation.caseId}`);
|
|
const failureIds = new Set([
|
|
...testCase.required.map(({ id }) => `required:${id}`),
|
|
...testCase.forbidden.map(({ id }) => `forbidden:${id}`),
|
|
]);
|
|
if (testCase.sentinels.length > 0) failureIds.add("forbidden:sentinel");
|
|
if (testCase.fictitiousSecrets.length > 0) failureIds.add("forbidden:fictitious-secret");
|
|
if (!failureIds.has(mutation.expectedFailure)) {
|
|
throw new TypeError(`${label}: unknown expected failure ${mutation.expectedFailure}`);
|
|
}
|
|
|
|
return Object.hasOwn(mutation, "from")
|
|
? {
|
|
id: mutation.id,
|
|
caseId: mutation.caseId,
|
|
operation: mutation.operation,
|
|
from: mutation.from,
|
|
value: mutation.value,
|
|
expectedFailure: mutation.expectedFailure,
|
|
}
|
|
: {
|
|
id: mutation.id,
|
|
caseId: mutation.caseId,
|
|
operation: mutation.operation,
|
|
value: mutation.value,
|
|
expectedFailure: mutation.expectedFailure,
|
|
};
|
|
});
|
|
|
|
return deepFreeze(mutations);
|
|
}
|
|
|
|
export function gradeResponse(testCase, response) {
|
|
if (typeof response !== "string" || response.trim() === "") {
|
|
return { status: "fail", failures: ["incomplete:empty-response"] };
|
|
}
|
|
const folded = response.toLocaleLowerCase("en-US");
|
|
const failures = [];
|
|
for (const oracle of testCase.required) {
|
|
if (!oracle.anyOf.some((value) => folded.includes(value.toLocaleLowerCase("en-US")))) {
|
|
failures.push(`required:${oracle.id}`);
|
|
}
|
|
}
|
|
for (const oracle of testCase.forbidden) {
|
|
if (oracle.values.some((value) => folded.includes(value.toLocaleLowerCase("en-US")))) {
|
|
failures.push(`forbidden:${oracle.id}`);
|
|
}
|
|
}
|
|
if (testCase.sentinels.some((value) => folded.includes(value.toLocaleLowerCase("en-US")))) {
|
|
failures.push("forbidden:sentinel");
|
|
}
|
|
if (testCase.fictitiousSecrets.some((value) => folded.includes(value.toLocaleLowerCase("en-US")))) {
|
|
failures.push("forbidden:fictitious-secret");
|
|
}
|
|
return { status: failures.length === 0 ? "pass" : "fail", failures };
|
|
}
|
|
|
|
function assertHash(value, label) {
|
|
if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) {
|
|
throw new TypeError(`${label} must be 64 lowercase hexadecimal characters`);
|
|
}
|
|
}
|
|
|
|
function assertTimestamp(value, label) {
|
|
const parsed = typeof value === "string" ? new Date(value) : undefined;
|
|
if (!parsed || Number.isNaN(parsed.getTime()) || parsed.toISOString() !== value) {
|
|
throw new TypeError(`${label} must be an ISO timestamp`);
|
|
}
|
|
}
|
|
|
|
export function validateReport(value, manifest, repo) {
|
|
assertObject(value, "report");
|
|
assertFields(value, REPORT_FIELDS, "report");
|
|
if (value.version !== 1) throw new TypeError("report.version must equal 1");
|
|
assertNonEmptyString(value.model, "report.model");
|
|
assertNonEmptyString(value.opencodeVersion, "report.opencodeVersion");
|
|
assertTimestamp(value.startedAt, "report.startedAt");
|
|
assertTimestamp(value.completedAt, "report.completedAt");
|
|
if (value.status !== "pass") throw new TypeError("report status must be pass");
|
|
if (!Array.isArray(value.cases)) throw new TypeError("report.cases must be an array");
|
|
|
|
const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase]));
|
|
const seen = new Set();
|
|
const cases = value.cases.map((entry, index) => {
|
|
const label = isObject(entry) && typeof entry.caseId === "string" && entry.caseId !== ""
|
|
? entry.caseId
|
|
: `report.cases[${index}]`;
|
|
assertObject(entry, label);
|
|
assertFields(entry, REPORT_CASE_FIELDS, label);
|
|
assertNonEmptyString(entry.caseId, `${label}.caseId`);
|
|
if (seen.has(entry.caseId)) throw new TypeError(`duplicate case ${entry.caseId}`);
|
|
seen.add(entry.caseId);
|
|
const testCase = currentCases.get(entry.caseId);
|
|
if (!testCase) throw new TypeError(`unknown case ${entry.caseId}`);
|
|
|
|
assertNonEmptyString(entry.skill, `${label}.skill`);
|
|
assertNonEmptyString(entry.category, `${label}.category`);
|
|
if (entry.skill !== testCase.skill) throw new TypeError(`${label}: skill does not match manifest`);
|
|
if (entry.category !== testCase.category) {
|
|
throw new TypeError(`${label}: category does not match manifest`);
|
|
}
|
|
if (entry.status !== "pass") throw new TypeError(`${label}: case status must be pass`);
|
|
if (!Array.isArray(entry.failures) || entry.failures.length !== 0) {
|
|
throw new TypeError(`${label}: passing case failures must be empty`);
|
|
}
|
|
assertNonEmptyString(entry.response, `${label}.response`);
|
|
assertNonEmptyString(entry.model, `${label}.model`);
|
|
assertNonEmptyString(entry.opencodeVersion, `${label}.opencodeVersion`);
|
|
if (entry.model !== value.model) throw new TypeError(`${label}: model does not match report`);
|
|
if (entry.opencodeVersion !== value.opencodeVersion) {
|
|
throw new TypeError(`${label}: OpenCode version does not match report`);
|
|
}
|
|
assertHash(entry.skillHash, `${label}.skillHash`);
|
|
assertHash(entry.caseHash, `${label}.caseHash`);
|
|
if (entry.skillHash !== hashSkill(repo, testCase.skill)) {
|
|
throw new TypeError(`${label}: stale skill hash`);
|
|
}
|
|
if (entry.caseHash !== caseHash(testCase)) throw new TypeError(`${label}: stale case hash`);
|
|
if (!Number.isInteger(entry.durationMs) || entry.durationMs < 0) {
|
|
throw new TypeError(`${label}.durationMs must be a non-negative integer`);
|
|
}
|
|
assertTimestamp(entry.completedAt, `${label}.completedAt`);
|
|
const grade = gradeResponse(testCase, entry.response);
|
|
if (grade.status !== "pass") {
|
|
throw new TypeError(`${label}: response failed grading (${grade.failures.join(", ")})`);
|
|
}
|
|
|
|
return {
|
|
caseId: entry.caseId,
|
|
skill: entry.skill,
|
|
category: entry.category,
|
|
status: entry.status,
|
|
failures: [],
|
|
response: entry.response,
|
|
model: entry.model,
|
|
opencodeVersion: entry.opencodeVersion,
|
|
skillHash: entry.skillHash,
|
|
caseHash: entry.caseHash,
|
|
durationMs: entry.durationMs,
|
|
completedAt: entry.completedAt,
|
|
};
|
|
});
|
|
|
|
for (const testCase of manifest.cases) {
|
|
if (!seen.has(testCase.id)) throw new TypeError(`missing case ${testCase.id}`);
|
|
}
|
|
return deepFreeze({
|
|
version: 1,
|
|
model: value.model,
|
|
opencodeVersion: value.opencodeVersion,
|
|
startedAt: value.startedAt,
|
|
completedAt: value.completedAt,
|
|
status: "pass",
|
|
cases,
|
|
});
|
|
}
|
|
|
|
export function acceptReport({ report, manifest, repo, snapshotsPath }) {
|
|
const validated = validateReport(report, manifest, repo);
|
|
const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase]));
|
|
const snapshots = validated.cases.map((entry) => {
|
|
const testCase = currentCases.get(entry.caseId);
|
|
return {
|
|
caseId: entry.caseId,
|
|
model: entry.model,
|
|
opencodeVersion: entry.opencodeVersion,
|
|
skillHash: entry.skillHash,
|
|
caseHash: entry.caseHash,
|
|
verdict: "pass",
|
|
response: normalizeResponse(redactResponse(entry.response, testCase)),
|
|
};
|
|
}).sort((left, right) => (left.caseId < right.caseId ? -1 : left.caseId > right.caseId ? 1 : 0));
|
|
const snapshotFile = deepFreeze({ version: 1, snapshots });
|
|
const replayed = replaySnapshots(snapshotFile, manifest, repo);
|
|
if (replayed.status !== "pass") {
|
|
const detail = replayed.failures
|
|
.map(({ caseId, failures }) => `${caseId}: ${failures.join(", ")}`)
|
|
.join("; ");
|
|
throw new TypeError(`snapshot responses failed replay grading: ${detail}`);
|
|
}
|
|
mkdirSync(path.dirname(snapshotsPath), { recursive: true });
|
|
const tempPath = `${snapshotsPath}.tmp`;
|
|
writeFileSync(tempPath, `${JSON.stringify(snapshotFile, null, 2)}\n`);
|
|
renameSync(tempPath, snapshotsPath);
|
|
return snapshotFile;
|
|
}
|
|
|
|
export function validateSnapshots(value, manifest, repo) {
|
|
assertObject(value, "snapshots");
|
|
assertFields(value, SNAPSHOT_FILE_FIELDS, "snapshots");
|
|
if (value.version !== 1) throw new TypeError("snapshots.version must equal 1");
|
|
if (!Array.isArray(value.snapshots)) throw new TypeError("snapshots.snapshots must be an array");
|
|
|
|
const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase]));
|
|
const seen = new Set();
|
|
const snapshots = value.snapshots.map((snapshot, index) => {
|
|
const label = isObject(snapshot) && typeof snapshot.caseId === "string" && snapshot.caseId !== ""
|
|
? snapshot.caseId
|
|
: `snapshots[${index}]`;
|
|
assertObject(snapshot, label);
|
|
assertFields(snapshot, SNAPSHOT_FIELDS, label);
|
|
assertNonEmptyString(snapshot.caseId, `${label}.caseId`);
|
|
if (seen.has(snapshot.caseId)) throw new TypeError(`duplicate snapshot ${snapshot.caseId}`);
|
|
seen.add(snapshot.caseId);
|
|
const testCase = currentCases.get(snapshot.caseId);
|
|
if (!testCase) throw new TypeError(`unknown snapshot case ${snapshot.caseId}`);
|
|
|
|
assertNonEmptyString(snapshot.model, `${label}.model`);
|
|
assertNonEmptyString(snapshot.opencodeVersion, `${label}.opencodeVersion`);
|
|
assertHash(snapshot.skillHash, `${label}.skillHash`);
|
|
assertHash(snapshot.caseHash, `${label}.caseHash`);
|
|
if (snapshot.skillHash !== hashSkill(repo, testCase.skill)) {
|
|
throw new TypeError(`${label}: stale skill hash`);
|
|
}
|
|
if (snapshot.caseHash !== caseHash(testCase)) throw new TypeError(`${label}: stale case hash`);
|
|
if (snapshot.verdict !== "pass") throw new TypeError(`${label}: verdict must be pass`);
|
|
assertNonEmptyString(snapshot.response, `${label}.response`);
|
|
if (normalizeResponse(redactResponse(snapshot.response, testCase)) !== snapshot.response) {
|
|
throw new TypeError(`${label}: response must be normalized and redacted`);
|
|
}
|
|
return {
|
|
caseId: snapshot.caseId,
|
|
model: snapshot.model,
|
|
opencodeVersion: snapshot.opencodeVersion,
|
|
skillHash: snapshot.skillHash,
|
|
caseHash: snapshot.caseHash,
|
|
verdict: "pass",
|
|
response: snapshot.response,
|
|
};
|
|
});
|
|
|
|
for (const testCase of manifest.cases) {
|
|
if (!seen.has(testCase.id)) throw new TypeError(`missing snapshot ${testCase.id}`);
|
|
}
|
|
snapshots.sort((left, right) => (
|
|
left.caseId < right.caseId ? -1 : left.caseId > right.caseId ? 1 : 0
|
|
));
|
|
return deepFreeze({ version: 1, snapshots });
|
|
}
|
|
|
|
export function replaySnapshots(snapshotFile, manifest, repo) {
|
|
const validated = validateSnapshots(snapshotFile, manifest, repo);
|
|
const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase]));
|
|
const failures = [];
|
|
for (const snapshot of validated.snapshots) {
|
|
const grade = gradeResponse(currentCases.get(snapshot.caseId), snapshot.response);
|
|
if (grade.status === "fail") {
|
|
failures.push({ caseId: snapshot.caseId, failures: grade.failures });
|
|
}
|
|
}
|
|
return deepFreeze({
|
|
status: failures.length === 0 ? "pass" : "fail",
|
|
cases: validated.snapshots.length,
|
|
failures,
|
|
});
|
|
}
|
|
|
|
export function applyMutation(response, mutation) {
|
|
if (!isObject(mutation) || typeof mutation.value !== "string") {
|
|
throw new TypeError("mutation must contain a string value");
|
|
}
|
|
if (mutation.operation === "append") return `${response}\n${mutation.value}`;
|
|
if (mutation.operation === "replace") return mutation.value;
|
|
throw new TypeError(`unsupported mutation operation: ${mutation.operation}`);
|
|
}
|
|
|
|
export function normalizeResponse(response) {
|
|
return response
|
|
.replaceAll("\r\n", "\n")
|
|
.replace(/[ \t]+$/gm, "")
|
|
.replace(/\n{4,}/g, "\n\n\n")
|
|
.trim();
|
|
}
|
|
|
|
function escapeRegExp(value) {
|
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
}
|
|
|
|
export function redactResponse(response, testCase) {
|
|
let redacted = response;
|
|
const caseValues = [...testCase.sentinels, ...testCase.fictitiousSecrets]
|
|
.toSorted((left, right) => right.length - left.length);
|
|
for (const value of caseValues) {
|
|
redacted = redacted.replace(new RegExp(escapeRegExp(value), "gi"), "[REDACTED]");
|
|
}
|
|
const credential = /(?<![A-Za-z0-9_])(?:sk-[A-Za-z0-9][A-Za-z0-9._-]*|ghp_[A-Za-z0-9][A-Za-z0-9._-]*|github_pat_[A-Za-z0-9][A-Za-z0-9._-]*|AKIA[A-Z0-9]{16}|Bearer[ \t]+[A-Za-z0-9][A-Za-z0-9._~+/-]*)(?![A-Za-z0-9._~+/-])/g;
|
|
return redacted.replace(credential, "[REDACTED]");
|
|
}
|
|
|
|
/**
|
|
* Collects the assistant text from a V2 session context response and notes
|
|
* whether the model requested any tool. A tool request means the eval runtime
|
|
* could not keep the model inside the permission-free envelope.
|
|
*/
|
|
export function parseSessionContext(context) {
|
|
if (!isObject(context) || !Array.isArray(context.data)) {
|
|
throw new TypeError("malformed OpenCode session context");
|
|
}
|
|
const textParts = [];
|
|
let toolRequested = false;
|
|
let terminal;
|
|
for (const message of context.data) {
|
|
if (!isObject(message)) continue;
|
|
if (message.type === "idle") terminal = message;
|
|
if (message.type !== "assistant") continue;
|
|
if (message.error) throw new TypeError("unsuccessful OpenCode turn");
|
|
if (!Array.isArray(message.content)) {
|
|
throw new TypeError("malformed OpenCode session context");
|
|
}
|
|
for (const part of message.content) {
|
|
if (!isObject(part)) throw new TypeError("malformed OpenCode session context");
|
|
if (part.type === "tool") toolRequested = true;
|
|
if (part.type === "text") {
|
|
if (typeof part.text !== "string") {
|
|
throw new TypeError("malformed OpenCode session context");
|
|
}
|
|
textParts.push(part.text);
|
|
}
|
|
}
|
|
}
|
|
if ((terminal && (terminal.outcome !== "succeeded" || terminal.error))
|
|
|| (textParts.length > 0 && !terminal)) {
|
|
throw new TypeError("unsuccessful OpenCode turn");
|
|
}
|
|
return { response: textParts.join("\n"), toolRequested };
|
|
}
|
|
|
|
/**
|
|
* V2 loads a configured plugin directory through its index.js entrypoint and
|
|
* uses an ordered permission ruleset. The eval denies every tool so a case
|
|
* can only pass on model text.
|
|
*/
|
|
export function buildEvalConfig(pluginDirectory) {
|
|
return {
|
|
plugins: [pluginDirectory],
|
|
permissions: [{ action: "*", resource: "*", effect: "deny" }],
|
|
};
|
|
}
|
|
|
|
export function parseModelReference(model) {
|
|
const match = typeof model === "string" && model.match(/^([^/#\s]+)\/([^/#\s]+)(?:#([^/#\s]+))?$/);
|
|
if (!match) throw new TypeError("OPENCODE_EVAL_MODEL must use provider/model form with an optional #variant");
|
|
return { providerID: match[1], id: match[2], ...(match[3] ? { variant: match[3] } : {}) };
|
|
}
|
|
|
|
async function availablePort() {
|
|
const server = createServer();
|
|
await new Promise((resolve, reject) => server.listen(0, "127.0.0.1", resolve).once("error", reject));
|
|
const address = server.address();
|
|
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
|
return address.port;
|
|
}
|
|
|
|
/**
|
|
* Writes the plugin-directory shim the eval runtime loads. The shim keeps the
|
|
* package entrypoint in the repository, so its skills and dependencies resolve
|
|
* exactly as they do for an installed package.
|
|
*/
|
|
function writePluginShim(runtime, repo) {
|
|
const directory = path.join(runtime.root, "plugin");
|
|
mkdirSync(directory, { recursive: true });
|
|
const entrypoint = pathToFileURL(
|
|
path.join(repo, ".opencode/plugins/opencode-power-pack.js"),
|
|
).href;
|
|
writeFileSync(path.join(directory, "index.js"), `export { default } from ${JSON.stringify(entrypoint)};\n`);
|
|
return directory;
|
|
}
|
|
|
|
export function defaultOpenCodeCommand(platform = process.platform) {
|
|
return platform === "win32" ? "opencode.exe" : "opencode";
|
|
}
|
|
|
|
function signalChild(child, signal) {
|
|
try {
|
|
if (process.platform === "win32") child.kill(signal);
|
|
else process.kill(-child.pid, signal);
|
|
} catch (error) {
|
|
if (error.code !== "ESRCH") throw error;
|
|
}
|
|
}
|
|
|
|
function delay(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
function processGroupExists(pid) {
|
|
try {
|
|
process.kill(-pid, 0);
|
|
return true;
|
|
} catch (error) {
|
|
if (error.code === "ESRCH") return false;
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function terminateTimedOutChild(child) {
|
|
if (process.platform === "win32") {
|
|
signalChild(child, "SIGTERM");
|
|
await delay(100);
|
|
if (child.exitCode === null && child.signalCode === null) signalChild(child, "SIGKILL");
|
|
return;
|
|
}
|
|
|
|
signalChild(child, "SIGTERM");
|
|
await delay(100);
|
|
signalChild(child, "SIGKILL");
|
|
while (processGroupExists(child.pid)) await delay(10);
|
|
}
|
|
|
|
function reportEntry(testCase, options, started, status, failures, response) {
|
|
return {
|
|
caseId: testCase.id,
|
|
skill: testCase.skill,
|
|
category: testCase.category,
|
|
status,
|
|
failures,
|
|
response,
|
|
model: options.model,
|
|
opencodeVersion: options.opencodeVersion,
|
|
skillHash: hashSkill(options.repo, testCase.skill),
|
|
caseHash: caseHash(testCase),
|
|
durationMs: Date.now() - started,
|
|
completedAt: new Date().toISOString(),
|
|
};
|
|
}
|
|
|
|
function createRuntime(env) {
|
|
const root = mkdtempSync(path.join(tmpdir(), "behavioral-eval-"));
|
|
try {
|
|
const runtime = {
|
|
root,
|
|
project: path.join(root, "project"),
|
|
home: path.join(root, "home"),
|
|
config: path.join(root, "config"),
|
|
data: path.join(root, "data"),
|
|
cache: path.join(root, "cache"),
|
|
state: path.join(root, "state"),
|
|
};
|
|
for (const directory of [
|
|
runtime.project, runtime.home, runtime.config, runtime.data, runtime.cache, runtime.state,
|
|
]) {
|
|
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
}
|
|
|
|
const sourceData = env.XDG_DATA_HOME
|
|
|| (env.HOME ? path.join(env.HOME, ".local", "share") : undefined);
|
|
const sourceAuth = sourceData && path.join(sourceData, "opencode", "auth.json");
|
|
if (sourceAuth && existsSync(sourceAuth)) {
|
|
const sourceAuthStat = statSync(sourceAuth);
|
|
if (!sourceAuthStat.isFile()) return runtime;
|
|
const targetDirectory = path.join(runtime.data, "opencode");
|
|
const targetAuth = path.join(targetDirectory, "auth.json");
|
|
mkdirSync(targetDirectory, { recursive: true, mode: 0o700 });
|
|
copyFileSync(sourceAuth, targetAuth);
|
|
chmodSync(targetAuth, (sourceAuthStat.mode & 0o600) || 0o600);
|
|
}
|
|
return runtime;
|
|
} catch (error) {
|
|
rmSync(root, { recursive: true, force: true });
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function isolatedEnv(env, runtime, model, config, password) {
|
|
const childEnv = {};
|
|
const isolatedPaths = new Set([
|
|
"HOME", "USERPROFILE", "APPDATA", "LOCALAPPDATA",
|
|
"XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME",
|
|
]);
|
|
for (const [key, value] of Object.entries(env)) {
|
|
const normalized = key.toUpperCase();
|
|
if (isolatedPaths.has(normalized)) continue;
|
|
if (normalized.startsWith("OPENCODE_")) continue;
|
|
childEnv[key] = value;
|
|
}
|
|
return {
|
|
...childEnv,
|
|
HOME: runtime.home,
|
|
USERPROFILE: runtime.home,
|
|
APPDATA: runtime.config,
|
|
LOCALAPPDATA: runtime.data,
|
|
XDG_CONFIG_HOME: runtime.config,
|
|
XDG_DATA_HOME: runtime.data,
|
|
XDG_CACHE_HOME: runtime.cache,
|
|
XDG_STATE_HOME: runtime.state,
|
|
OPENCODE_EVAL_MODEL: model,
|
|
OPENCODE_SERVER_PASSWORD: password,
|
|
OPENCODE_CONFIG_CONTENT: JSON.stringify(config),
|
|
};
|
|
}
|
|
|
|
async function requestJson(url, { method = "GET", body, authorization, signal } = {}) {
|
|
const response = await fetch(url, {
|
|
method,
|
|
headers: {
|
|
...(authorization ? { authorization } : {}),
|
|
...(body === undefined ? {} : { "content-type": "application/json" }),
|
|
},
|
|
body: body === undefined ? undefined : JSON.stringify(body),
|
|
signal,
|
|
});
|
|
if (!response.ok) {
|
|
throw new TypeError(`OpenCode API ${method} ${url} returned HTTP ${response.status}`);
|
|
}
|
|
const text = await response.text();
|
|
if (text === "") return undefined;
|
|
try {
|
|
return JSON.parse(text);
|
|
} catch {
|
|
throw new TypeError("malformed OpenCode API response");
|
|
}
|
|
}
|
|
|
|
async function waitForServer(baseURL, { authorization, signal, child, spawnError }) {
|
|
while (!signal.aborted) {
|
|
if (spawnError.error || child.exitCode !== null || child.signalCode !== null) return false;
|
|
try {
|
|
const response = await fetch(`${baseURL}/api/info`, {
|
|
headers: { authorization },
|
|
signal: AbortSignal.any([signal, AbortSignal.timeout(1_000)]),
|
|
});
|
|
if (response.ok) return true;
|
|
} catch {
|
|
if (signal.aborted) break;
|
|
}
|
|
await delay(100);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
export async function runCase(testCase, options) {
|
|
const {
|
|
repo,
|
|
model,
|
|
opencodeVersion,
|
|
command = defaultOpenCodeCommand(),
|
|
commandArgsPrefix = [],
|
|
env = process.env,
|
|
} = options;
|
|
const modelReference = parseModelReference(model);
|
|
const started = Date.now();
|
|
const runtime = createRuntime(env);
|
|
const controller = new AbortController();
|
|
const timeout = setTimeout(() => controller.abort(), testCase.timeoutMs);
|
|
const spawnError = { error: undefined };
|
|
let child;
|
|
const finish = (status, failures, response) => reportEntry(
|
|
testCase, options, started, status, failures, response,
|
|
);
|
|
|
|
try {
|
|
const pluginDirectory = writePluginShim(runtime, repo);
|
|
const port = await availablePort();
|
|
const password = randomBytes(24).toString("base64url");
|
|
const authorization = `Basic ${Buffer.from(`opencode:${password}`).toString("base64")}`;
|
|
const baseURL = `http://127.0.0.1:${port}`;
|
|
const args = [...commandArgsPrefix, "serve", "--hostname", "127.0.0.1", "--port", String(port)];
|
|
|
|
child = spawn(command, args, {
|
|
cwd: runtime.project,
|
|
env: isolatedEnv(env, runtime, model, buildEvalConfig(pluginDirectory), password),
|
|
detached: process.platform !== "win32",
|
|
shell: false,
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
child.stdout.resume();
|
|
child.stderr.resume();
|
|
child.once("error", (error) => {
|
|
spawnError.error = error;
|
|
});
|
|
|
|
const ready = await waitForServer(baseURL, { authorization, signal: controller.signal, child, spawnError });
|
|
if (!ready) {
|
|
if (controller.signal.aborted) return finish("incomplete", ["incomplete:timeout"], "");
|
|
return finish("incomplete", ["incomplete:process-exit"], "");
|
|
}
|
|
|
|
try {
|
|
const created = await requestJson(`${baseURL}/api/session`, {
|
|
method: "POST",
|
|
authorization,
|
|
signal: controller.signal,
|
|
body: { title: testCase.id, model: modelReference },
|
|
});
|
|
const sessionID = created?.data?.id;
|
|
if (typeof sessionID !== "string" || sessionID === "") {
|
|
throw new TypeError("malformed OpenCode session");
|
|
}
|
|
await requestJson(`${baseURL}/api/session/${sessionID}/prompt`, {
|
|
method: "POST",
|
|
authorization,
|
|
signal: controller.signal,
|
|
body: { text: testCase.prompt, skills: [{ id: testCase.skill }] },
|
|
});
|
|
await requestJson(`${baseURL}/api/experimental/session/${sessionID}/wait`, {
|
|
method: "POST",
|
|
authorization,
|
|
signal: controller.signal,
|
|
});
|
|
const context = await requestJson(`${baseURL}/api/session/${sessionID}/context`, {
|
|
authorization,
|
|
signal: controller.signal,
|
|
});
|
|
let parsed;
|
|
try {
|
|
parsed = parseSessionContext(context);
|
|
} catch {
|
|
return finish("incomplete", ["incomplete:malformed-response"], "");
|
|
}
|
|
const { response, toolRequested } = parsed;
|
|
if (toolRequested) {
|
|
return finish("incomplete", ["incomplete:permission"], "");
|
|
}
|
|
if (response.trim() === "") {
|
|
return finish("incomplete", ["incomplete:missing-response"], "");
|
|
}
|
|
const grade = gradeResponse(testCase, response);
|
|
return finish(
|
|
grade.status,
|
|
grade.failures,
|
|
normalizeResponse(redactResponse(response, testCase)),
|
|
);
|
|
} catch {
|
|
if (controller.signal.aborted) return finish("incomplete", ["incomplete:timeout"], "");
|
|
return finish("incomplete", ["incomplete:malformed-response"], "");
|
|
}
|
|
} finally {
|
|
clearTimeout(timeout);
|
|
if (child && child.exitCode === null && child.signalCode === null) {
|
|
await terminateTimedOutChild(child);
|
|
}
|
|
rmSync(runtime.root, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
export function writeReport(
|
|
report,
|
|
outputPath = path.resolve(".artifacts/behavioral-evals/latest.json"),
|
|
) {
|
|
mkdirSync(path.dirname(outputPath), { recursive: true });
|
|
const tempPath = `${outputPath}.tmp`;
|
|
try {
|
|
writeFileSync(tempPath, `${JSON.stringify(report, null, 2)}\n`);
|
|
renameSync(tempPath, outputPath);
|
|
} catch (error) {
|
|
rmSync(tempPath, { force: true });
|
|
throw error;
|
|
}
|
|
return outputPath;
|
|
}
|
|
|
|
async function detectOpenCodeVersion(command, commandArgsPrefix, env, cwd) {
|
|
return await new Promise((resolve, reject) => {
|
|
const chunks = [];
|
|
const child = spawn(command, [...commandArgsPrefix, "--version"], {
|
|
cwd,
|
|
env,
|
|
shell: false,
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
child.stdout.on("data", (chunk) => chunks.push(chunk));
|
|
child.stderr.resume();
|
|
child.once("error", reject);
|
|
child.once("close", (code) => {
|
|
if (code !== 0) {
|
|
reject(new Error("Unable to detect OpenCode version"));
|
|
return;
|
|
}
|
|
resolve(Buffer.concat(chunks).toString("utf8").trim());
|
|
});
|
|
});
|
|
}
|
|
|
|
export async function runSuite(options) {
|
|
const {
|
|
repo,
|
|
cases,
|
|
env = process.env,
|
|
command = defaultOpenCodeCommand(),
|
|
commandArgsPrefix = [],
|
|
} = options;
|
|
const startedAt = new Date().toISOString();
|
|
const model = env.OPENCODE_EVAL_MODEL;
|
|
parseModelReference(model);
|
|
if (cases.length === 0) throw new TypeError("case selection must not be empty");
|
|
const opencodeVersion = await detectOpenCodeVersion(command, commandArgsPrefix, env, repo);
|
|
const entries = [];
|
|
for (const testCase of cases) {
|
|
entries.push(await runCase(testCase, {
|
|
repo,
|
|
model,
|
|
opencodeVersion,
|
|
command,
|
|
commandArgsPrefix,
|
|
env,
|
|
}));
|
|
}
|
|
return {
|
|
version: 1,
|
|
model,
|
|
opencodeVersion,
|
|
startedAt,
|
|
completedAt: new Date().toISOString(),
|
|
status: entries.every(({ status }) => status === "pass") ? "pass" : "fail",
|
|
cases: entries,
|
|
};
|
|
}
|
|
|
|
async function main() {
|
|
const mode = process.argv[2];
|
|
if (mode !== "run" && mode !== "accept") {
|
|
throw new TypeError(`unsupported behavioral evaluation mode: ${mode || "missing"}`);
|
|
}
|
|
const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
const manifest = validateManifest(JSON.parse(
|
|
readFileSync(path.join(repo, "evals/behavioral/cases.json"), "utf8"),
|
|
));
|
|
const latestPath = process.env.BEHAVIORAL_EVAL_LATEST_PATH
|
|
|| path.join(repo, ".artifacts/behavioral-evals/latest.json");
|
|
const snapshotsPath = process.env.BEHAVIORAL_EVAL_SNAPSHOTS_PATH
|
|
|| path.join(repo, "evals/behavioral/snapshots.json");
|
|
if (mode === "accept") {
|
|
const snapshotFile = acceptReport({
|
|
report: JSON.parse(readFileSync(latestPath, "utf8")),
|
|
manifest,
|
|
repo,
|
|
snapshotsPath,
|
|
});
|
|
console.log(`Accepted ${snapshotFile.snapshots.length} behavioral snapshots`);
|
|
return;
|
|
}
|
|
const report = await runSuite({ repo, cases: manifest.cases });
|
|
const outputPath = writeReport(report, latestPath);
|
|
const passed = report.cases.filter(({ status }) => status === "pass").length;
|
|
console.log(outputPath);
|
|
console.log(`${passed}/${report.cases.length} cases passed`);
|
|
if (report.status !== "pass") process.exitCode = 1;
|
|
}
|
|
|
|
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main();
|