import { createHash, randomBytes } from "node:crypto"; import { spawn } from "node:child_process"; import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, statSync, writeFileSync, } from "node:fs"; import { createServer } from "node:net"; import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; const MANIFEST_FIELDS = new Set(["version", "targetSkills", "cases"]); const CASE_FIELDS = new Set([ "id", "skill", "category", "timeoutMs", "prompt", "sentinels", "fictitiousSecrets", "required", "forbidden", ]); const REQUIRED_FIELDS = new Set(["id", "anyOf"]); const FORBIDDEN_FIELDS = new Set(["id", "values"]); const MUTATION_FIELDS = new Set(["id", "caseId", "operation", "from", "value", "expectedFailure"]); const REPORT_FIELDS = new Set([ "version", "model", "opencodeVersion", "startedAt", "completedAt", "status", "cases", ]); const REPORT_CASE_FIELDS = new Set([ "caseId", "skill", "category", "status", "failures", "response", "model", "opencodeVersion", "skillHash", "caseHash", "durationMs", "completedAt", ]); const SNAPSHOT_FILE_FIELDS = new Set(["version", "snapshots"]); const SNAPSHOT_FIELDS = new Set([ "caseId", "model", "opencodeVersion", "skillHash", "caseHash", "verdict", "response", ]); export function sha256(value) { return createHash("sha256").update(value).digest("hex"); } function canonical(value) { if (Array.isArray(value)) return value.map(canonical); if (value && typeof value === "object") { return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])])); } return value; } export function caseHash(testCase) { return sha256(JSON.stringify(canonical(testCase))); } export function hashSkill(repo, skill) { const skillPath = path.join(repo, "skills", skill, "SKILL.md"); if (!existsSync(skillPath)) throw new TypeError(`skill ${skill} must have a SKILL.md`); return sha256(readFileSync(skillPath)); } function isObject(value) { return value !== null && typeof value === "object" && !Array.isArray(value); } function assertObject(value, label) { if (!isObject(value)) throw new TypeError(`${label} must be an object`); } function assertFields(value, fields, label) { for (const field of Object.keys(value)) { if (!fields.has(field)) throw new TypeError(`${label}: unknown field ${field}`); } for (const field of fields) { if (!Object.hasOwn(value, field)) throw new TypeError(`${label}: missing field ${field}`); } } function assertNonEmptyString(value, label) { if (typeof value !== "string" || value.trim() === "") { throw new TypeError(`${label} must be a non-empty string`); } } function copyStringArray(value, label, allowEmpty = true) { if (!Array.isArray(value) || (!allowEmpty && value.length === 0)) { throw new TypeError(`${label} must be a non-empty array`); } return value.map((entry, index) => { assertNonEmptyString(entry, `${label}[${index}]`); return entry; }); } function copyOracles(value, fields, valuesField, label) { if (!Array.isArray(value) || value.length === 0) { throw new TypeError(`${label} must be a non-empty array`); } return value.map((oracle, index) => { const oracleLabel = `${label}[${index}]`; assertObject(oracle, oracleLabel); assertFields(oracle, fields, oracleLabel); assertNonEmptyString(oracle.id, `${oracleLabel}.id`); return { id: oracle.id, [valuesField]: copyStringArray(oracle[valuesField], `${oracleLabel}.${valuesField}`, false), }; }); } function deepFreeze(value) { if (value && typeof value === "object" && !Object.isFrozen(value)) { for (const entry of Object.values(value)) deepFreeze(entry); Object.freeze(value); } return value; } export function validateManifest(value) { assertObject(value, "manifest"); assertFields(value, MANIFEST_FIELDS, "manifest"); if (value.version !== 1) throw new TypeError("manifest.version must equal 1"); const targetSkills = copyStringArray(value.targetSkills, "manifest.targetSkills", false); const targetSkillSet = new Set(); for (const skill of targetSkills) { if (targetSkillSet.has(skill)) throw new TypeError(`duplicate target skill ${skill}`); targetSkillSet.add(skill); } if (!Array.isArray(value.cases)) throw new TypeError("manifest.cases must be an array"); const caseIds = new Set(); const sentinels = new Set(); const coveredSkills = new Set(); const cases = value.cases.map((testCase, index) => { const initialLabel = isObject(testCase) && typeof testCase.id === "string" && testCase.id !== "" ? testCase.id : `case[${index}]`; assertObject(testCase, initialLabel); assertFields(testCase, CASE_FIELDS, initialLabel); assertNonEmptyString(testCase.id, `${initialLabel}.id`); assertNonEmptyString(testCase.skill, `${testCase.id}.skill`); assertNonEmptyString(testCase.category, `${testCase.id}.category`); assertNonEmptyString(testCase.prompt, `${testCase.id}.prompt`); if (!Number.isInteger(testCase.timeoutMs) || testCase.timeoutMs < 10 || testCase.timeoutMs > 300000) { throw new TypeError(`${testCase.id}.timeoutMs must be an integer from 10 through 300000`); } if (caseIds.has(testCase.id)) throw new TypeError(`duplicate case id ${testCase.id}`); if (!targetSkillSet.has(testCase.skill)) { throw new TypeError(`${testCase.id}: case uses undeclared skill ${testCase.skill}`); } const caseSentinels = copyStringArray(testCase.sentinels, `${testCase.id}.sentinels`); for (const sentinel of caseSentinels) { if (sentinels.has(sentinel)) throw new TypeError(`duplicate sentinel ${sentinel}`); sentinels.add(sentinel); } caseIds.add(testCase.id); coveredSkills.add(testCase.skill); return { id: testCase.id, skill: testCase.skill, category: testCase.category, timeoutMs: testCase.timeoutMs, prompt: testCase.prompt, sentinels: caseSentinels, fictitiousSecrets: copyStringArray( testCase.fictitiousSecrets, `${testCase.id}.fictitiousSecrets`, ), required: copyOracles( testCase.required, REQUIRED_FIELDS, "anyOf", `${testCase.id}.required`, ), forbidden: copyOracles( testCase.forbidden, FORBIDDEN_FIELDS, "values", `${testCase.id}.forbidden`, ), }; }); for (const skill of targetSkills) { if (!coveredSkills.has(skill)) throw new TypeError(`target skill ${skill} is without a case`); } return deepFreeze({ version: 1, targetSkills, cases }); } export function validateMutations(value, manifest) { if (!Array.isArray(value)) throw new TypeError("mutations must be an array"); const cases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase])); const mutationIds = new Set(); const mutations = value.map((mutation, index) => { const label = isObject(mutation) && typeof mutation.id === "string" && mutation.id !== "" ? mutation.id : `mutation[${index}]`; assertObject(mutation, label); for (const field of Object.keys(mutation)) { if (!MUTATION_FIELDS.has(field)) throw new TypeError(`${label}: unknown field ${field}`); } for (const field of ["id", "caseId", "operation", "value", "expectedFailure"]) { if (!Object.hasOwn(mutation, field)) throw new TypeError(`${label}: missing field ${field}`); assertNonEmptyString(mutation[field], `${label}.${field}`); } if (mutationIds.has(mutation.id)) throw new TypeError(`duplicate mutation id ${mutation.id}`); mutationIds.add(mutation.id); if (mutation.operation === "append") { if (Object.hasOwn(mutation, "from")) { throw new TypeError(`${label}: append operation forbids from`); } } else if (mutation.operation === "replace") { if (!Object.hasOwn(mutation, "from")) { throw new TypeError(`${label}: replace operation requires from`); } assertNonEmptyString(mutation.from, `${label}.from`); } else { throw new TypeError(`${label}.operation must be append or replace`); } const testCase = cases.get(mutation.caseId); if (!testCase) throw new TypeError(`${label}: unknown case ${mutation.caseId}`); const failureIds = new Set([ ...testCase.required.map(({ id }) => `required:${id}`), ...testCase.forbidden.map(({ id }) => `forbidden:${id}`), ]); if (testCase.sentinels.length > 0) failureIds.add("forbidden:sentinel"); if (testCase.fictitiousSecrets.length > 0) failureIds.add("forbidden:fictitious-secret"); if (!failureIds.has(mutation.expectedFailure)) { throw new TypeError(`${label}: unknown expected failure ${mutation.expectedFailure}`); } return Object.hasOwn(mutation, "from") ? { id: mutation.id, caseId: mutation.caseId, operation: mutation.operation, from: mutation.from, value: mutation.value, expectedFailure: mutation.expectedFailure, } : { id: mutation.id, caseId: mutation.caseId, operation: mutation.operation, value: mutation.value, expectedFailure: mutation.expectedFailure, }; }); return deepFreeze(mutations); } export function gradeResponse(testCase, response) { if (typeof response !== "string" || response.trim() === "") { return { status: "fail", failures: ["incomplete:empty-response"] }; } const folded = response.toLocaleLowerCase("en-US"); const failures = []; for (const oracle of testCase.required) { if (!oracle.anyOf.some((value) => folded.includes(value.toLocaleLowerCase("en-US")))) { failures.push(`required:${oracle.id}`); } } for (const oracle of testCase.forbidden) { if (oracle.values.some((value) => folded.includes(value.toLocaleLowerCase("en-US")))) { failures.push(`forbidden:${oracle.id}`); } } if (testCase.sentinels.some((value) => folded.includes(value.toLocaleLowerCase("en-US")))) { failures.push("forbidden:sentinel"); } if (testCase.fictitiousSecrets.some((value) => folded.includes(value.toLocaleLowerCase("en-US")))) { failures.push("forbidden:fictitious-secret"); } return { status: failures.length === 0 ? "pass" : "fail", failures }; } function assertHash(value, label) { if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) { throw new TypeError(`${label} must be 64 lowercase hexadecimal characters`); } } function assertTimestamp(value, label) { const parsed = typeof value === "string" ? new Date(value) : undefined; if (!parsed || Number.isNaN(parsed.getTime()) || parsed.toISOString() !== value) { throw new TypeError(`${label} must be an ISO timestamp`); } } export function validateReport(value, manifest, repo) { assertObject(value, "report"); assertFields(value, REPORT_FIELDS, "report"); if (value.version !== 1) throw new TypeError("report.version must equal 1"); assertNonEmptyString(value.model, "report.model"); assertNonEmptyString(value.opencodeVersion, "report.opencodeVersion"); assertTimestamp(value.startedAt, "report.startedAt"); assertTimestamp(value.completedAt, "report.completedAt"); if (value.status !== "pass") throw new TypeError("report status must be pass"); if (!Array.isArray(value.cases)) throw new TypeError("report.cases must be an array"); const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase])); const seen = new Set(); const cases = value.cases.map((entry, index) => { const label = isObject(entry) && typeof entry.caseId === "string" && entry.caseId !== "" ? entry.caseId : `report.cases[${index}]`; assertObject(entry, label); assertFields(entry, REPORT_CASE_FIELDS, label); assertNonEmptyString(entry.caseId, `${label}.caseId`); if (seen.has(entry.caseId)) throw new TypeError(`duplicate case ${entry.caseId}`); seen.add(entry.caseId); const testCase = currentCases.get(entry.caseId); if (!testCase) throw new TypeError(`unknown case ${entry.caseId}`); assertNonEmptyString(entry.skill, `${label}.skill`); assertNonEmptyString(entry.category, `${label}.category`); if (entry.skill !== testCase.skill) throw new TypeError(`${label}: skill does not match manifest`); if (entry.category !== testCase.category) { throw new TypeError(`${label}: category does not match manifest`); } if (entry.status !== "pass") throw new TypeError(`${label}: case status must be pass`); if (!Array.isArray(entry.failures) || entry.failures.length !== 0) { throw new TypeError(`${label}: passing case failures must be empty`); } assertNonEmptyString(entry.response, `${label}.response`); assertNonEmptyString(entry.model, `${label}.model`); assertNonEmptyString(entry.opencodeVersion, `${label}.opencodeVersion`); if (entry.model !== value.model) throw new TypeError(`${label}: model does not match report`); if (entry.opencodeVersion !== value.opencodeVersion) { throw new TypeError(`${label}: OpenCode version does not match report`); } assertHash(entry.skillHash, `${label}.skillHash`); assertHash(entry.caseHash, `${label}.caseHash`); if (entry.skillHash !== hashSkill(repo, testCase.skill)) { throw new TypeError(`${label}: stale skill hash`); } if (entry.caseHash !== caseHash(testCase)) throw new TypeError(`${label}: stale case hash`); if (!Number.isInteger(entry.durationMs) || entry.durationMs < 0) { throw new TypeError(`${label}.durationMs must be a non-negative integer`); } assertTimestamp(entry.completedAt, `${label}.completedAt`); const grade = gradeResponse(testCase, entry.response); if (grade.status !== "pass") { throw new TypeError(`${label}: response failed grading (${grade.failures.join(", ")})`); } return { caseId: entry.caseId, skill: entry.skill, category: entry.category, status: entry.status, failures: [], response: entry.response, model: entry.model, opencodeVersion: entry.opencodeVersion, skillHash: entry.skillHash, caseHash: entry.caseHash, durationMs: entry.durationMs, completedAt: entry.completedAt, }; }); for (const testCase of manifest.cases) { if (!seen.has(testCase.id)) throw new TypeError(`missing case ${testCase.id}`); } return deepFreeze({ version: 1, model: value.model, opencodeVersion: value.opencodeVersion, startedAt: value.startedAt, completedAt: value.completedAt, status: "pass", cases, }); } export function acceptReport({ report, manifest, repo, snapshotsPath }) { const validated = validateReport(report, manifest, repo); const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase])); const snapshots = validated.cases.map((entry) => { const testCase = currentCases.get(entry.caseId); return { caseId: entry.caseId, model: entry.model, opencodeVersion: entry.opencodeVersion, skillHash: entry.skillHash, caseHash: entry.caseHash, verdict: "pass", response: normalizeResponse(redactResponse(entry.response, testCase)), }; }).sort((left, right) => (left.caseId < right.caseId ? -1 : left.caseId > right.caseId ? 1 : 0)); const snapshotFile = deepFreeze({ version: 1, snapshots }); const replayed = replaySnapshots(snapshotFile, manifest, repo); if (replayed.status !== "pass") { const detail = replayed.failures .map(({ caseId, failures }) => `${caseId}: ${failures.join(", ")}`) .join("; "); throw new TypeError(`snapshot responses failed replay grading: ${detail}`); } mkdirSync(path.dirname(snapshotsPath), { recursive: true }); const tempPath = `${snapshotsPath}.tmp`; writeFileSync(tempPath, `${JSON.stringify(snapshotFile, null, 2)}\n`); renameSync(tempPath, snapshotsPath); return snapshotFile; } export function validateSnapshots(value, manifest, repo) { assertObject(value, "snapshots"); assertFields(value, SNAPSHOT_FILE_FIELDS, "snapshots"); if (value.version !== 1) throw new TypeError("snapshots.version must equal 1"); if (!Array.isArray(value.snapshots)) throw new TypeError("snapshots.snapshots must be an array"); const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase])); const seen = new Set(); const snapshots = value.snapshots.map((snapshot, index) => { const label = isObject(snapshot) && typeof snapshot.caseId === "string" && snapshot.caseId !== "" ? snapshot.caseId : `snapshots[${index}]`; assertObject(snapshot, label); assertFields(snapshot, SNAPSHOT_FIELDS, label); assertNonEmptyString(snapshot.caseId, `${label}.caseId`); if (seen.has(snapshot.caseId)) throw new TypeError(`duplicate snapshot ${snapshot.caseId}`); seen.add(snapshot.caseId); const testCase = currentCases.get(snapshot.caseId); if (!testCase) throw new TypeError(`unknown snapshot case ${snapshot.caseId}`); assertNonEmptyString(snapshot.model, `${label}.model`); assertNonEmptyString(snapshot.opencodeVersion, `${label}.opencodeVersion`); assertHash(snapshot.skillHash, `${label}.skillHash`); assertHash(snapshot.caseHash, `${label}.caseHash`); if (snapshot.skillHash !== hashSkill(repo, testCase.skill)) { throw new TypeError(`${label}: stale skill hash`); } if (snapshot.caseHash !== caseHash(testCase)) throw new TypeError(`${label}: stale case hash`); if (snapshot.verdict !== "pass") throw new TypeError(`${label}: verdict must be pass`); assertNonEmptyString(snapshot.response, `${label}.response`); if (normalizeResponse(redactResponse(snapshot.response, testCase)) !== snapshot.response) { throw new TypeError(`${label}: response must be normalized and redacted`); } return { caseId: snapshot.caseId, model: snapshot.model, opencodeVersion: snapshot.opencodeVersion, skillHash: snapshot.skillHash, caseHash: snapshot.caseHash, verdict: "pass", response: snapshot.response, }; }); for (const testCase of manifest.cases) { if (!seen.has(testCase.id)) throw new TypeError(`missing snapshot ${testCase.id}`); } snapshots.sort((left, right) => ( left.caseId < right.caseId ? -1 : left.caseId > right.caseId ? 1 : 0 )); return deepFreeze({ version: 1, snapshots }); } export function replaySnapshots(snapshotFile, manifest, repo) { const validated = validateSnapshots(snapshotFile, manifest, repo); const currentCases = new Map(manifest.cases.map((testCase) => [testCase.id, testCase])); const failures = []; for (const snapshot of validated.snapshots) { const grade = gradeResponse(currentCases.get(snapshot.caseId), snapshot.response); if (grade.status === "fail") { failures.push({ caseId: snapshot.caseId, failures: grade.failures }); } } return deepFreeze({ status: failures.length === 0 ? "pass" : "fail", cases: validated.snapshots.length, failures, }); } export function applyMutation(response, mutation) { if (!isObject(mutation) || typeof mutation.value !== "string") { throw new TypeError("mutation must contain a string value"); } if (mutation.operation === "append") return `${response}\n${mutation.value}`; if (mutation.operation === "replace") return mutation.value; throw new TypeError(`unsupported mutation operation: ${mutation.operation}`); } export function normalizeResponse(response) { return response .replaceAll("\r\n", "\n") .replace(/[ \t]+$/gm, "") .replace(/\n{4,}/g, "\n\n\n") .trim(); } function escapeRegExp(value) { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } export function redactResponse(response, testCase) { let redacted = response; const caseValues = [...testCase.sentinels, ...testCase.fictitiousSecrets] .toSorted((left, right) => right.length - left.length); for (const value of caseValues) { redacted = redacted.replace(new RegExp(escapeRegExp(value), "gi"), "[REDACTED]"); } const credential = /(? 0 && !terminal)) { throw new TypeError("unsuccessful OpenCode turn"); } return { response: textParts.join("\n"), toolRequested }; } /** * V2 loads a configured plugin directory through its index.js entrypoint and * uses an ordered permission ruleset. The eval denies every tool so a case * can only pass on model text. */ export function buildEvalConfig(pluginDirectory) { return { plugins: [pluginDirectory], permissions: [{ action: "*", resource: "*", effect: "deny" }], }; } export function parseModelReference(model) { const match = typeof model === "string" && model.match(/^([^/#\s]+)\/([^/#\s]+)(?:#([^/#\s]+))?$/); if (!match) throw new TypeError("OPENCODE_EVAL_MODEL must use provider/model form with an optional #variant"); return { providerID: match[1], id: match[2], ...(match[3] ? { variant: match[3] } : {}) }; } async function availablePort() { const server = createServer(); await new Promise((resolve, reject) => server.listen(0, "127.0.0.1", resolve).once("error", reject)); const address = server.address(); await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); return address.port; } /** * Writes the plugin-directory shim the eval runtime loads. The shim keeps the * package entrypoint in the repository, so its skills and dependencies resolve * exactly as they do for an installed package. */ function writePluginShim(runtime, repo) { const directory = path.join(runtime.root, "plugin"); mkdirSync(directory, { recursive: true }); const entrypoint = pathToFileURL( path.join(repo, ".opencode/plugins/opencode-power-pack.js"), ).href; writeFileSync(path.join(directory, "index.js"), `export { default } from ${JSON.stringify(entrypoint)};\n`); return directory; } export function defaultOpenCodeCommand(platform = process.platform) { return platform === "win32" ? "opencode.exe" : "opencode"; } function signalChild(child, signal) { try { if (process.platform === "win32") child.kill(signal); else process.kill(-child.pid, signal); } catch (error) { if (error.code !== "ESRCH") throw error; } } function delay(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } function processGroupExists(pid) { try { process.kill(-pid, 0); return true; } catch (error) { if (error.code === "ESRCH") return false; throw error; } } async function terminateTimedOutChild(child) { if (process.platform === "win32") { signalChild(child, "SIGTERM"); await delay(100); if (child.exitCode === null && child.signalCode === null) signalChild(child, "SIGKILL"); return; } signalChild(child, "SIGTERM"); await delay(100); signalChild(child, "SIGKILL"); while (processGroupExists(child.pid)) await delay(10); } function reportEntry(testCase, options, started, status, failures, response) { return { caseId: testCase.id, skill: testCase.skill, category: testCase.category, status, failures, response, model: options.model, opencodeVersion: options.opencodeVersion, skillHash: hashSkill(options.repo, testCase.skill), caseHash: caseHash(testCase), durationMs: Date.now() - started, completedAt: new Date().toISOString(), }; } function createRuntime(env) { const root = mkdtempSync(path.join(tmpdir(), "behavioral-eval-")); try { const runtime = { root, project: path.join(root, "project"), home: path.join(root, "home"), config: path.join(root, "config"), data: path.join(root, "data"), cache: path.join(root, "cache"), state: path.join(root, "state"), }; for (const directory of [ runtime.project, runtime.home, runtime.config, runtime.data, runtime.cache, runtime.state, ]) { mkdirSync(directory, { recursive: true, mode: 0o700 }); } const sourceData = env.XDG_DATA_HOME || (env.HOME ? path.join(env.HOME, ".local", "share") : undefined); const sourceAuth = sourceData && path.join(sourceData, "opencode", "auth.json"); if (sourceAuth && existsSync(sourceAuth)) { const sourceAuthStat = statSync(sourceAuth); if (!sourceAuthStat.isFile()) return runtime; const targetDirectory = path.join(runtime.data, "opencode"); const targetAuth = path.join(targetDirectory, "auth.json"); mkdirSync(targetDirectory, { recursive: true, mode: 0o700 }); copyFileSync(sourceAuth, targetAuth); chmodSync(targetAuth, (sourceAuthStat.mode & 0o600) || 0o600); } return runtime; } catch (error) { rmSync(root, { recursive: true, force: true }); throw error; } } function isolatedEnv(env, runtime, model, config, password) { const childEnv = {}; const isolatedPaths = new Set([ "HOME", "USERPROFILE", "APPDATA", "LOCALAPPDATA", "XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME", ]); for (const [key, value] of Object.entries(env)) { const normalized = key.toUpperCase(); if (isolatedPaths.has(normalized)) continue; if (normalized.startsWith("OPENCODE_")) continue; childEnv[key] = value; } return { ...childEnv, HOME: runtime.home, USERPROFILE: runtime.home, APPDATA: runtime.config, LOCALAPPDATA: runtime.data, XDG_CONFIG_HOME: runtime.config, XDG_DATA_HOME: runtime.data, XDG_CACHE_HOME: runtime.cache, XDG_STATE_HOME: runtime.state, OPENCODE_EVAL_MODEL: model, OPENCODE_SERVER_PASSWORD: password, OPENCODE_CONFIG_CONTENT: JSON.stringify(config), }; } async function requestJson(url, { method = "GET", body, authorization, signal } = {}) { const response = await fetch(url, { method, headers: { ...(authorization ? { authorization } : {}), ...(body === undefined ? {} : { "content-type": "application/json" }), }, body: body === undefined ? undefined : JSON.stringify(body), signal, }); if (!response.ok) { throw new TypeError(`OpenCode API ${method} ${url} returned HTTP ${response.status}`); } const text = await response.text(); if (text === "") return undefined; try { return JSON.parse(text); } catch { throw new TypeError("malformed OpenCode API response"); } } async function waitForServer(baseURL, { authorization, signal, child, spawnError }) { while (!signal.aborted) { if (spawnError.error || child.exitCode !== null || child.signalCode !== null) return false; try { const response = await fetch(`${baseURL}/api/info`, { headers: { authorization }, signal: AbortSignal.any([signal, AbortSignal.timeout(1_000)]), }); if (response.ok) return true; } catch { if (signal.aborted) break; } await delay(100); } return false; } export async function runCase(testCase, options) { const { repo, model, opencodeVersion, command = defaultOpenCodeCommand(), commandArgsPrefix = [], env = process.env, } = options; const modelReference = parseModelReference(model); const started = Date.now(); const runtime = createRuntime(env); const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), testCase.timeoutMs); const spawnError = { error: undefined }; let child; const finish = (status, failures, response) => reportEntry( testCase, options, started, status, failures, response, ); try { const pluginDirectory = writePluginShim(runtime, repo); const port = await availablePort(); const password = randomBytes(24).toString("base64url"); const authorization = `Basic ${Buffer.from(`opencode:${password}`).toString("base64")}`; const baseURL = `http://127.0.0.1:${port}`; const args = [...commandArgsPrefix, "serve", "--hostname", "127.0.0.1", "--port", String(port)]; child = spawn(command, args, { cwd: runtime.project, env: isolatedEnv(env, runtime, model, buildEvalConfig(pluginDirectory), password), detached: process.platform !== "win32", shell: false, stdio: ["ignore", "pipe", "pipe"], }); child.stdout.resume(); child.stderr.resume(); child.once("error", (error) => { spawnError.error = error; }); const ready = await waitForServer(baseURL, { authorization, signal: controller.signal, child, spawnError }); if (!ready) { if (controller.signal.aborted) return finish("incomplete", ["incomplete:timeout"], ""); return finish("incomplete", ["incomplete:process-exit"], ""); } try { const created = await requestJson(`${baseURL}/api/session`, { method: "POST", authorization, signal: controller.signal, body: { title: testCase.id, model: modelReference }, }); const sessionID = created?.data?.id; if (typeof sessionID !== "string" || sessionID === "") { throw new TypeError("malformed OpenCode session"); } await requestJson(`${baseURL}/api/session/${sessionID}/prompt`, { method: "POST", authorization, signal: controller.signal, body: { text: testCase.prompt, skills: [{ id: testCase.skill }] }, }); await requestJson(`${baseURL}/api/experimental/session/${sessionID}/wait`, { method: "POST", authorization, signal: controller.signal, }); const context = await requestJson(`${baseURL}/api/session/${sessionID}/context`, { authorization, signal: controller.signal, }); let parsed; try { parsed = parseSessionContext(context); } catch { return finish("incomplete", ["incomplete:malformed-response"], ""); } const { response, toolRequested } = parsed; if (toolRequested) { return finish("incomplete", ["incomplete:permission"], ""); } if (response.trim() === "") { return finish("incomplete", ["incomplete:missing-response"], ""); } const grade = gradeResponse(testCase, response); return finish( grade.status, grade.failures, normalizeResponse(redactResponse(response, testCase)), ); } catch { if (controller.signal.aborted) return finish("incomplete", ["incomplete:timeout"], ""); return finish("incomplete", ["incomplete:malformed-response"], ""); } } finally { clearTimeout(timeout); if (child && child.exitCode === null && child.signalCode === null) { await terminateTimedOutChild(child); } rmSync(runtime.root, { recursive: true, force: true }); } } export function writeReport( report, outputPath = path.resolve(".artifacts/behavioral-evals/latest.json"), ) { mkdirSync(path.dirname(outputPath), { recursive: true }); const tempPath = `${outputPath}.tmp`; try { writeFileSync(tempPath, `${JSON.stringify(report, null, 2)}\n`); renameSync(tempPath, outputPath); } catch (error) { rmSync(tempPath, { force: true }); throw error; } return outputPath; } async function detectOpenCodeVersion(command, commandArgsPrefix, env, cwd) { return await new Promise((resolve, reject) => { const chunks = []; const child = spawn(command, [...commandArgsPrefix, "--version"], { cwd, env, shell: false, stdio: ["ignore", "pipe", "pipe"], }); child.stdout.on("data", (chunk) => chunks.push(chunk)); child.stderr.resume(); child.once("error", reject); child.once("close", (code) => { if (code !== 0) { reject(new Error("Unable to detect OpenCode version")); return; } resolve(Buffer.concat(chunks).toString("utf8").trim()); }); }); } export async function runSuite(options) { const { repo, cases, env = process.env, command = defaultOpenCodeCommand(), commandArgsPrefix = [], } = options; const startedAt = new Date().toISOString(); const model = env.OPENCODE_EVAL_MODEL; parseModelReference(model); if (cases.length === 0) throw new TypeError("case selection must not be empty"); const opencodeVersion = await detectOpenCodeVersion(command, commandArgsPrefix, env, repo); const entries = []; for (const testCase of cases) { entries.push(await runCase(testCase, { repo, model, opencodeVersion, command, commandArgsPrefix, env, })); } return { version: 1, model, opencodeVersion, startedAt, completedAt: new Date().toISOString(), status: entries.every(({ status }) => status === "pass") ? "pass" : "fail", cases: entries, }; } async function main() { const mode = process.argv[2]; if (mode !== "run" && mode !== "accept") { throw new TypeError(`unsupported behavioral evaluation mode: ${mode || "missing"}`); } const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const manifest = validateManifest(JSON.parse( readFileSync(path.join(repo, "evals/behavioral/cases.json"), "utf8"), )); const latestPath = process.env.BEHAVIORAL_EVAL_LATEST_PATH || path.join(repo, ".artifacts/behavioral-evals/latest.json"); const snapshotsPath = process.env.BEHAVIORAL_EVAL_SNAPSHOTS_PATH || path.join(repo, "evals/behavioral/snapshots.json"); if (mode === "accept") { const snapshotFile = acceptReport({ report: JSON.parse(readFileSync(latestPath, "utf8")), manifest, repo, snapshotsPath, }); console.log(`Accepted ${snapshotFile.snapshots.length} behavioral snapshots`); return; } const report = await runSuite({ repo, cases: manifest.cases }); const outputPath = writeReport(report, latestPath); const passed = report.cases.filter(({ status }) => status === "pass").length; console.log(outputPath); console.log(`${passed}/${report.cases.length} cases passed`); if (report.status !== "pass") process.exitCode = 1; } if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main();