Files
opencode-power-pack/scripts/behavioral-evals.mjs
T

242 lines
8.4 KiB
JavaScript

import { createHash } from "node:crypto";
const MANIFEST_FIELDS = new Set(["version", "targetSkills", "cases"]);
const CASE_FIELDS = new Set([
"id", "skill", "category", "timeoutMs", "prompt", "sentinels",
"fictitiousSecrets", "required", "forbidden",
]);
const REQUIRED_FIELDS = new Set(["id", "anyOf"]);
const FORBIDDEN_FIELDS = new Set(["id", "values"]);
export function sha256(value) {
return createHash("sha256").update(value).digest("hex");
}
function canonical(value) {
if (Array.isArray(value)) return value.map(canonical);
if (value && typeof value === "object") {
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])]));
}
return value;
}
export function caseHash(testCase) {
return sha256(JSON.stringify(canonical(testCase)));
}
function isObject(value) {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
function assertObject(value, label) {
if (!isObject(value)) throw new TypeError(`${label} must be an object`);
}
function assertFields(value, fields, label) {
for (const field of Object.keys(value)) {
if (!fields.has(field)) throw new TypeError(`${label}: unknown field ${field}`);
}
for (const field of fields) {
if (!Object.hasOwn(value, field)) throw new TypeError(`${label}: missing field ${field}`);
}
}
function assertNonEmptyString(value, label) {
if (typeof value !== "string" || value.trim() === "") {
throw new TypeError(`${label} must be a non-empty string`);
}
}
function copyStringArray(value, label, allowEmpty = true) {
if (!Array.isArray(value) || (!allowEmpty && value.length === 0)) {
throw new TypeError(`${label} must be a non-empty array`);
}
return value.map((entry, index) => {
assertNonEmptyString(entry, `${label}[${index}]`);
return entry;
});
}
function copyOracles(value, fields, valuesField, label) {
if (!Array.isArray(value) || value.length === 0) {
throw new TypeError(`${label} must be a non-empty array`);
}
return value.map((oracle, index) => {
const oracleLabel = `${label}[${index}]`;
assertObject(oracle, oracleLabel);
assertFields(oracle, fields, oracleLabel);
assertNonEmptyString(oracle.id, `${oracleLabel}.id`);
return {
id: oracle.id,
[valuesField]: copyStringArray(oracle[valuesField], `${oracleLabel}.${valuesField}`, false),
};
});
}
function deepFreeze(value) {
if (value && typeof value === "object" && !Object.isFrozen(value)) {
for (const entry of Object.values(value)) deepFreeze(entry);
Object.freeze(value);
}
return value;
}
export function validateManifest(value) {
assertObject(value, "manifest");
assertFields(value, MANIFEST_FIELDS, "manifest");
if (value.version !== 1) throw new TypeError("manifest.version must equal 1");
const targetSkills = copyStringArray(value.targetSkills, "manifest.targetSkills", false);
const targetSkillSet = new Set();
for (const skill of targetSkills) {
if (targetSkillSet.has(skill)) throw new TypeError(`duplicate target skill ${skill}`);
targetSkillSet.add(skill);
}
if (!Array.isArray(value.cases)) throw new TypeError("manifest.cases must be an array");
const caseIds = new Set();
const sentinels = new Set();
const coveredSkills = new Set();
const cases = value.cases.map((testCase, index) => {
const initialLabel = isObject(testCase) && typeof testCase.id === "string" && testCase.id !== ""
? testCase.id
: `case[${index}]`;
assertObject(testCase, initialLabel);
assertFields(testCase, CASE_FIELDS, initialLabel);
assertNonEmptyString(testCase.id, `${initialLabel}.id`);
assertNonEmptyString(testCase.skill, `${testCase.id}.skill`);
assertNonEmptyString(testCase.category, `${testCase.id}.category`);
assertNonEmptyString(testCase.prompt, `${testCase.id}.prompt`);
if (!Number.isInteger(testCase.timeoutMs)
|| testCase.timeoutMs < 10
|| testCase.timeoutMs > 300000) {
throw new TypeError(`${testCase.id}.timeoutMs must be an integer from 10 through 300000`);
}
if (caseIds.has(testCase.id)) throw new TypeError(`duplicate case id ${testCase.id}`);
if (!targetSkillSet.has(testCase.skill)) {
throw new TypeError(`${testCase.id}: case uses undeclared skill ${testCase.skill}`);
}
const caseSentinels = copyStringArray(testCase.sentinels, `${testCase.id}.sentinels`);
for (const sentinel of caseSentinels) {
if (sentinels.has(sentinel)) throw new TypeError(`duplicate sentinel ${sentinel}`);
sentinels.add(sentinel);
}
caseIds.add(testCase.id);
coveredSkills.add(testCase.skill);
return {
id: testCase.id,
skill: testCase.skill,
category: testCase.category,
timeoutMs: testCase.timeoutMs,
prompt: testCase.prompt,
sentinels: caseSentinels,
fictitiousSecrets: copyStringArray(
testCase.fictitiousSecrets,
`${testCase.id}.fictitiousSecrets`,
),
required: copyOracles(
testCase.required,
REQUIRED_FIELDS,
"anyOf",
`${testCase.id}.required`,
),
forbidden: copyOracles(
testCase.forbidden,
FORBIDDEN_FIELDS,
"values",
`${testCase.id}.forbidden`,
),
};
});
for (const skill of targetSkills) {
if (!coveredSkills.has(skill)) throw new TypeError(`target skill ${skill} is without a case`);
}
return deepFreeze({ version: 1, targetSkills, cases });
}
export function gradeResponse(testCase, response) {
if (typeof response !== "string" || response.trim() === "") {
return { status: "fail", failures: ["incomplete:empty-response"] };
}
const folded = response.toLocaleLowerCase("en-US");
const failures = [];
for (const oracle of testCase.required) {
if (!oracle.anyOf.some((value) => folded.includes(value.toLocaleLowerCase("en-US")))) {
failures.push(`required:${oracle.id}`);
}
}
for (const oracle of testCase.forbidden) {
if (oracle.values.some((value) => folded.includes(value.toLocaleLowerCase("en-US")))) {
failures.push(`forbidden:${oracle.id}`);
}
}
if (testCase.sentinels.some((value) => folded.includes(value.toLocaleLowerCase("en-US")))) {
failures.push("forbidden:sentinel");
}
if (testCase.fictitiousSecrets.some((value) => folded.includes(value.toLocaleLowerCase("en-US")))) {
failures.push("forbidden:fictitious-secret");
}
return { status: failures.length === 0 ? "pass" : "fail", failures };
}
export function applyMutation(response, mutation) {
if (!isObject(mutation) || typeof mutation.value !== "string") {
throw new TypeError("mutation must contain a string value");
}
if (mutation.operation === "append") return `${response}\n${mutation.value}`;
if (mutation.operation === "replace") return mutation.value;
throw new TypeError(`unsupported mutation operation: ${mutation.operation}`);
}
export function normalizeResponse(response) {
return response
.replaceAll("\r\n", "\n")
.replace(/[ \t]+$/gm, "")
.replace(/\n{4,}/g, "\n\n\n")
.trim();
}
function escapeRegExp(value) {
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
export function redactResponse(response, testCase) {
let redacted = response;
const caseValues = [...testCase.sentinels, ...testCase.fictitiousSecrets]
.toSorted((left, right) => right.length - left.length);
for (const value of caseValues) {
redacted = redacted.replace(new RegExp(escapeRegExp(value), "gi"), "[REDACTED]");
}
const credential = /(?<![A-Za-z0-9_])(?:sk-[A-Za-z0-9][A-Za-z0-9._-]*|ghp_[A-Za-z0-9][A-Za-z0-9._-]*|github_pat_[A-Za-z0-9][A-Za-z0-9._-]*|AKIA[A-Z0-9]{16}|Bearer[ \t]+[A-Za-z0-9][A-Za-z0-9._~+/-]*)(?![A-Za-z0-9._~+/-])/g;
return redacted.replace(credential, "[REDACTED]");
}
export function parseJsonEvents(stdout) {
const textParts = [];
let toolRequested = false;
for (const line of stdout.split(/\r?\n/)) {
if (line.trim() === "") continue;
let event;
try {
event = JSON.parse(line);
} catch {
throw new TypeError("malformed OpenCode JSON event");
}
if (!isObject(event) || !isObject(event.part)) {
throw new TypeError("malformed OpenCode JSON event");
}
if (event.part.type === "tool") toolRequested = true;
if (event.type === "text" && event.part.type === "text") {
if (typeof event.part.text !== "string") {
throw new TypeError("malformed OpenCode JSON event");
}
textParts.push(event.part.text);
}
}
return { response: textParts.join("\n"), toolRequested };
}