mirror of
https://github.com/waybarrios/opencode-power-pack.git
synced 2026-11-19 02:16:19 +00:00
242 lines
8.4 KiB
JavaScript
242 lines
8.4 KiB
JavaScript
import { createHash } from "node:crypto";
|
|
|
|
const MANIFEST_FIELDS = new Set(["version", "targetSkills", "cases"]);
|
|
const CASE_FIELDS = new Set([
|
|
"id", "skill", "category", "timeoutMs", "prompt", "sentinels",
|
|
"fictitiousSecrets", "required", "forbidden",
|
|
]);
|
|
const REQUIRED_FIELDS = new Set(["id", "anyOf"]);
|
|
const FORBIDDEN_FIELDS = new Set(["id", "values"]);
|
|
|
|
export function sha256(value) {
|
|
return createHash("sha256").update(value).digest("hex");
|
|
}
|
|
|
|
function canonical(value) {
|
|
if (Array.isArray(value)) return value.map(canonical);
|
|
if (value && typeof value === "object") {
|
|
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])]));
|
|
}
|
|
return value;
|
|
}
|
|
|
|
export function caseHash(testCase) {
|
|
return sha256(JSON.stringify(canonical(testCase)));
|
|
}
|
|
|
|
function isObject(value) {
|
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
}
|
|
|
|
function assertObject(value, label) {
|
|
if (!isObject(value)) throw new TypeError(`${label} must be an object`);
|
|
}
|
|
|
|
function assertFields(value, fields, label) {
|
|
for (const field of Object.keys(value)) {
|
|
if (!fields.has(field)) throw new TypeError(`${label}: unknown field ${field}`);
|
|
}
|
|
for (const field of fields) {
|
|
if (!Object.hasOwn(value, field)) throw new TypeError(`${label}: missing field ${field}`);
|
|
}
|
|
}
|
|
|
|
function assertNonEmptyString(value, label) {
|
|
if (typeof value !== "string" || value.trim() === "") {
|
|
throw new TypeError(`${label} must be a non-empty string`);
|
|
}
|
|
}
|
|
|
|
function copyStringArray(value, label, allowEmpty = true) {
|
|
if (!Array.isArray(value) || (!allowEmpty && value.length === 0)) {
|
|
throw new TypeError(`${label} must be a non-empty array`);
|
|
}
|
|
return value.map((entry, index) => {
|
|
assertNonEmptyString(entry, `${label}[${index}]`);
|
|
return entry;
|
|
});
|
|
}
|
|
|
|
function copyOracles(value, fields, valuesField, label) {
|
|
if (!Array.isArray(value) || value.length === 0) {
|
|
throw new TypeError(`${label} must be a non-empty array`);
|
|
}
|
|
return value.map((oracle, index) => {
|
|
const oracleLabel = `${label}[${index}]`;
|
|
assertObject(oracle, oracleLabel);
|
|
assertFields(oracle, fields, oracleLabel);
|
|
assertNonEmptyString(oracle.id, `${oracleLabel}.id`);
|
|
return {
|
|
id: oracle.id,
|
|
[valuesField]: copyStringArray(oracle[valuesField], `${oracleLabel}.${valuesField}`, false),
|
|
};
|
|
});
|
|
}
|
|
|
|
function deepFreeze(value) {
|
|
if (value && typeof value === "object" && !Object.isFrozen(value)) {
|
|
for (const entry of Object.values(value)) deepFreeze(entry);
|
|
Object.freeze(value);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
export function validateManifest(value) {
|
|
assertObject(value, "manifest");
|
|
assertFields(value, MANIFEST_FIELDS, "manifest");
|
|
if (value.version !== 1) throw new TypeError("manifest.version must equal 1");
|
|
|
|
const targetSkills = copyStringArray(value.targetSkills, "manifest.targetSkills", false);
|
|
const targetSkillSet = new Set();
|
|
for (const skill of targetSkills) {
|
|
if (targetSkillSet.has(skill)) throw new TypeError(`duplicate target skill ${skill}`);
|
|
targetSkillSet.add(skill);
|
|
}
|
|
if (!Array.isArray(value.cases)) throw new TypeError("manifest.cases must be an array");
|
|
|
|
const caseIds = new Set();
|
|
const sentinels = new Set();
|
|
const coveredSkills = new Set();
|
|
const cases = value.cases.map((testCase, index) => {
|
|
const initialLabel = isObject(testCase) && typeof testCase.id === "string" && testCase.id !== ""
|
|
? testCase.id
|
|
: `case[${index}]`;
|
|
assertObject(testCase, initialLabel);
|
|
assertFields(testCase, CASE_FIELDS, initialLabel);
|
|
assertNonEmptyString(testCase.id, `${initialLabel}.id`);
|
|
assertNonEmptyString(testCase.skill, `${testCase.id}.skill`);
|
|
assertNonEmptyString(testCase.category, `${testCase.id}.category`);
|
|
assertNonEmptyString(testCase.prompt, `${testCase.id}.prompt`);
|
|
if (!Number.isInteger(testCase.timeoutMs)
|
|
|| testCase.timeoutMs < 10
|
|
|| testCase.timeoutMs > 300000) {
|
|
throw new TypeError(`${testCase.id}.timeoutMs must be an integer from 10 through 300000`);
|
|
}
|
|
if (caseIds.has(testCase.id)) throw new TypeError(`duplicate case id ${testCase.id}`);
|
|
if (!targetSkillSet.has(testCase.skill)) {
|
|
throw new TypeError(`${testCase.id}: case uses undeclared skill ${testCase.skill}`);
|
|
}
|
|
|
|
const caseSentinels = copyStringArray(testCase.sentinels, `${testCase.id}.sentinels`);
|
|
for (const sentinel of caseSentinels) {
|
|
if (sentinels.has(sentinel)) throw new TypeError(`duplicate sentinel ${sentinel}`);
|
|
sentinels.add(sentinel);
|
|
}
|
|
caseIds.add(testCase.id);
|
|
coveredSkills.add(testCase.skill);
|
|
|
|
return {
|
|
id: testCase.id,
|
|
skill: testCase.skill,
|
|
category: testCase.category,
|
|
timeoutMs: testCase.timeoutMs,
|
|
prompt: testCase.prompt,
|
|
sentinels: caseSentinels,
|
|
fictitiousSecrets: copyStringArray(
|
|
testCase.fictitiousSecrets,
|
|
`${testCase.id}.fictitiousSecrets`,
|
|
),
|
|
required: copyOracles(
|
|
testCase.required,
|
|
REQUIRED_FIELDS,
|
|
"anyOf",
|
|
`${testCase.id}.required`,
|
|
),
|
|
forbidden: copyOracles(
|
|
testCase.forbidden,
|
|
FORBIDDEN_FIELDS,
|
|
"values",
|
|
`${testCase.id}.forbidden`,
|
|
),
|
|
};
|
|
});
|
|
|
|
for (const skill of targetSkills) {
|
|
if (!coveredSkills.has(skill)) throw new TypeError(`target skill ${skill} is without a case`);
|
|
}
|
|
|
|
return deepFreeze({ version: 1, targetSkills, cases });
|
|
}
|
|
|
|
export function gradeResponse(testCase, response) {
|
|
if (typeof response !== "string" || response.trim() === "") {
|
|
return { status: "fail", failures: ["incomplete:empty-response"] };
|
|
}
|
|
const folded = response.toLocaleLowerCase("en-US");
|
|
const failures = [];
|
|
for (const oracle of testCase.required) {
|
|
if (!oracle.anyOf.some((value) => folded.includes(value.toLocaleLowerCase("en-US")))) {
|
|
failures.push(`required:${oracle.id}`);
|
|
}
|
|
}
|
|
for (const oracle of testCase.forbidden) {
|
|
if (oracle.values.some((value) => folded.includes(value.toLocaleLowerCase("en-US")))) {
|
|
failures.push(`forbidden:${oracle.id}`);
|
|
}
|
|
}
|
|
if (testCase.sentinels.some((value) => folded.includes(value.toLocaleLowerCase("en-US")))) {
|
|
failures.push("forbidden:sentinel");
|
|
}
|
|
if (testCase.fictitiousSecrets.some((value) => folded.includes(value.toLocaleLowerCase("en-US")))) {
|
|
failures.push("forbidden:fictitious-secret");
|
|
}
|
|
return { status: failures.length === 0 ? "pass" : "fail", failures };
|
|
}
|
|
|
|
export function applyMutation(response, mutation) {
|
|
if (!isObject(mutation) || typeof mutation.value !== "string") {
|
|
throw new TypeError("mutation must contain a string value");
|
|
}
|
|
if (mutation.operation === "append") return `${response}\n${mutation.value}`;
|
|
if (mutation.operation === "replace") return mutation.value;
|
|
throw new TypeError(`unsupported mutation operation: ${mutation.operation}`);
|
|
}
|
|
|
|
export function normalizeResponse(response) {
|
|
return response
|
|
.replaceAll("\r\n", "\n")
|
|
.replace(/[ \t]+$/gm, "")
|
|
.replace(/\n{4,}/g, "\n\n\n")
|
|
.trim();
|
|
}
|
|
|
|
function escapeRegExp(value) {
|
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
}
|
|
|
|
export function redactResponse(response, testCase) {
|
|
let redacted = response;
|
|
const caseValues = [...testCase.sentinels, ...testCase.fictitiousSecrets]
|
|
.toSorted((left, right) => right.length - left.length);
|
|
for (const value of caseValues) {
|
|
redacted = redacted.replace(new RegExp(escapeRegExp(value), "gi"), "[REDACTED]");
|
|
}
|
|
const credential = /(?<![A-Za-z0-9_])(?:sk-[A-Za-z0-9][A-Za-z0-9._-]*|ghp_[A-Za-z0-9][A-Za-z0-9._-]*|github_pat_[A-Za-z0-9][A-Za-z0-9._-]*|AKIA[A-Z0-9]{16}|Bearer[ \t]+[A-Za-z0-9][A-Za-z0-9._~+/-]*)(?![A-Za-z0-9._~+/-])/g;
|
|
return redacted.replace(credential, "[REDACTED]");
|
|
}
|
|
|
|
export function parseJsonEvents(stdout) {
|
|
const textParts = [];
|
|
let toolRequested = false;
|
|
for (const line of stdout.split(/\r?\n/)) {
|
|
if (line.trim() === "") continue;
|
|
let event;
|
|
try {
|
|
event = JSON.parse(line);
|
|
} catch {
|
|
throw new TypeError("malformed OpenCode JSON event");
|
|
}
|
|
if (!isObject(event) || !isObject(event.part)) {
|
|
throw new TypeError("malformed OpenCode JSON event");
|
|
}
|
|
if (event.part.type === "tool") toolRequested = true;
|
|
if (event.type === "text" && event.part.type === "text") {
|
|
if (typeof event.part.text !== "string") {
|
|
throw new TypeError("malformed OpenCode JSON event");
|
|
}
|
|
textParts.push(event.part.text);
|
|
}
|
|
}
|
|
return { response: textParts.join("\n"), toolRequested };
|
|
}
|