Harden behavioral evaluation isolation

This commit is contained in:
Wayner Barrios
2026-07-31 16:50:05 -04:00
parent 4a74b6f013
commit cd5c150b08
2 changed files with 349 additions and 18 deletions
+123 -17
View File
@@ -1,6 +1,16 @@
import { createHash } from "node:crypto";
import { spawn } from "node:child_process";
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import {
chmodSync,
copyFileSync,
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
statSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
@@ -316,6 +326,10 @@ export function buildEvalConfig(pluginUrl) {
return { plugin: [pluginUrl], permission: { "*": "deny" } };
}
export function defaultOpenCodeCommand(platform = process.platform) {
return platform === "win32" ? "opencode.exe" : "opencode";
}
function signalChild(child, signal) {
try {
if (process.platform === "win32") child.kill(signal);
@@ -325,6 +339,34 @@ function signalChild(child, signal) {
}
}
function delay(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function processGroupExists(pid) {
try {
process.kill(-pid, 0);
return true;
} catch (error) {
if (error.code === "ESRCH") return false;
throw error;
}
}
async function terminateTimedOutChild(child) {
if (process.platform === "win32") {
signalChild(child, "SIGTERM");
await delay(100);
if (child.exitCode === null && child.signalCode === null) signalChild(child, "SIGKILL");
return;
}
signalChild(child, "SIGTERM");
await delay(100);
signalChild(child, "SIGKILL");
while (processGroupExists(child.pid)) await delay(10);
}
function reportEntry(testCase, options, started, status, failures, response) {
return {
caseId: testCase.id,
@@ -342,17 +384,81 @@ function reportEntry(testCase, options, started, status, failures, response) {
};
}
function createRuntime(env) {
const root = mkdtempSync(path.join(tmpdir(), "behavioral-eval-"));
try {
const runtime = {
root,
project: path.join(root, "project"),
home: path.join(root, "home"),
config: path.join(root, "config"),
data: path.join(root, "data"),
cache: path.join(root, "cache"),
state: path.join(root, "state"),
};
for (const directory of [
runtime.project, runtime.home, runtime.config, runtime.data, runtime.cache, runtime.state,
]) {
mkdirSync(directory, { recursive: true, mode: 0o700 });
}
const sourceData = env.XDG_DATA_HOME
|| (env.HOME ? path.join(env.HOME, ".local", "share") : undefined);
const sourceAuth = sourceData && path.join(sourceData, "opencode", "auth.json");
if (sourceAuth && existsSync(sourceAuth)) {
const sourceAuthStat = statSync(sourceAuth);
if (!sourceAuthStat.isFile()) return runtime;
const targetDirectory = path.join(runtime.data, "opencode");
const targetAuth = path.join(targetDirectory, "auth.json");
mkdirSync(targetDirectory, { recursive: true, mode: 0o700 });
copyFileSync(sourceAuth, targetAuth);
chmodSync(targetAuth, (sourceAuthStat.mode & 0o600) || 0o600);
}
return runtime;
} catch (error) {
rmSync(root, { recursive: true, force: true });
throw error;
}
}
function isolatedEnv(env, runtime, model, pluginUrl) {
const childEnv = {};
const isolatedPaths = new Set([
"HOME", "USERPROFILE", "APPDATA", "LOCALAPPDATA",
"XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME",
]);
for (const [key, value] of Object.entries(env)) {
const normalized = key.toUpperCase();
if (isolatedPaths.has(normalized)) continue;
if (normalized.startsWith("OPENCODE_")) continue;
childEnv[key] = value;
}
return {
...childEnv,
HOME: runtime.home,
USERPROFILE: runtime.home,
APPDATA: runtime.config,
LOCALAPPDATA: runtime.data,
XDG_CONFIG_HOME: runtime.config,
XDG_DATA_HOME: runtime.data,
XDG_CACHE_HOME: runtime.cache,
XDG_STATE_HOME: runtime.state,
OPENCODE_EVAL_MODEL: model,
OPENCODE_CONFIG_CONTENT: JSON.stringify(buildEvalConfig(pluginUrl)),
};
}
export async function runCase(testCase, options) {
const {
repo,
model,
opencodeVersion,
command = process.platform === "win32" ? "opencode.cmd" : "opencode",
command = defaultOpenCodeCommand(),
commandArgsPrefix = [],
env = process.env,
} = options;
const started = Date.now();
const project = mkdtempSync(path.join(tmpdir(), "behavioral-eval-"));
const runtime = createRuntime(env);
const pluginUrl = pathToFileURL(path.join(repo, ".opencode/plugins/opencode-power-pack.js")).href;
const args = [
...commandArgsPrefix,
@@ -369,13 +475,10 @@ export async function runCase(testCase, options) {
outcome = await new Promise((resolve, reject) => {
const chunks = [];
let timedOut = false;
let killTimer;
let termination;
const child = spawn(command, args, {
cwd: project,
env: {
...env,
OPENCODE_CONFIG_CONTENT: JSON.stringify(buildEvalConfig(pluginUrl)),
},
cwd: runtime.project,
env: isolatedEnv(env, runtime, model, pluginUrl),
detached: process.platform !== "win32",
shell: false,
stdio: ["ignore", "pipe", "pipe"],
@@ -384,18 +487,20 @@ export async function runCase(testCase, options) {
child.stderr.resume();
const timeout = setTimeout(() => {
timedOut = true;
signalChild(child, "SIGTERM");
killTimer = setTimeout(() => signalChild(child, "SIGKILL"), 100);
killTimer.unref();
termination = terminateTimedOutChild(child);
termination.catch(reject);
}, testCase.timeoutMs);
child.once("error", (error) => {
clearTimeout(timeout);
clearTimeout(killTimer);
reject(error);
});
child.once("close", (code) => {
child.once("close", async (code) => {
clearTimeout(timeout);
clearTimeout(killTimer);
try {
if (termination) await termination;
} catch {
return;
}
resolve({ stdout: Buffer.concat(chunks).toString("utf8"), code, timedOut });
});
});
@@ -466,7 +571,7 @@ export async function runCase(testCase, options) {
normalizeResponse(redactResponse(response, testCase)),
);
} finally {
rmSync(project, { recursive: true, force: true });
rmSync(runtime.root, { recursive: true, force: true });
}
}
@@ -506,7 +611,7 @@ export async function runSuite(options) {
repo,
cases,
env = process.env,
command = process.platform === "win32" ? "opencode.cmd" : "opencode",
command = defaultOpenCodeCommand(),
commandArgsPrefix = [],
} = options;
const startedAt = new Date().toISOString();
@@ -553,6 +658,7 @@ async function main() {
const passed = report.cases.filter(({ status }) => status === "pass").length;
console.log(outputPath);
console.log(`${passed}/${report.cases.length} cases passed`);
if (report.status !== "pass") process.exitCode = 1;
}
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main();
+226 -1
View File
@@ -1,13 +1,16 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import {
chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { delimiter, join } from "node:path";
import {
applyMutation,
buildEvalConfig,
caseHash,
defaultOpenCodeCommand,
gradeResponse,
normalizeResponse,
parseJsonEvents,
@@ -117,6 +120,12 @@ test("buildEvalConfig loads only this plugin and denies every model tool", () =>
});
});
test("defaultOpenCodeCommand selects a native executable without a shell", () => {
assert.equal(defaultOpenCodeCommand("win32"), "opencode.exe");
assert.equal(defaultOpenCodeCommand("linux"), "opencode");
assert.equal(defaultOpenCodeCommand("darwin"), "opencode");
});
test("runCase grades text events and redacts the persisted response", async () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-runner-test-"));
const fake = join(temp, "fake-opencode.mjs");
@@ -165,6 +174,123 @@ test("runCase grades text events and redacts the persisted response", async () =
}
});
test("runCase isolates hostile OpenCode controls and copies only auth credentials", async () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-profile-test-"));
const fake = join(temp, "fake-profile.mjs");
const sourceHome = join(temp, "source-home");
const sourceConfig = join(temp, "source-config");
const sourceData = join(temp, "source-data");
const sourceCache = join(temp, "source-cache");
const sourceState = join(temp, "source-state");
const authDir = join(sourceData, "opencode");
const authPath = join(authDir, "auth.json");
const runtimePath = join(temp, "runtime-path.txt");
mkdirSync(authDir, { recursive: true });
writeFileSync(authPath, "auth-sensitive-content", { mode: 0o600 });
mkdirSync(join(sourceConfig, "opencode"), { recursive: true });
writeFileSync(join(sourceConfig, "opencode", "opencode.json"), "hostile-global-config");
writeFileSync(join(sourceData, "opencode", "sessions.json"), "hostile-session-state");
writeFileSync(fake, [
'import assert from "node:assert/strict";',
'import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";',
'import { dirname, join } from "node:path";',
'const runtime = dirname(process.env.HOME);',
'assert.deepEqual([process.env.HOME, process.env.XDG_CONFIG_HOME, process.env.XDG_DATA_HOME, process.env.XDG_CACHE_HOME, process.env.XDG_STATE_HOME], ["home", "config", "data", "cache", "state"].map((name) => join(runtime, name)));',
'assert.equal(process.env.OPENCODE_CONFIG, undefined);',
'assert.equal(process.env.OPENCODE_CONFIG_DIR, undefined);',
'assert.equal(process.env.OPENCODE_SERVER_PASSWORD, undefined);',
'assert.equal(process.env.OPENCODE_EVAL_MODEL, "provider/model");',
'assert.equal(process.env.ANTHROPIC_API_KEY, "provider-sensitive-value");',
'const copiedAuth = join(process.env.XDG_DATA_HOME, "opencode", "auth.json");',
'assert.equal(readFileSync(copiedAuth, "utf8"), "auth-sensitive-content");',
'assert.equal(statSync(copiedAuth).mode & 0o777, 0o600);',
'assert.equal(existsSync(join(process.env.XDG_DATA_HOME, "opencode", "sessions.json")), false);',
'assert.equal(existsSync(join(process.env.XDG_CONFIG_HOME, "opencode", "opencode.json")), false);',
'writeFileSync(process.env.RUNTIME_PATH, runtime);',
'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED\\nSCOPE=PRESERVED"}}));',
].join("\n"));
try {
const result = await runCase(validCase, {
repo: REPO,
model: "provider/model",
opencodeVersion: "1.18.9",
command: process.execPath,
commandArgsPrefix: [fake],
env: {
HOME: sourceHome,
XDG_CONFIG_HOME: sourceConfig,
XDG_DATA_HOME: sourceData,
XDG_CACHE_HOME: sourceCache,
XDG_STATE_HOME: sourceState,
OPENCODE_CONFIG: join(temp, "hostile.json"),
OPENCODE_CONFIG_DIR: join(temp, "hostile-config"),
OPENCODE_CONFIG_CONTENT: "hostile-content",
OPENCODE_SERVER_PASSWORD: "hostile-password",
OPENCODE_EVAL_MODEL: "provider/model",
ANTHROPIC_API_KEY: "provider-sensitive-value",
RUNTIME_PATH: runtimePath,
},
});
assert.equal(result.status, "pass");
assert.equal(existsSync(readFileSync(runtimePath, "utf8")), false);
assert.doesNotMatch(
JSON.stringify(result),
/auth-sensitive-content|provider-sensitive-value|hostile-password/,
);
} finally {
rmSync(temp, { recursive: true, force: true });
}
});
test("runCase confines generated state to its removable runtime root", async () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-state-test-"));
const fake = join(temp, "fake-state.mjs");
const sourceHome = join(temp, "source-home");
const sourceConfig = join(temp, "source-config");
const sourceData = join(temp, "source-data");
const sourceCache = join(temp, "source-cache");
const sourceState = join(temp, "source-state");
const runtimePath = join(temp, "runtime-path.txt");
for (const directory of [sourceHome, sourceConfig, sourceData, sourceCache, sourceState]) {
mkdirSync(directory, { recursive: true });
}
writeFileSync(join(sourceData, "existing-state"), "unchanged");
writeFileSync(fake, [
'import { mkdirSync, writeFileSync } from "node:fs";',
'import { dirname, join } from "node:path";',
'const runtime = dirname(process.env.HOME);',
'for (const root of [process.env.HOME, process.env.XDG_CONFIG_HOME, process.env.XDG_DATA_HOME, process.env.XDG_CACHE_HOME, process.env.XDG_STATE_HOME]) { mkdirSync(root, {recursive:true}); writeFileSync(join(root, "generated-state"), "state"); }',
'writeFileSync(process.env.RUNTIME_PATH, runtime);',
'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED\\nSCOPE=PRESERVED"}}));',
].join("\n"));
try {
const result = await runCase(validCase, {
repo: REPO,
model: "provider/model",
opencodeVersion: "1.18.9",
command: process.execPath,
commandArgsPrefix: [fake],
env: {
HOME: sourceHome,
XDG_CONFIG_HOME: sourceConfig,
XDG_DATA_HOME: sourceData,
XDG_CACHE_HOME: sourceCache,
XDG_STATE_HOME: sourceState,
OPENCODE_EVAL_MODEL: "provider/model",
RUNTIME_PATH: runtimePath,
},
});
assert.equal(result.status, "pass");
assert.equal(readFileSync(join(sourceData, "existing-state"), "utf8"), "unchanged");
for (const directory of [sourceHome, sourceConfig, sourceData, sourceCache, sourceState]) {
assert.equal(existsSync(join(directory, "generated-state")), false);
}
assert.equal(existsSync(readFileSync(runtimePath, "utf8")), false);
} finally {
rmSync(temp, { recursive: true, force: true });
}
});
test("runCase fails closed on timeout without persisting child output", async () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-timeout-test-"));
const fake = join(temp, "fake-timeout.mjs");
@@ -192,6 +318,55 @@ test("runCase fails closed on timeout without persisting child output", async ()
}
});
test("runCase completes process-group escalation before returning on timeout", {
skip: process.platform === "win32",
}, async () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-escalation-test-"));
const fake = join(temp, "fake-escalation.mjs");
const pidPath = join(temp, "descendant-pid.txt");
const readyPath = join(temp, "descendant-ready.txt");
let descendantPid;
writeFileSync(fake, [
'import { spawn } from "node:child_process";',
'import { writeFileSync } from "node:fs";',
'const source = `import { writeFileSync } from "node:fs"; process.on("SIGTERM", () => {}); writeFileSync(process.env.DESCENDANT_READY, "ready"); setInterval(() => {}, 1000);`;',
'const descendant = spawn(process.execPath, ["--input-type=module", "-e", source], {stdio:"ignore"});',
'writeFileSync(process.env.DESCENDANT_PID, String(descendant.pid));',
'setInterval(() => {}, 1000);',
].join("\n"));
try {
const result = await runCase({ ...validCase, timeoutMs: 500 }, {
repo: REPO,
model: "provider/model",
opencodeVersion: "1.18.9",
command: process.execPath,
commandArgsPrefix: [fake],
env: {
...process.env,
DESCENDANT_PID: pidPath,
DESCENDANT_READY: readyPath,
},
});
descendantPid = Number(readFileSync(pidPath, "utf8"));
assert.equal(existsSync(readyPath), true);
assert.equal(result.status, "incomplete");
assert.deepEqual(result.failures, ["incomplete:timeout"]);
assert.throws(
() => process.kill(descendantPid, 0),
(error) => error.code === "ESRCH",
);
} finally {
if (descendantPid) {
try {
process.kill(descendantPid, "SIGKILL");
} catch (error) {
if (error.code !== "ESRCH") throw error;
}
}
rmSync(temp, { recursive: true, force: true });
}
});
test("runCase fails closed on malformed events without persisting raw output", async () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-malformed-test-"));
const fake = join(temp, "fake-malformed.mjs");
@@ -427,6 +602,56 @@ test("CLI run writes the default report and prints no responses", () => {
}
});
test("CLI run exits nonzero after writing failed and incomplete reports", () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-cli-failure-test-"));
const fake = join(temp, "opencode");
const reportDir = join(REPO, ".artifacts/behavioral-evals");
const reportPath = join(reportDir, "latest.json");
const scenarios = [
{
status: "fail",
event: 'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED"}}));',
},
{
status: "incomplete",
event: 'console.log(JSON.stringify({type:"step_finish",part:{type:"step-finish"}}));',
},
];
chmodSync(temp, 0o755);
rmSync(reportDir, { recursive: true, force: true });
try {
for (const scenario of scenarios) {
writeFileSync(fake, [
`#!${process.execPath}`,
'if (process.argv[2] === "--version") { console.log("1.18.9"); process.exit(0); }',
scenario.event,
].join("\n"), { mode: 0o755 });
const result = spawnSync(
process.execPath,
[new URL("../scripts/behavioral-evals.mjs", import.meta.url).pathname, "run"],
{
cwd: REPO,
encoding: "utf8",
env: {
...process.env,
OPENCODE_EVAL_MODEL: "provider/model",
PATH: `${temp}${delimiter}${process.env.PATH}`,
},
},
);
assert.equal(result.status, 1, `${scenario.status}: ${result.stderr}`);
assert.equal(result.stdout, `${reportPath}\n0/12 cases passed\n`);
assert.doesNotMatch(result.stdout, /BOUNDARY=PRESERVED/);
const report = JSON.parse(readFileSync(reportPath, "utf8"));
assert.equal(report.status, "fail");
assert.deepEqual(new Set(report.cases.map(({ status }) => status)), new Set([scenario.status]));
}
} finally {
rmSync(temp, { recursive: true, force: true });
rmSync(reportDir, { recursive: true, force: true });
}
});
test("behavioral corpus covers the approved twelve cases", () => {
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
assert.deepEqual(manifest.cases.map(({ id }) => id).toSorted(), EXPECTED_CASE_IDS);