From cd5c150b087e984f57434149c35d6dca1216e357 Mon Sep 17 00:00:00 2001 From: Wayner Barrios Date: Fri, 31 Jul 2026 16:50:05 -0400 Subject: [PATCH] Harden behavioral evaluation isolation --- scripts/behavioral-evals.mjs | 140 +++++++++++++++++--- tests/behavioral-evals.test.mjs | 227 +++++++++++++++++++++++++++++++- 2 files changed, 349 insertions(+), 18 deletions(-) diff --git a/scripts/behavioral-evals.mjs b/scripts/behavioral-evals.mjs index 296c456..3d06419 100644 --- a/scripts/behavioral-evals.mjs +++ b/scripts/behavioral-evals.mjs @@ -1,6 +1,16 @@ import { createHash } from "node:crypto"; import { spawn } from "node:child_process"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; @@ -316,6 +326,10 @@ export function buildEvalConfig(pluginUrl) { return { plugin: [pluginUrl], permission: { "*": "deny" } }; } +export function defaultOpenCodeCommand(platform = process.platform) { + return platform === "win32" ? "opencode.exe" : "opencode"; +} + function signalChild(child, signal) { try { if (process.platform === "win32") child.kill(signal); @@ -325,6 +339,34 @@ function signalChild(child, signal) { } } +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function processGroupExists(pid) { + try { + process.kill(-pid, 0); + return true; + } catch (error) { + if (error.code === "ESRCH") return false; + throw error; + } +} + +async function terminateTimedOutChild(child) { + if (process.platform === "win32") { + signalChild(child, "SIGTERM"); + await delay(100); + if (child.exitCode === null && child.signalCode === null) signalChild(child, "SIGKILL"); + return; + } + + signalChild(child, "SIGTERM"); + await delay(100); + signalChild(child, "SIGKILL"); + while (processGroupExists(child.pid)) await delay(10); +} + function reportEntry(testCase, options, started, status, failures, response) { return { caseId: testCase.id, @@ -342,17 +384,81 @@ function reportEntry(testCase, options, started, status, failures, response) { }; } +function createRuntime(env) { + const root = mkdtempSync(path.join(tmpdir(), "behavioral-eval-")); + try { + const runtime = { + root, + project: path.join(root, "project"), + home: path.join(root, "home"), + config: path.join(root, "config"), + data: path.join(root, "data"), + cache: path.join(root, "cache"), + state: path.join(root, "state"), + }; + for (const directory of [ + runtime.project, runtime.home, runtime.config, runtime.data, runtime.cache, runtime.state, + ]) { + mkdirSync(directory, { recursive: true, mode: 0o700 }); + } + + const sourceData = env.XDG_DATA_HOME + || (env.HOME ? path.join(env.HOME, ".local", "share") : undefined); + const sourceAuth = sourceData && path.join(sourceData, "opencode", "auth.json"); + if (sourceAuth && existsSync(sourceAuth)) { + const sourceAuthStat = statSync(sourceAuth); + if (!sourceAuthStat.isFile()) return runtime; + const targetDirectory = path.join(runtime.data, "opencode"); + const targetAuth = path.join(targetDirectory, "auth.json"); + mkdirSync(targetDirectory, { recursive: true, mode: 0o700 }); + copyFileSync(sourceAuth, targetAuth); + chmodSync(targetAuth, (sourceAuthStat.mode & 0o600) || 0o600); + } + return runtime; + } catch (error) { + rmSync(root, { recursive: true, force: true }); + throw error; + } +} + +function isolatedEnv(env, runtime, model, pluginUrl) { + const childEnv = {}; + const isolatedPaths = new Set([ + "HOME", "USERPROFILE", "APPDATA", "LOCALAPPDATA", + "XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME", + ]); + for (const [key, value] of Object.entries(env)) { + const normalized = key.toUpperCase(); + if (isolatedPaths.has(normalized)) continue; + if (normalized.startsWith("OPENCODE_")) continue; + childEnv[key] = value; + } + return { + ...childEnv, + HOME: runtime.home, + USERPROFILE: runtime.home, + APPDATA: runtime.config, + LOCALAPPDATA: runtime.data, + XDG_CONFIG_HOME: runtime.config, + XDG_DATA_HOME: runtime.data, + XDG_CACHE_HOME: runtime.cache, + XDG_STATE_HOME: runtime.state, + OPENCODE_EVAL_MODEL: model, + OPENCODE_CONFIG_CONTENT: JSON.stringify(buildEvalConfig(pluginUrl)), + }; +} + export async function runCase(testCase, options) { const { repo, model, opencodeVersion, - command = process.platform === "win32" ? "opencode.cmd" : "opencode", + command = defaultOpenCodeCommand(), commandArgsPrefix = [], env = process.env, } = options; const started = Date.now(); - const project = mkdtempSync(path.join(tmpdir(), "behavioral-eval-")); + const runtime = createRuntime(env); const pluginUrl = pathToFileURL(path.join(repo, ".opencode/plugins/opencode-power-pack.js")).href; const args = [ ...commandArgsPrefix, @@ -369,13 +475,10 @@ export async function runCase(testCase, options) { outcome = await new Promise((resolve, reject) => { const chunks = []; let timedOut = false; - let killTimer; + let termination; const child = spawn(command, args, { - cwd: project, - env: { - ...env, - OPENCODE_CONFIG_CONTENT: JSON.stringify(buildEvalConfig(pluginUrl)), - }, + cwd: runtime.project, + env: isolatedEnv(env, runtime, model, pluginUrl), detached: process.platform !== "win32", shell: false, stdio: ["ignore", "pipe", "pipe"], @@ -384,18 +487,20 @@ export async function runCase(testCase, options) { child.stderr.resume(); const timeout = setTimeout(() => { timedOut = true; - signalChild(child, "SIGTERM"); - killTimer = setTimeout(() => signalChild(child, "SIGKILL"), 100); - killTimer.unref(); + termination = terminateTimedOutChild(child); + termination.catch(reject); }, testCase.timeoutMs); child.once("error", (error) => { clearTimeout(timeout); - clearTimeout(killTimer); reject(error); }); - child.once("close", (code) => { + child.once("close", async (code) => { clearTimeout(timeout); - clearTimeout(killTimer); + try { + if (termination) await termination; + } catch { + return; + } resolve({ stdout: Buffer.concat(chunks).toString("utf8"), code, timedOut }); }); }); @@ -466,7 +571,7 @@ export async function runCase(testCase, options) { normalizeResponse(redactResponse(response, testCase)), ); } finally { - rmSync(project, { recursive: true, force: true }); + rmSync(runtime.root, { recursive: true, force: true }); } } @@ -506,7 +611,7 @@ export async function runSuite(options) { repo, cases, env = process.env, - command = process.platform === "win32" ? "opencode.cmd" : "opencode", + command = defaultOpenCodeCommand(), commandArgsPrefix = [], } = options; const startedAt = new Date().toISOString(); @@ -553,6 +658,7 @@ async function main() { const passed = report.cases.filter(({ status }) => status === "pass").length; console.log(outputPath); console.log(`${passed}/${report.cases.length} cases passed`); + if (report.status !== "pass") process.exitCode = 1; } if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main(); diff --git a/tests/behavioral-evals.test.mjs b/tests/behavioral-evals.test.mjs index 6387c27..faad40f 100644 --- a/tests/behavioral-evals.test.mjs +++ b/tests/behavioral-evals.test.mjs @@ -1,13 +1,16 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; -import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { delimiter, join } from "node:path"; import { applyMutation, buildEvalConfig, caseHash, + defaultOpenCodeCommand, gradeResponse, normalizeResponse, parseJsonEvents, @@ -117,6 +120,12 @@ test("buildEvalConfig loads only this plugin and denies every model tool", () => }); }); +test("defaultOpenCodeCommand selects a native executable without a shell", () => { + assert.equal(defaultOpenCodeCommand("win32"), "opencode.exe"); + assert.equal(defaultOpenCodeCommand("linux"), "opencode"); + assert.equal(defaultOpenCodeCommand("darwin"), "opencode"); +}); + test("runCase grades text events and redacts the persisted response", async () => { const temp = mkdtempSync(join(tmpdir(), "behavioral-runner-test-")); const fake = join(temp, "fake-opencode.mjs"); @@ -165,6 +174,123 @@ test("runCase grades text events and redacts the persisted response", async () = } }); +test("runCase isolates hostile OpenCode controls and copies only auth credentials", async () => { + const temp = mkdtempSync(join(tmpdir(), "behavioral-profile-test-")); + const fake = join(temp, "fake-profile.mjs"); + const sourceHome = join(temp, "source-home"); + const sourceConfig = join(temp, "source-config"); + const sourceData = join(temp, "source-data"); + const sourceCache = join(temp, "source-cache"); + const sourceState = join(temp, "source-state"); + const authDir = join(sourceData, "opencode"); + const authPath = join(authDir, "auth.json"); + const runtimePath = join(temp, "runtime-path.txt"); + mkdirSync(authDir, { recursive: true }); + writeFileSync(authPath, "auth-sensitive-content", { mode: 0o600 }); + mkdirSync(join(sourceConfig, "opencode"), { recursive: true }); + writeFileSync(join(sourceConfig, "opencode", "opencode.json"), "hostile-global-config"); + writeFileSync(join(sourceData, "opencode", "sessions.json"), "hostile-session-state"); + writeFileSync(fake, [ + 'import assert from "node:assert/strict";', + 'import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";', + 'import { dirname, join } from "node:path";', + 'const runtime = dirname(process.env.HOME);', + 'assert.deepEqual([process.env.HOME, process.env.XDG_CONFIG_HOME, process.env.XDG_DATA_HOME, process.env.XDG_CACHE_HOME, process.env.XDG_STATE_HOME], ["home", "config", "data", "cache", "state"].map((name) => join(runtime, name)));', + 'assert.equal(process.env.OPENCODE_CONFIG, undefined);', + 'assert.equal(process.env.OPENCODE_CONFIG_DIR, undefined);', + 'assert.equal(process.env.OPENCODE_SERVER_PASSWORD, undefined);', + 'assert.equal(process.env.OPENCODE_EVAL_MODEL, "provider/model");', + 'assert.equal(process.env.ANTHROPIC_API_KEY, "provider-sensitive-value");', + 'const copiedAuth = join(process.env.XDG_DATA_HOME, "opencode", "auth.json");', + 'assert.equal(readFileSync(copiedAuth, "utf8"), "auth-sensitive-content");', + 'assert.equal(statSync(copiedAuth).mode & 0o777, 0o600);', + 'assert.equal(existsSync(join(process.env.XDG_DATA_HOME, "opencode", "sessions.json")), false);', + 'assert.equal(existsSync(join(process.env.XDG_CONFIG_HOME, "opencode", "opencode.json")), false);', + 'writeFileSync(process.env.RUNTIME_PATH, runtime);', + 'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED\\nSCOPE=PRESERVED"}}));', + ].join("\n")); + try { + const result = await runCase(validCase, { + repo: REPO, + model: "provider/model", + opencodeVersion: "1.18.9", + command: process.execPath, + commandArgsPrefix: [fake], + env: { + HOME: sourceHome, + XDG_CONFIG_HOME: sourceConfig, + XDG_DATA_HOME: sourceData, + XDG_CACHE_HOME: sourceCache, + XDG_STATE_HOME: sourceState, + OPENCODE_CONFIG: join(temp, "hostile.json"), + OPENCODE_CONFIG_DIR: join(temp, "hostile-config"), + OPENCODE_CONFIG_CONTENT: "hostile-content", + OPENCODE_SERVER_PASSWORD: "hostile-password", + OPENCODE_EVAL_MODEL: "provider/model", + ANTHROPIC_API_KEY: "provider-sensitive-value", + RUNTIME_PATH: runtimePath, + }, + }); + assert.equal(result.status, "pass"); + assert.equal(existsSync(readFileSync(runtimePath, "utf8")), false); + assert.doesNotMatch( + JSON.stringify(result), + /auth-sensitive-content|provider-sensitive-value|hostile-password/, + ); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); + +test("runCase confines generated state to its removable runtime root", async () => { + const temp = mkdtempSync(join(tmpdir(), "behavioral-state-test-")); + const fake = join(temp, "fake-state.mjs"); + const sourceHome = join(temp, "source-home"); + const sourceConfig = join(temp, "source-config"); + const sourceData = join(temp, "source-data"); + const sourceCache = join(temp, "source-cache"); + const sourceState = join(temp, "source-state"); + const runtimePath = join(temp, "runtime-path.txt"); + for (const directory of [sourceHome, sourceConfig, sourceData, sourceCache, sourceState]) { + mkdirSync(directory, { recursive: true }); + } + writeFileSync(join(sourceData, "existing-state"), "unchanged"); + writeFileSync(fake, [ + 'import { mkdirSync, writeFileSync } from "node:fs";', + 'import { dirname, join } from "node:path";', + 'const runtime = dirname(process.env.HOME);', + 'for (const root of [process.env.HOME, process.env.XDG_CONFIG_HOME, process.env.XDG_DATA_HOME, process.env.XDG_CACHE_HOME, process.env.XDG_STATE_HOME]) { mkdirSync(root, {recursive:true}); writeFileSync(join(root, "generated-state"), "state"); }', + 'writeFileSync(process.env.RUNTIME_PATH, runtime);', + 'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED\\nSCOPE=PRESERVED"}}));', + ].join("\n")); + try { + const result = await runCase(validCase, { + repo: REPO, + model: "provider/model", + opencodeVersion: "1.18.9", + command: process.execPath, + commandArgsPrefix: [fake], + env: { + HOME: sourceHome, + XDG_CONFIG_HOME: sourceConfig, + XDG_DATA_HOME: sourceData, + XDG_CACHE_HOME: sourceCache, + XDG_STATE_HOME: sourceState, + OPENCODE_EVAL_MODEL: "provider/model", + RUNTIME_PATH: runtimePath, + }, + }); + assert.equal(result.status, "pass"); + assert.equal(readFileSync(join(sourceData, "existing-state"), "utf8"), "unchanged"); + for (const directory of [sourceHome, sourceConfig, sourceData, sourceCache, sourceState]) { + assert.equal(existsSync(join(directory, "generated-state")), false); + } + assert.equal(existsSync(readFileSync(runtimePath, "utf8")), false); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); + test("runCase fails closed on timeout without persisting child output", async () => { const temp = mkdtempSync(join(tmpdir(), "behavioral-timeout-test-")); const fake = join(temp, "fake-timeout.mjs"); @@ -192,6 +318,55 @@ test("runCase fails closed on timeout without persisting child output", async () } }); +test("runCase completes process-group escalation before returning on timeout", { + skip: process.platform === "win32", +}, async () => { + const temp = mkdtempSync(join(tmpdir(), "behavioral-escalation-test-")); + const fake = join(temp, "fake-escalation.mjs"); + const pidPath = join(temp, "descendant-pid.txt"); + const readyPath = join(temp, "descendant-ready.txt"); + let descendantPid; + writeFileSync(fake, [ + 'import { spawn } from "node:child_process";', + 'import { writeFileSync } from "node:fs";', + 'const source = `import { writeFileSync } from "node:fs"; process.on("SIGTERM", () => {}); writeFileSync(process.env.DESCENDANT_READY, "ready"); setInterval(() => {}, 1000);`;', + 'const descendant = spawn(process.execPath, ["--input-type=module", "-e", source], {stdio:"ignore"});', + 'writeFileSync(process.env.DESCENDANT_PID, String(descendant.pid));', + 'setInterval(() => {}, 1000);', + ].join("\n")); + try { + const result = await runCase({ ...validCase, timeoutMs: 500 }, { + repo: REPO, + model: "provider/model", + opencodeVersion: "1.18.9", + command: process.execPath, + commandArgsPrefix: [fake], + env: { + ...process.env, + DESCENDANT_PID: pidPath, + DESCENDANT_READY: readyPath, + }, + }); + descendantPid = Number(readFileSync(pidPath, "utf8")); + assert.equal(existsSync(readyPath), true); + assert.equal(result.status, "incomplete"); + assert.deepEqual(result.failures, ["incomplete:timeout"]); + assert.throws( + () => process.kill(descendantPid, 0), + (error) => error.code === "ESRCH", + ); + } finally { + if (descendantPid) { + try { + process.kill(descendantPid, "SIGKILL"); + } catch (error) { + if (error.code !== "ESRCH") throw error; + } + } + rmSync(temp, { recursive: true, force: true }); + } +}); + test("runCase fails closed on malformed events without persisting raw output", async () => { const temp = mkdtempSync(join(tmpdir(), "behavioral-malformed-test-")); const fake = join(temp, "fake-malformed.mjs"); @@ -427,6 +602,56 @@ test("CLI run writes the default report and prints no responses", () => { } }); +test("CLI run exits nonzero after writing failed and incomplete reports", () => { + const temp = mkdtempSync(join(tmpdir(), "behavioral-cli-failure-test-")); + const fake = join(temp, "opencode"); + const reportDir = join(REPO, ".artifacts/behavioral-evals"); + const reportPath = join(reportDir, "latest.json"); + const scenarios = [ + { + status: "fail", + event: 'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED"}}));', + }, + { + status: "incomplete", + event: 'console.log(JSON.stringify({type:"step_finish",part:{type:"step-finish"}}));', + }, + ]; + chmodSync(temp, 0o755); + rmSync(reportDir, { recursive: true, force: true }); + try { + for (const scenario of scenarios) { + writeFileSync(fake, [ + `#!${process.execPath}`, + 'if (process.argv[2] === "--version") { console.log("1.18.9"); process.exit(0); }', + scenario.event, + ].join("\n"), { mode: 0o755 }); + const result = spawnSync( + process.execPath, + [new URL("../scripts/behavioral-evals.mjs", import.meta.url).pathname, "run"], + { + cwd: REPO, + encoding: "utf8", + env: { + ...process.env, + OPENCODE_EVAL_MODEL: "provider/model", + PATH: `${temp}${delimiter}${process.env.PATH}`, + }, + }, + ); + assert.equal(result.status, 1, `${scenario.status}: ${result.stderr}`); + assert.equal(result.stdout, `${reportPath}\n0/12 cases passed\n`); + assert.doesNotMatch(result.stdout, /BOUNDARY=PRESERVED/); + const report = JSON.parse(readFileSync(reportPath, "utf8")); + assert.equal(report.status, "fail"); + assert.deepEqual(new Set(report.cases.map(({ status }) => status)), new Set([scenario.status])); + } + } finally { + rmSync(temp, { recursive: true, force: true }); + rmSync(reportDir, { recursive: true, force: true }); + } +}); + test("behavioral corpus covers the approved twelve cases", () => { const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8"))); assert.deepEqual(manifest.cases.map(({ id }) => id).toSorted(), EXPECTED_CASE_IDS);