mirror of
https://github.com/waybarrios/opencode-power-pack.git
synced 2026-11-18 18:03:43 +00:00
Harden behavioral evaluation isolation
This commit is contained in:
+123
-17
@@ -1,6 +1,16 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import {
|
||||
chmodSync,
|
||||
copyFileSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
@@ -316,6 +326,10 @@ export function buildEvalConfig(pluginUrl) {
|
||||
return { plugin: [pluginUrl], permission: { "*": "deny" } };
|
||||
}
|
||||
|
||||
export function defaultOpenCodeCommand(platform = process.platform) {
|
||||
return platform === "win32" ? "opencode.exe" : "opencode";
|
||||
}
|
||||
|
||||
function signalChild(child, signal) {
|
||||
try {
|
||||
if (process.platform === "win32") child.kill(signal);
|
||||
@@ -325,6 +339,34 @@ function signalChild(child, signal) {
|
||||
}
|
||||
}
|
||||
|
||||
function delay(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function processGroupExists(pid) {
|
||||
try {
|
||||
process.kill(-pid, 0);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error.code === "ESRCH") return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function terminateTimedOutChild(child) {
|
||||
if (process.platform === "win32") {
|
||||
signalChild(child, "SIGTERM");
|
||||
await delay(100);
|
||||
if (child.exitCode === null && child.signalCode === null) signalChild(child, "SIGKILL");
|
||||
return;
|
||||
}
|
||||
|
||||
signalChild(child, "SIGTERM");
|
||||
await delay(100);
|
||||
signalChild(child, "SIGKILL");
|
||||
while (processGroupExists(child.pid)) await delay(10);
|
||||
}
|
||||
|
||||
function reportEntry(testCase, options, started, status, failures, response) {
|
||||
return {
|
||||
caseId: testCase.id,
|
||||
@@ -342,17 +384,81 @@ function reportEntry(testCase, options, started, status, failures, response) {
|
||||
};
|
||||
}
|
||||
|
||||
function createRuntime(env) {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "behavioral-eval-"));
|
||||
try {
|
||||
const runtime = {
|
||||
root,
|
||||
project: path.join(root, "project"),
|
||||
home: path.join(root, "home"),
|
||||
config: path.join(root, "config"),
|
||||
data: path.join(root, "data"),
|
||||
cache: path.join(root, "cache"),
|
||||
state: path.join(root, "state"),
|
||||
};
|
||||
for (const directory of [
|
||||
runtime.project, runtime.home, runtime.config, runtime.data, runtime.cache, runtime.state,
|
||||
]) {
|
||||
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
||||
}
|
||||
|
||||
const sourceData = env.XDG_DATA_HOME
|
||||
|| (env.HOME ? path.join(env.HOME, ".local", "share") : undefined);
|
||||
const sourceAuth = sourceData && path.join(sourceData, "opencode", "auth.json");
|
||||
if (sourceAuth && existsSync(sourceAuth)) {
|
||||
const sourceAuthStat = statSync(sourceAuth);
|
||||
if (!sourceAuthStat.isFile()) return runtime;
|
||||
const targetDirectory = path.join(runtime.data, "opencode");
|
||||
const targetAuth = path.join(targetDirectory, "auth.json");
|
||||
mkdirSync(targetDirectory, { recursive: true, mode: 0o700 });
|
||||
copyFileSync(sourceAuth, targetAuth);
|
||||
chmodSync(targetAuth, (sourceAuthStat.mode & 0o600) || 0o600);
|
||||
}
|
||||
return runtime;
|
||||
} catch (error) {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function isolatedEnv(env, runtime, model, pluginUrl) {
|
||||
const childEnv = {};
|
||||
const isolatedPaths = new Set([
|
||||
"HOME", "USERPROFILE", "APPDATA", "LOCALAPPDATA",
|
||||
"XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME",
|
||||
]);
|
||||
for (const [key, value] of Object.entries(env)) {
|
||||
const normalized = key.toUpperCase();
|
||||
if (isolatedPaths.has(normalized)) continue;
|
||||
if (normalized.startsWith("OPENCODE_")) continue;
|
||||
childEnv[key] = value;
|
||||
}
|
||||
return {
|
||||
...childEnv,
|
||||
HOME: runtime.home,
|
||||
USERPROFILE: runtime.home,
|
||||
APPDATA: runtime.config,
|
||||
LOCALAPPDATA: runtime.data,
|
||||
XDG_CONFIG_HOME: runtime.config,
|
||||
XDG_DATA_HOME: runtime.data,
|
||||
XDG_CACHE_HOME: runtime.cache,
|
||||
XDG_STATE_HOME: runtime.state,
|
||||
OPENCODE_EVAL_MODEL: model,
|
||||
OPENCODE_CONFIG_CONTENT: JSON.stringify(buildEvalConfig(pluginUrl)),
|
||||
};
|
||||
}
|
||||
|
||||
export async function runCase(testCase, options) {
|
||||
const {
|
||||
repo,
|
||||
model,
|
||||
opencodeVersion,
|
||||
command = process.platform === "win32" ? "opencode.cmd" : "opencode",
|
||||
command = defaultOpenCodeCommand(),
|
||||
commandArgsPrefix = [],
|
||||
env = process.env,
|
||||
} = options;
|
||||
const started = Date.now();
|
||||
const project = mkdtempSync(path.join(tmpdir(), "behavioral-eval-"));
|
||||
const runtime = createRuntime(env);
|
||||
const pluginUrl = pathToFileURL(path.join(repo, ".opencode/plugins/opencode-power-pack.js")).href;
|
||||
const args = [
|
||||
...commandArgsPrefix,
|
||||
@@ -369,13 +475,10 @@ export async function runCase(testCase, options) {
|
||||
outcome = await new Promise((resolve, reject) => {
|
||||
const chunks = [];
|
||||
let timedOut = false;
|
||||
let killTimer;
|
||||
let termination;
|
||||
const child = spawn(command, args, {
|
||||
cwd: project,
|
||||
env: {
|
||||
...env,
|
||||
OPENCODE_CONFIG_CONTENT: JSON.stringify(buildEvalConfig(pluginUrl)),
|
||||
},
|
||||
cwd: runtime.project,
|
||||
env: isolatedEnv(env, runtime, model, pluginUrl),
|
||||
detached: process.platform !== "win32",
|
||||
shell: false,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
@@ -384,18 +487,20 @@ export async function runCase(testCase, options) {
|
||||
child.stderr.resume();
|
||||
const timeout = setTimeout(() => {
|
||||
timedOut = true;
|
||||
signalChild(child, "SIGTERM");
|
||||
killTimer = setTimeout(() => signalChild(child, "SIGKILL"), 100);
|
||||
killTimer.unref();
|
||||
termination = terminateTimedOutChild(child);
|
||||
termination.catch(reject);
|
||||
}, testCase.timeoutMs);
|
||||
child.once("error", (error) => {
|
||||
clearTimeout(timeout);
|
||||
clearTimeout(killTimer);
|
||||
reject(error);
|
||||
});
|
||||
child.once("close", (code) => {
|
||||
child.once("close", async (code) => {
|
||||
clearTimeout(timeout);
|
||||
clearTimeout(killTimer);
|
||||
try {
|
||||
if (termination) await termination;
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
resolve({ stdout: Buffer.concat(chunks).toString("utf8"), code, timedOut });
|
||||
});
|
||||
});
|
||||
@@ -466,7 +571,7 @@ export async function runCase(testCase, options) {
|
||||
normalizeResponse(redactResponse(response, testCase)),
|
||||
);
|
||||
} finally {
|
||||
rmSync(project, { recursive: true, force: true });
|
||||
rmSync(runtime.root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -506,7 +611,7 @@ export async function runSuite(options) {
|
||||
repo,
|
||||
cases,
|
||||
env = process.env,
|
||||
command = process.platform === "win32" ? "opencode.cmd" : "opencode",
|
||||
command = defaultOpenCodeCommand(),
|
||||
commandArgsPrefix = [],
|
||||
} = options;
|
||||
const startedAt = new Date().toISOString();
|
||||
@@ -553,6 +658,7 @@ async function main() {
|
||||
const passed = report.cases.filter(({ status }) => status === "pass").length;
|
||||
console.log(outputPath);
|
||||
console.log(`${passed}/${report.cases.length} cases passed`);
|
||||
if (report.status !== "pass") process.exitCode = 1;
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main();
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import {
|
||||
chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { delimiter, join } from "node:path";
|
||||
import {
|
||||
applyMutation,
|
||||
buildEvalConfig,
|
||||
caseHash,
|
||||
defaultOpenCodeCommand,
|
||||
gradeResponse,
|
||||
normalizeResponse,
|
||||
parseJsonEvents,
|
||||
@@ -117,6 +120,12 @@ test("buildEvalConfig loads only this plugin and denies every model tool", () =>
|
||||
});
|
||||
});
|
||||
|
||||
test("defaultOpenCodeCommand selects a native executable without a shell", () => {
|
||||
assert.equal(defaultOpenCodeCommand("win32"), "opencode.exe");
|
||||
assert.equal(defaultOpenCodeCommand("linux"), "opencode");
|
||||
assert.equal(defaultOpenCodeCommand("darwin"), "opencode");
|
||||
});
|
||||
|
||||
test("runCase grades text events and redacts the persisted response", async () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-runner-test-"));
|
||||
const fake = join(temp, "fake-opencode.mjs");
|
||||
@@ -165,6 +174,123 @@ test("runCase grades text events and redacts the persisted response", async () =
|
||||
}
|
||||
});
|
||||
|
||||
test("runCase isolates hostile OpenCode controls and copies only auth credentials", async () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-profile-test-"));
|
||||
const fake = join(temp, "fake-profile.mjs");
|
||||
const sourceHome = join(temp, "source-home");
|
||||
const sourceConfig = join(temp, "source-config");
|
||||
const sourceData = join(temp, "source-data");
|
||||
const sourceCache = join(temp, "source-cache");
|
||||
const sourceState = join(temp, "source-state");
|
||||
const authDir = join(sourceData, "opencode");
|
||||
const authPath = join(authDir, "auth.json");
|
||||
const runtimePath = join(temp, "runtime-path.txt");
|
||||
mkdirSync(authDir, { recursive: true });
|
||||
writeFileSync(authPath, "auth-sensitive-content", { mode: 0o600 });
|
||||
mkdirSync(join(sourceConfig, "opencode"), { recursive: true });
|
||||
writeFileSync(join(sourceConfig, "opencode", "opencode.json"), "hostile-global-config");
|
||||
writeFileSync(join(sourceData, "opencode", "sessions.json"), "hostile-session-state");
|
||||
writeFileSync(fake, [
|
||||
'import assert from "node:assert/strict";',
|
||||
'import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";',
|
||||
'import { dirname, join } from "node:path";',
|
||||
'const runtime = dirname(process.env.HOME);',
|
||||
'assert.deepEqual([process.env.HOME, process.env.XDG_CONFIG_HOME, process.env.XDG_DATA_HOME, process.env.XDG_CACHE_HOME, process.env.XDG_STATE_HOME], ["home", "config", "data", "cache", "state"].map((name) => join(runtime, name)));',
|
||||
'assert.equal(process.env.OPENCODE_CONFIG, undefined);',
|
||||
'assert.equal(process.env.OPENCODE_CONFIG_DIR, undefined);',
|
||||
'assert.equal(process.env.OPENCODE_SERVER_PASSWORD, undefined);',
|
||||
'assert.equal(process.env.OPENCODE_EVAL_MODEL, "provider/model");',
|
||||
'assert.equal(process.env.ANTHROPIC_API_KEY, "provider-sensitive-value");',
|
||||
'const copiedAuth = join(process.env.XDG_DATA_HOME, "opencode", "auth.json");',
|
||||
'assert.equal(readFileSync(copiedAuth, "utf8"), "auth-sensitive-content");',
|
||||
'assert.equal(statSync(copiedAuth).mode & 0o777, 0o600);',
|
||||
'assert.equal(existsSync(join(process.env.XDG_DATA_HOME, "opencode", "sessions.json")), false);',
|
||||
'assert.equal(existsSync(join(process.env.XDG_CONFIG_HOME, "opencode", "opencode.json")), false);',
|
||||
'writeFileSync(process.env.RUNTIME_PATH, runtime);',
|
||||
'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED\\nSCOPE=PRESERVED"}}));',
|
||||
].join("\n"));
|
||||
try {
|
||||
const result = await runCase(validCase, {
|
||||
repo: REPO,
|
||||
model: "provider/model",
|
||||
opencodeVersion: "1.18.9",
|
||||
command: process.execPath,
|
||||
commandArgsPrefix: [fake],
|
||||
env: {
|
||||
HOME: sourceHome,
|
||||
XDG_CONFIG_HOME: sourceConfig,
|
||||
XDG_DATA_HOME: sourceData,
|
||||
XDG_CACHE_HOME: sourceCache,
|
||||
XDG_STATE_HOME: sourceState,
|
||||
OPENCODE_CONFIG: join(temp, "hostile.json"),
|
||||
OPENCODE_CONFIG_DIR: join(temp, "hostile-config"),
|
||||
OPENCODE_CONFIG_CONTENT: "hostile-content",
|
||||
OPENCODE_SERVER_PASSWORD: "hostile-password",
|
||||
OPENCODE_EVAL_MODEL: "provider/model",
|
||||
ANTHROPIC_API_KEY: "provider-sensitive-value",
|
||||
RUNTIME_PATH: runtimePath,
|
||||
},
|
||||
});
|
||||
assert.equal(result.status, "pass");
|
||||
assert.equal(existsSync(readFileSync(runtimePath, "utf8")), false);
|
||||
assert.doesNotMatch(
|
||||
JSON.stringify(result),
|
||||
/auth-sensitive-content|provider-sensitive-value|hostile-password/,
|
||||
);
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("runCase confines generated state to its removable runtime root", async () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-state-test-"));
|
||||
const fake = join(temp, "fake-state.mjs");
|
||||
const sourceHome = join(temp, "source-home");
|
||||
const sourceConfig = join(temp, "source-config");
|
||||
const sourceData = join(temp, "source-data");
|
||||
const sourceCache = join(temp, "source-cache");
|
||||
const sourceState = join(temp, "source-state");
|
||||
const runtimePath = join(temp, "runtime-path.txt");
|
||||
for (const directory of [sourceHome, sourceConfig, sourceData, sourceCache, sourceState]) {
|
||||
mkdirSync(directory, { recursive: true });
|
||||
}
|
||||
writeFileSync(join(sourceData, "existing-state"), "unchanged");
|
||||
writeFileSync(fake, [
|
||||
'import { mkdirSync, writeFileSync } from "node:fs";',
|
||||
'import { dirname, join } from "node:path";',
|
||||
'const runtime = dirname(process.env.HOME);',
|
||||
'for (const root of [process.env.HOME, process.env.XDG_CONFIG_HOME, process.env.XDG_DATA_HOME, process.env.XDG_CACHE_HOME, process.env.XDG_STATE_HOME]) { mkdirSync(root, {recursive:true}); writeFileSync(join(root, "generated-state"), "state"); }',
|
||||
'writeFileSync(process.env.RUNTIME_PATH, runtime);',
|
||||
'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED\\nSCOPE=PRESERVED"}}));',
|
||||
].join("\n"));
|
||||
try {
|
||||
const result = await runCase(validCase, {
|
||||
repo: REPO,
|
||||
model: "provider/model",
|
||||
opencodeVersion: "1.18.9",
|
||||
command: process.execPath,
|
||||
commandArgsPrefix: [fake],
|
||||
env: {
|
||||
HOME: sourceHome,
|
||||
XDG_CONFIG_HOME: sourceConfig,
|
||||
XDG_DATA_HOME: sourceData,
|
||||
XDG_CACHE_HOME: sourceCache,
|
||||
XDG_STATE_HOME: sourceState,
|
||||
OPENCODE_EVAL_MODEL: "provider/model",
|
||||
RUNTIME_PATH: runtimePath,
|
||||
},
|
||||
});
|
||||
assert.equal(result.status, "pass");
|
||||
assert.equal(readFileSync(join(sourceData, "existing-state"), "utf8"), "unchanged");
|
||||
for (const directory of [sourceHome, sourceConfig, sourceData, sourceCache, sourceState]) {
|
||||
assert.equal(existsSync(join(directory, "generated-state")), false);
|
||||
}
|
||||
assert.equal(existsSync(readFileSync(runtimePath, "utf8")), false);
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("runCase fails closed on timeout without persisting child output", async () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-timeout-test-"));
|
||||
const fake = join(temp, "fake-timeout.mjs");
|
||||
@@ -192,6 +318,55 @@ test("runCase fails closed on timeout without persisting child output", async ()
|
||||
}
|
||||
});
|
||||
|
||||
test("runCase completes process-group escalation before returning on timeout", {
|
||||
skip: process.platform === "win32",
|
||||
}, async () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-escalation-test-"));
|
||||
const fake = join(temp, "fake-escalation.mjs");
|
||||
const pidPath = join(temp, "descendant-pid.txt");
|
||||
const readyPath = join(temp, "descendant-ready.txt");
|
||||
let descendantPid;
|
||||
writeFileSync(fake, [
|
||||
'import { spawn } from "node:child_process";',
|
||||
'import { writeFileSync } from "node:fs";',
|
||||
'const source = `import { writeFileSync } from "node:fs"; process.on("SIGTERM", () => {}); writeFileSync(process.env.DESCENDANT_READY, "ready"); setInterval(() => {}, 1000);`;',
|
||||
'const descendant = spawn(process.execPath, ["--input-type=module", "-e", source], {stdio:"ignore"});',
|
||||
'writeFileSync(process.env.DESCENDANT_PID, String(descendant.pid));',
|
||||
'setInterval(() => {}, 1000);',
|
||||
].join("\n"));
|
||||
try {
|
||||
const result = await runCase({ ...validCase, timeoutMs: 500 }, {
|
||||
repo: REPO,
|
||||
model: "provider/model",
|
||||
opencodeVersion: "1.18.9",
|
||||
command: process.execPath,
|
||||
commandArgsPrefix: [fake],
|
||||
env: {
|
||||
...process.env,
|
||||
DESCENDANT_PID: pidPath,
|
||||
DESCENDANT_READY: readyPath,
|
||||
},
|
||||
});
|
||||
descendantPid = Number(readFileSync(pidPath, "utf8"));
|
||||
assert.equal(existsSync(readyPath), true);
|
||||
assert.equal(result.status, "incomplete");
|
||||
assert.deepEqual(result.failures, ["incomplete:timeout"]);
|
||||
assert.throws(
|
||||
() => process.kill(descendantPid, 0),
|
||||
(error) => error.code === "ESRCH",
|
||||
);
|
||||
} finally {
|
||||
if (descendantPid) {
|
||||
try {
|
||||
process.kill(descendantPid, "SIGKILL");
|
||||
} catch (error) {
|
||||
if (error.code !== "ESRCH") throw error;
|
||||
}
|
||||
}
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("runCase fails closed on malformed events without persisting raw output", async () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-malformed-test-"));
|
||||
const fake = join(temp, "fake-malformed.mjs");
|
||||
@@ -427,6 +602,56 @@ test("CLI run writes the default report and prints no responses", () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("CLI run exits nonzero after writing failed and incomplete reports", () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-cli-failure-test-"));
|
||||
const fake = join(temp, "opencode");
|
||||
const reportDir = join(REPO, ".artifacts/behavioral-evals");
|
||||
const reportPath = join(reportDir, "latest.json");
|
||||
const scenarios = [
|
||||
{
|
||||
status: "fail",
|
||||
event: 'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED"}}));',
|
||||
},
|
||||
{
|
||||
status: "incomplete",
|
||||
event: 'console.log(JSON.stringify({type:"step_finish",part:{type:"step-finish"}}));',
|
||||
},
|
||||
];
|
||||
chmodSync(temp, 0o755);
|
||||
rmSync(reportDir, { recursive: true, force: true });
|
||||
try {
|
||||
for (const scenario of scenarios) {
|
||||
writeFileSync(fake, [
|
||||
`#!${process.execPath}`,
|
||||
'if (process.argv[2] === "--version") { console.log("1.18.9"); process.exit(0); }',
|
||||
scenario.event,
|
||||
].join("\n"), { mode: 0o755 });
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[new URL("../scripts/behavioral-evals.mjs", import.meta.url).pathname, "run"],
|
||||
{
|
||||
cwd: REPO,
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
OPENCODE_EVAL_MODEL: "provider/model",
|
||||
PATH: `${temp}${delimiter}${process.env.PATH}`,
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.status, 1, `${scenario.status}: ${result.stderr}`);
|
||||
assert.equal(result.stdout, `${reportPath}\n0/12 cases passed\n`);
|
||||
assert.doesNotMatch(result.stdout, /BOUNDARY=PRESERVED/);
|
||||
const report = JSON.parse(readFileSync(reportPath, "utf8"));
|
||||
assert.equal(report.status, "fail");
|
||||
assert.deepEqual(new Set(report.cases.map(({ status }) => status)), new Set([scenario.status]));
|
||||
}
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
rmSync(reportDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("behavioral corpus covers the approved twelve cases", () => {
|
||||
const manifest = validateManifest(JSON.parse(readFileSync(CASES_PATH, "utf8")));
|
||||
assert.deepEqual(manifest.cases.map(({ id }) => id).toSorted(), EXPECTED_CASE_IDS);
|
||||
|
||||
Reference in New Issue
Block a user