Files
opencode-power-pack/scripts/opencode-smoke.mjs
T
Yefta Sutanto 3b04f7381d feat(opencode): port the plugin and behavioral evals to the OpenCode 2 plugin API
The OpenCode 2 plugin API replaces the V1 config hook, so the shipped
plugin no longer loads on 2.x. Port every OpenCode surface in one change:

- Register all bundled skills and the three specialist subagents through
  ctx.skill.transform and ctx.agent.transform, with ordered V2 permission
  rulesets and frontmatter-stripped skill content.
- Move the shared definitions helper to .opencode/lib so the plugin
  loader does not discover it as a plugin entrypoint.
- Rewrite the behavioral eval runner around opencode serve, the V2
  session API, and skill attachments; keep the grading, redaction, and
  snapshot evidence model unchanged.
- Port the smoke test to @opencode/cli 2.0.14 and verify registration
  through a companion plugin, and update CI to install dependencies.
- Require OpenCode >= 2.0.0 and depend on @opencode/plugin ^2.0.14.
2026-09-23 18:20:17 +10:00

238 lines
8.8 KiB
JavaScript

import assert from "node:assert/strict";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import { createServer } from "node:net";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { spawn } from "node:child_process";
const version = process.argv[2] || "2.0.14";
const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const pluginEntrypoint = pathToFileURL(
path.join(repo, ".opencode/plugins/opencode-power-pack.js"),
).href;
const expectedSkills = readdirSync(path.join(repo, "skills"), { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
.sort();
const expectedAgents = ["code-architect", "code-explorer", "code-reviewer"];
const serverPassword = "opencode-smoke-password";
const authorization = `Basic ${Buffer.from(`opencode:${serverPassword}`).toString("base64")}`;
const verifierSource = [
'import { writeFileSync } from "node:fs";',
"",
"export default {",
' id: "opencode-power-pack-smoke",',
" async setup(ctx) {",
" const skills = await ctx.skill.list();",
" const agents = await ctx.agent.list();",
" writeFileSync(process.env.OPENCODE_SMOKE_RESULT, JSON.stringify({",
" skills: skills.data,",
" agents: agents.data,",
" }));",
" },",
"};",
"",
].join("\n");
async function availablePort() {
const server = createServer();
await new Promise((resolve, reject) => server.listen(0, "127.0.0.1", resolve).once("error", reject));
const address = server.address();
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
return address.port;
}
async function waitForServer(url, child, stderr) {
const deadline = Date.now() + 60_000;
while (Date.now() < deadline) {
if (child.exitCode !== null || child.signalCode !== null) {
throw new Error(`OpenCode exited early:\n${stderr.join("")}`);
}
try {
const response = await fetch(`${url}/api/info`, {
headers: { authorization, Connection: "close" },
signal: AbortSignal.timeout(1_000),
});
if (response.ok) return;
} catch {
// Startup can take a few seconds while npx resolves the pinned package.
}
await new Promise((resolve) => setTimeout(resolve, 200));
}
throw new Error(`Timed out waiting for OpenCode:\n${stderr.join("")}`);
}
export async function fetchJson(url, timeoutMs = 10_000, retries = 5, headers = {}) {
for (let attempt = 0; ; attempt++) {
try {
return await fetchJsonOnce(url, timeoutMs, headers);
} catch (error) {
// The health endpoint can report ready slightly before OpenCode finishes
// registering every plugin's skills. With enough skills, that window is
// long enough for a socket to be reset mid-response. Retry with backoff
// instead of failing on the first transient close.
const transient = error.cause?.code === "UND_ERR_SOCKET" || error.name === "TimeoutError";
if (!transient || attempt >= retries) throw error;
await new Promise((resolve) => setTimeout(resolve, 500 * (attempt + 1)));
}
}
}
async function fetchJsonOnce(url, timeoutMs, headers) {
try {
// No `Connection: close` header here: pairing it with a large, still-being-
// generated response body (the skill list grows with every skill in the
// pack) reproducibly triggers a premature socket reset in undici. Let the
// client negotiate keep-alive normally instead.
const response = await fetch(url, {
headers,
signal: AbortSignal.timeout(timeoutMs),
});
if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}`);
return await response.json();
} catch (error) {
if (error.name === "TimeoutError") {
throw new Error(`Request timed out after ${timeoutMs}ms: ${url}`, { cause: error });
}
throw error;
}
}
export async function stopServer(child, graceMs = 5_000) {
if (child.exitCode !== null || child.signalCode !== null) return;
const exited = new Promise((resolve) => child.once("exit", resolve));
try {
if (process.platform === "win32") child.kill("SIGTERM");
else process.kill(-child.pid, "SIGTERM");
} catch (error) {
if (error.code === "ESRCH") return;
throw error;
}
await Promise.race([
exited,
new Promise((resolve) => setTimeout(resolve, graceMs)),
]);
if (child.exitCode !== null || child.signalCode !== null) return;
try {
if (process.platform === "win32") child.kill("SIGKILL");
else process.kill(-child.pid, "SIGKILL");
} catch (error) {
if (error.code === "ESRCH") return;
throw error;
}
await Promise.race([
exited,
new Promise((resolve) => setTimeout(resolve, graceMs)),
]);
}
function lastRule(agent, action, resource) {
return agent.permissions
.filter((rule) => rule.action === action && rule.resource === resource)
.at(-1)?.effect;
}
async function waitForResult(resultPath, child, stderr) {
const deadline = Date.now() + 60_000;
while (Date.now() < deadline) {
if (existsSync(resultPath)) return JSON.parse(readFileSync(resultPath, "utf8"));
if (child.exitCode !== null || child.signalCode !== null) {
throw new Error(`OpenCode exited early:\n${stderr.join("")}`);
}
await new Promise((resolve) => setTimeout(resolve, 200));
}
throw new Error(`Timed out waiting for the plugin verifier:\n${stderr.join("")}`);
}
async function main() {
const home = mkdtempSync(path.join(tmpdir(), "opencode-power-pack-"));
const project = path.join(home, "project");
mkdirSync(project);
const pluginDirectory = path.join(home, "plugin");
mkdirSync(pluginDirectory);
writeFileSync(
path.join(pluginDirectory, "index.js"),
`export { default } from ${JSON.stringify(pluginEntrypoint)};\n`,
);
const verifierDirectory = path.join(home, "verify");
mkdirSync(verifierDirectory);
writeFileSync(path.join(verifierDirectory, "index.js"), verifierSource);
const resultPath = path.join(home, "result.json");
const port = await availablePort();
const url = `http://127.0.0.1:${port}`;
const stderr = [];
const child = spawn(
process.platform === "win32" ? "npx.cmd" : "npx",
["-y", "-p", `@opencode/cli@${version}`, "opencode", "serve", "--hostname", "127.0.0.1", "--port", String(port)],
{
cwd: project,
env: {
...process.env,
HOME: home,
XDG_CONFIG_HOME: path.join(home, "config"),
XDG_DATA_HOME: path.join(home, "data"),
XDG_CACHE_HOME: path.join(home, "cache"),
XDG_STATE_HOME: path.join(home, "state"),
npm_config_cache: process.env.npm_config_cache || path.join(process.env.HOME || tmpdir(), ".npm"),
OPENCODE_SERVER_PASSWORD: serverPassword,
OPENCODE_CONFIG_CONTENT: JSON.stringify({ plugins: [pluginDirectory, verifierDirectory] }),
OPENCODE_SMOKE_RESULT: resultPath,
},
detached: process.platform !== "win32",
stdio: ["ignore", "ignore", "pipe"],
},
);
child.stderr.on("data", (chunk) => stderr.push(chunk.toString()));
try {
await waitForServer(url, child, stderr);
// The first location request loads the configured plugins and runs the
// verifier during plugin setup.
await fetch(`${url}/api/skill`, { headers: { authorization } });
const { skills, agents } = await waitForResult(resultPath, child, stderr);
for (const name of expectedSkills) {
const skill = skills.find((entry) => entry.id === name);
assert.ok(skill, `skill exists for ${name}`);
assert.match(skill.description ?? "", /\S/, `${name} has a description`);
assert.equal(
skill.path,
path.join(repo, "skills", name, "SKILL.md"),
`${name} resolves to its bundled SKILL.md`,
);
assert.doesNotMatch(skill.content, /^---/, `${name} registers its body without frontmatter`);
}
for (const name of expectedAgents) {
const agent = agents.find((entry) => entry.id === name);
assert.ok(agent, `${name} agent exists`);
assert.equal(agent.mode, "subagent");
assert.equal(lastRule(agent, "read", "*.env"), "deny", `${name} cannot read environment files`);
assert.equal(lastRule(agent, "edit", "*"), "deny", `${name} cannot edit files`);
}
const reviewer = agents.find((entry) => entry.id === "code-reviewer");
assert.equal(
lastRule(reviewer, "shell", "git *--output*"),
"deny",
"code-reviewer cannot write through Git output options",
);
console.log(
`OpenCode ${version}: ${expectedSkills.length} skills and ${expectedAgents.length} agents verified`,
);
} finally {
try {
await stopServer(child);
} finally {
rmSync(home, { recursive: true, force: true });
}
}
}
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main();