mirror of
https://github.com/waybarrios/opencode-power-pack.git
synced 2026-11-18 18:03:43 +00:00
Update the skills for OpenCode 2 and keep OpenCode 1 support
This commit is contained in:
@@ -4,19 +4,32 @@ on:
|
||||
pull_request:
|
||||
paths:
|
||||
- ".opencode/**"
|
||||
- "index.js"
|
||||
- "skills/**"
|
||||
- "scripts/opencode-smoke.mjs"
|
||||
- "tests/opencode-smoke.test.mjs"
|
||||
- "package.json"
|
||||
- "package-lock.json"
|
||||
- ".github/workflows/opencode-smoke.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
opencode-1-18-7:
|
||||
opencode:
|
||||
name: opencode-${{ matrix.version }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
timeout-minutes: 10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
version: ["1.18.7", "2.0.14", "2.0.32"]
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: "24"
|
||||
- run: node scripts/opencode-smoke.mjs 1.18.7
|
||||
cache: "npm"
|
||||
- run: npm ci
|
||||
- run: node scripts/opencode-smoke.mjs ${{ matrix.version }}
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import { readdirSync, readFileSync, statSync } from "fs";
|
||||
import path from "path";
|
||||
|
||||
const AGENT_NAMES = ["code-explorer", "code-architect", "code-reviewer"];
|
||||
const READ_ONLY_PERMISSION = {
|
||||
"*": "deny",
|
||||
read: {
|
||||
"*": "allow",
|
||||
"*.env": "deny",
|
||||
"*.env.*": "deny",
|
||||
"*.env.example": "allow",
|
||||
},
|
||||
glob: "allow",
|
||||
grep: "allow",
|
||||
list: "allow",
|
||||
edit: "deny",
|
||||
task: "deny",
|
||||
webfetch: "deny",
|
||||
websearch: "deny",
|
||||
external_directory: "deny",
|
||||
};
|
||||
const REVIEW_BASH_PERMISSION = {
|
||||
"*": "deny",
|
||||
"git status*": "allow",
|
||||
"git diff*": "allow",
|
||||
"git show*": "allow",
|
||||
"git log*": "allow",
|
||||
"git blame*": "allow",
|
||||
"git rev-parse*": "allow",
|
||||
"git merge-base*": "allow",
|
||||
"git ls-files*": "allow",
|
||||
"git *--output*": "deny",
|
||||
"git *--ext-diff*": "deny",
|
||||
"git *>*": "deny",
|
||||
};
|
||||
// OpenCode 2 renamed these V1 permission keys and folded `list` into `read`.
|
||||
const V2_ACTIONS = { bash: "shell", task: "subagent", list: null };
|
||||
|
||||
function readSkill(skillsDir, id) {
|
||||
const file = path.join(skillsDir, id, "SKILL.md");
|
||||
const match = readFileSync(file, "utf8").match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/);
|
||||
if (!match) throw new Error(`Invalid frontmatter in ${id}/SKILL.md`);
|
||||
|
||||
const field = (key) => match[1]
|
||||
.split(/\r?\n/)
|
||||
.find((line) => line.startsWith(`${key}:`))
|
||||
?.slice(key.length + 1)
|
||||
.trim();
|
||||
const description = field("description");
|
||||
if (!description) throw new Error(`Missing description in ${id}/SKILL.md`);
|
||||
|
||||
// Native OpenCode 2 discovery strips frontmatter, so the body is the content.
|
||||
return { id, name: field("name") || id, description, path: file, content: match[2] };
|
||||
}
|
||||
|
||||
/** Parses every bundled SKILL.md into the OpenCode 2 Skill.Info shape. */
|
||||
export function loadSkills(skillsDir) {
|
||||
return readdirSync(skillsDir)
|
||||
.filter((entry) => statSync(path.join(skillsDir, entry, "SKILL.md"), { throwIfNoEntry: false })?.isFile())
|
||||
.sort()
|
||||
.map((id) => readSkill(skillsDir, id));
|
||||
}
|
||||
|
||||
function specialists(skills) {
|
||||
return AGENT_NAMES.map((name) => {
|
||||
const skill = skills.find(({ id }) => id === name);
|
||||
if (!skill) throw new Error(`Missing specialist skill ${name}`);
|
||||
return skill;
|
||||
});
|
||||
}
|
||||
|
||||
// Returns a fresh copy on every call: OpenCode edits agent permissions in place.
|
||||
function permissionFor(name) {
|
||||
return structuredClone({
|
||||
...READ_ONLY_PERMISSION,
|
||||
bash: name === "code-reviewer" ? REVIEW_BASH_PERMISSION : "deny",
|
||||
});
|
||||
}
|
||||
|
||||
/** Converts a V1 permission map into the ordered V2 ruleset, preserving key order. */
|
||||
function toRuleset(permission) {
|
||||
return Object.entries(permission).flatMap(([key, value]) => {
|
||||
const action = key in V2_ACTIONS ? V2_ACTIONS[key] : key;
|
||||
if (!action) return [];
|
||||
const patterns = typeof value === "string" ? { "*": value } : value;
|
||||
return Object.entries(patterns).map(([resource, effect]) => ({ action, resource, effect }));
|
||||
});
|
||||
}
|
||||
|
||||
/** Matches the host's whole-value * and ? permission wildcards. */
|
||||
function matches(pattern, value) {
|
||||
const expression = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&")
|
||||
.replace(/\*/g, ".*").replace(/\?/g, ".");
|
||||
return new RegExp(`^${expression}$`, process.platform === "win32" ? "is" : "s").test(value);
|
||||
}
|
||||
|
||||
/** Evaluates our ordered policy without granting access denied by the host. */
|
||||
export function permissionEffect(rules, action, resource) {
|
||||
// The reviewer's Git prefixes authorize one inspection command, not shell
|
||||
// composition, redirection, or command substitution following that prefix.
|
||||
if (action === "shell" && /[;&|<>`$\r\n]/.test(resource)) return "deny";
|
||||
const normalized = ["read", "edit", "external_directory"].includes(action)
|
||||
? resource.replaceAll("\\", "/") : resource;
|
||||
return rules.findLast((rule) => matches(rule.action, action) && matches(rule.resource, normalized))?.effect ?? "deny";
|
||||
}
|
||||
|
||||
/** OpenCode 1 agent definitions, merged through the plugin config hook. */
|
||||
export function legacyAgents(skills) {
|
||||
return Object.fromEntries(specialists(skills).map((skill) => [skill.id, {
|
||||
description: skill.description,
|
||||
prompt: skill.content.trim(),
|
||||
mode: "subagent",
|
||||
permission: permissionFor(skill.id),
|
||||
}]));
|
||||
}
|
||||
|
||||
/** OpenCode 2 Agent.Info patches, applied through an agent transform. */
|
||||
export function agents(skills) {
|
||||
return Object.fromEntries(specialists(skills).map((skill) => [skill.id, {
|
||||
name: skill.name,
|
||||
description: skill.description,
|
||||
system: skill.content.trim(),
|
||||
mode: "subagent",
|
||||
hidden: false,
|
||||
permissions: toRuleset(permissionFor(skill.id)),
|
||||
}]));
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
import { readFileSync } from "fs";
|
||||
import path from "path";
|
||||
|
||||
const AGENT_NAMES = ["code-explorer", "code-architect", "code-reviewer"];
|
||||
const READ_ONLY_PERMISSION = {
|
||||
"*": "deny",
|
||||
read: {
|
||||
"*": "allow",
|
||||
"*.env": "deny",
|
||||
"*.env.*": "deny",
|
||||
"*.env.example": "allow",
|
||||
},
|
||||
glob: "allow",
|
||||
grep: "allow",
|
||||
list: "allow",
|
||||
edit: "deny",
|
||||
task: "deny",
|
||||
webfetch: "deny",
|
||||
websearch: "deny",
|
||||
external_directory: "deny",
|
||||
};
|
||||
const REVIEW_BASH_PERMISSION = {
|
||||
"*": "deny",
|
||||
"git status*": "allow",
|
||||
"git diff*": "allow",
|
||||
"git show*": "allow",
|
||||
"git log*": "allow",
|
||||
"git blame*": "allow",
|
||||
"git rev-parse*": "allow",
|
||||
"git merge-base*": "allow",
|
||||
"git ls-files*": "allow",
|
||||
"git *--output*": "deny",
|
||||
"git *--ext-diff*": "deny",
|
||||
"git *>*": "deny",
|
||||
};
|
||||
|
||||
function readSkill(skillsDir, name) {
|
||||
const source = readFileSync(path.join(skillsDir, name, "SKILL.md"), "utf8");
|
||||
const match = source.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/);
|
||||
if (!match) throw new Error(`Invalid frontmatter in ${name}/SKILL.md`);
|
||||
|
||||
const description = match[1]
|
||||
.split(/\r?\n/)
|
||||
.find((line) => line.startsWith("description:"))
|
||||
?.slice("description:".length)
|
||||
.trim();
|
||||
if (!description) throw new Error(`Missing description in ${name}/SKILL.md`);
|
||||
|
||||
return { description, prompt: match[2].trim() };
|
||||
}
|
||||
|
||||
export function loadAgentConfigs(skillsDir) {
|
||||
return Object.fromEntries(AGENT_NAMES.map((name) => {
|
||||
const skill = readSkill(skillsDir, name);
|
||||
const permission = name === "code-reviewer"
|
||||
? { ...READ_ONLY_PERMISSION, bash: REVIEW_BASH_PERMISSION }
|
||||
: { ...READ_ONLY_PERMISSION, bash: "deny" };
|
||||
|
||||
return [name, {
|
||||
...skill,
|
||||
mode: "subagent",
|
||||
permission,
|
||||
}];
|
||||
}));
|
||||
}
|
||||
@@ -1,22 +1,21 @@
|
||||
/**
|
||||
* opencode-power-pack
|
||||
*
|
||||
* Auto-registers the bundled skills directory so OpenCode discovers all
|
||||
* skills shipped by this plugin (code-review, feature-dev, code-explorer,
|
||||
* code-architect, code-reviewer, security-review, frontend-design,
|
||||
* mcp-builder, skill-creator, agents-md-improver, agents-md-revise) without
|
||||
* requiring symlinks or manual config.
|
||||
* Registers every bundled skill and the feature workflow's specialist roles
|
||||
* as read-only subagents derived from their SKILL.md bodies, on both hosts:
|
||||
*
|
||||
* OpenCode 1.18.7+ exposes discovered skills as same-named slash commands.
|
||||
* The plugin also registers the feature workflow's specialist roles as
|
||||
* read-only subagents derived from their SKILL.md bodies.
|
||||
* - OpenCode 2 calls `setup` and receives the skill and agent transforms.
|
||||
* - OpenCode 1 (1.18.7+) calls `server` on the default export and receives
|
||||
* the config hook, which registers skills/ in `config.skills.paths` and
|
||||
* merges the agents into `config.agent`.
|
||||
*
|
||||
* User-defined agents with the same names are preserved.
|
||||
*
|
||||
* ──── Attribution ────────────────────────────────────────────────────────
|
||||
*
|
||||
* The plugin loader pattern (importing fs/path via fileURLToPath, exporting
|
||||
* an async ctx → hooks function, pushing into config.skills.paths via the
|
||||
* `config` hook) is adapted directly from Jesse Vincent's superpowers
|
||||
* plugin: https://github.com/obra/superpowers
|
||||
* The OpenCode 1 loader pattern (importing fs/path via fileURLToPath and
|
||||
* pushing into config.skills.paths via the `config` hook) is adapted directly
|
||||
* from Jesse Vincent's superpowers plugin: https://github.com/obra/superpowers
|
||||
*
|
||||
* The skills under skills/ are modified upstream works. See UPSTREAMS.json
|
||||
* for immutable source commits and blobs, and THIRD_PARTY_NOTICES.md for
|
||||
@@ -26,24 +25,53 @@
|
||||
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { loadAgentConfigs } from './agent-config.js';
|
||||
import { agents, legacyAgents, loadSkills, permissionEffect } from '../lib/agent-config.js';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const skillsDir = path.resolve(__dirname, '../../skills');
|
||||
const bundledAgents = loadAgentConfigs(skillsDir);
|
||||
const skillsDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../skills');
|
||||
const skills = loadSkills(skillsDir);
|
||||
|
||||
export const OpencodePowerPack = async () => {
|
||||
return {
|
||||
config: async (config) => {
|
||||
config.skills = config.skills || {};
|
||||
config.skills.paths = config.skills.paths || [];
|
||||
if (!config.skills.paths.includes(skillsDir)) {
|
||||
config.skills.paths.push(skillsDir);
|
||||
export default {
|
||||
id: 'opencode-power-pack',
|
||||
|
||||
async setup(ctx) {
|
||||
const policies = agents(skills);
|
||||
await ctx.skill.transform((editor) => {
|
||||
for (const skill of skills) editor.add({ ...skill });
|
||||
});
|
||||
await ctx.agent.transform((editor) => {
|
||||
for (const [name, { permissions, ...agent }] of Object.entries(policies)) {
|
||||
// V2 applies user agent configuration later. Keep its permission defaults
|
||||
// intact so a same-named user replacement does not inherit our denials.
|
||||
if (!editor.get(name)) editor.update(name, (draft) => Object.assign(draft, agent));
|
||||
}
|
||||
config.agent = config.agent || {};
|
||||
for (const [name, agent] of Object.entries(bundledAgents)) {
|
||||
if (!config.agent[name]) config.agent[name] = agent;
|
||||
});
|
||||
await ctx.permission.hook('evaluate', async (event) => {
|
||||
const name = event.agent ?? (await ctx.session.get({ sessionID: event.sessionID })).data?.agent;
|
||||
const policy = Object.hasOwn(policies, name) ? policies[name] : undefined;
|
||||
if (!policy) return;
|
||||
const { data: agent } = await ctx.agent.get({ agentID: name });
|
||||
if (agent?.system !== policy.system || agent.mode !== 'subagent') return;
|
||||
// Global and saved allows must not weaken the bundled read-only policy.
|
||||
// This hook only denies; explicit user denies and approval prompts remain.
|
||||
if (event.resources.some((resource) => permissionEffect(policy.permissions, event.action, resource) === 'deny')) {
|
||||
event.effect = 'deny';
|
||||
event.message = `The bundled ${name} agent is read-only.`;
|
||||
}
|
||||
},
|
||||
};
|
||||
});
|
||||
},
|
||||
|
||||
async server() {
|
||||
return {
|
||||
config: async (config) => {
|
||||
config.skills = config.skills || {};
|
||||
config.skills.paths = config.skills.paths || [];
|
||||
if (!config.skills.paths.includes(skillsDir)) config.skills.paths.push(skillsDir);
|
||||
|
||||
config.agent = config.agent || {};
|
||||
for (const [name, agent] of Object.entries(legacyAgents(skills))) {
|
||||
if (!config.agent[name]) config.agent[name] = agent;
|
||||
}
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<a href="https://github.com/hashgraph-online/hol-guard"><img alt="HOL Guard scanner" src="https://img.shields.io/badge/HOL%20Guard-scanned-00a67e?style=flat-square"></a>
|
||||
<img alt="Skills: 55" src="https://img.shields.io/badge/skills-55-FFD60A?style=flat-square&labelColor=0B0F14">
|
||||
<img alt="Claude Code plugin" src="https://img.shields.io/badge/Claude_Code-plugin-0B0F14?style=flat-square&labelColor=FFD60A">
|
||||
<img alt="OpenCode 1.18.7+" src="https://img.shields.io/badge/opencode-1.18.7%2B-0B0F14?style=flat-square&labelColor=FFD60A">
|
||||
<img alt="OpenCode 1.18.7+ / 2.x" src="https://img.shields.io/badge/opencode-1.18.7%2B%20%2F%202.x-0B0F14?style=flat-square&labelColor=FFD60A">
|
||||
<img alt="Codex plugin" src="https://img.shields.io/badge/Codex-plugin-0B0F14?style=flat-square&labelColor=FFD60A">
|
||||
<img alt="Pi package" src="https://img.shields.io/badge/Pi-package-0B0F14?style=flat-square&labelColor=FFD60A">
|
||||
</p>
|
||||
@@ -329,7 +329,7 @@ They are complementary rather than duplicate workflows. Codex shows distinct use
|
||||
- Node.js 20.11.0 or newer, including npm/npx, for [selective installation](#selective-install-with-npm) and the sandbox runner
|
||||
- One supported host:
|
||||
- A current Claude Code installation with plugin support: <https://code.claude.com/docs/en/plugins>
|
||||
- OpenCode 1.18.7 or newer: <https://opencode.ai>
|
||||
- OpenCode 1.18.7 or newer, including OpenCode 2: <https://opencode.ai>
|
||||
- A current Codex CLI or Codex desktop environment with plugin support: <https://developers.openai.com/codex/>
|
||||
- A current Pi coding agent installation: <https://pi.dev/>
|
||||
|
||||
@@ -360,26 +360,36 @@ For a smaller personal or repository-specific set, use the [selective npm instal
|
||||
|
||||
### OpenCode From GitHub
|
||||
|
||||
On OpenCode 2:
|
||||
|
||||
```bash
|
||||
opencode plugin add "opencode-power-pack@git+https://github.com/waybarrios/opencode-power-pack.git"
|
||||
```
|
||||
|
||||
On OpenCode 1:
|
||||
|
||||
```bash
|
||||
opencode plugin --global "opencode-power-pack@git+https://github.com/waybarrios/opencode-power-pack.git"
|
||||
```
|
||||
|
||||
Restart OpenCode after installation. OpenCode discovers each skill and creates its same-named slash command automatically.
|
||||
Restart OpenCode after installation. One plugin adapts to the host version: OpenCode 2 registers all fifty-five skills and the three specialist agents through the V2 plugin API, and OpenCode 1 keeps the original `config.skills.paths` and `config.agent` hook. Either way the skills are invocable by name and the three roles are registered as read-only subagents.
|
||||
|
||||
To pin a published release, append its tag:
|
||||
The compatibility smoke tests cover OpenCode **1.18.7, 2.0.14, and 2.0.32**. The `SKILL.md` format stays the same across these hosts.
|
||||
|
||||
To pin a published release, append its tag to the same spec:
|
||||
|
||||
```bash
|
||||
opencode plugin --global "opencode-power-pack@git+https://github.com/waybarrios/opencode-power-pack.git#<tag>"
|
||||
opencode plugin add "opencode-power-pack@git+https://github.com/waybarrios/opencode-power-pack.git#<tag>"
|
||||
```
|
||||
|
||||
### OpenCode From A Local Clone
|
||||
|
||||
```bash
|
||||
git clone https://github.com/waybarrios/opencode-power-pack.git ~/code/opencode-power-pack
|
||||
opencode plugin --global "opencode-power-pack@git+file:///home/you/code/opencode-power-pack"
|
||||
opencode plugin add "opencode-power-pack@git+file:///home/you/code/opencode-power-pack"
|
||||
```
|
||||
|
||||
Use an absolute `file://` URL adjusted for your operating system. The target directory must be a Git repository.
|
||||
Use an absolute `file://` URL adjusted for your operating system. The target directory must be a Git repository. On OpenCode 1, replace `plugin add` with `plugin --global` as shown above.
|
||||
|
||||
### Pi From GitHub
|
||||
|
||||
@@ -403,6 +413,18 @@ After marketplace installation, run `/plugin` and confirm that `opencode-power-p
|
||||
|
||||
### Verify OpenCode
|
||||
|
||||
Restart OpenCode, then check the registered skills and specialist agents.
|
||||
|
||||
On OpenCode 2:
|
||||
|
||||
```bash
|
||||
opencode debug agents
|
||||
```
|
||||
|
||||
The output should include `code-explorer`, `code-architect`, and `code-reviewer` as `subagent` entries. OpenCode 2 enforces their read-only restrictions when evaluating permission requests, so the agent listing can show the host's base permissions. In the TUI, the slash catalog should list `/code-review`, `/feature-dev`, `/frontend-design`, and the other bundled skills.
|
||||
|
||||
On OpenCode 1:
|
||||
|
||||
```bash
|
||||
opencode debug skill
|
||||
opencode debug agent code-explorer
|
||||
@@ -452,13 +474,19 @@ For Pi:
|
||||
pi update git:github.com/waybarrios/opencode-power-pack
|
||||
```
|
||||
|
||||
For an OpenCode GitHub installation:
|
||||
For an OpenCode GitHub installation on OpenCode 2:
|
||||
|
||||
```bash
|
||||
opencode plugin --global --force "opencode-power-pack@git+https://github.com/waybarrios/opencode-power-pack.git"
|
||||
opencode plugin update "opencode-power-pack@git+https://github.com/waybarrios/opencode-power-pack.git"
|
||||
```
|
||||
|
||||
Restart OpenCode after the command finishes. For a pinned installation, update the tag first. Local-clone users should run `git pull` in the clone and reinstall with the same `git+file://` spec.
|
||||
On OpenCode 1, reinstall with the same spec:
|
||||
|
||||
```bash
|
||||
opencode plugin --global "opencode-power-pack@git+https://github.com/waybarrios/opencode-power-pack.git"
|
||||
```
|
||||
|
||||
Restart OpenCode after the command finishes. Pass the exact configured package specification, including any pinned tag. Local-clone users should run `git pull` in the clone, then update using their original `opencode-power-pack@git+file://...` specification on OpenCode 2, or reinstall it on OpenCode 1.
|
||||
|
||||
## Uninstalling
|
||||
|
||||
@@ -480,7 +508,13 @@ For Pi:
|
||||
pi remove git:github.com/waybarrios/opencode-power-pack
|
||||
```
|
||||
|
||||
For OpenCode, remove the `opencode-power-pack@...` entry from the `plugin` array in the config where it was installed, then restart OpenCode. There are no command symlinks or copied command files to remove.
|
||||
For OpenCode 2, remove the plugin from the global configuration:
|
||||
|
||||
```bash
|
||||
opencode plugin remove "opencode-power-pack@git+https://github.com/waybarrios/opencode-power-pack.git"
|
||||
```
|
||||
|
||||
Use your exact configured package specification if it differs from this example. Alternatively, delete the `opencode-power-pack@...` entry from the `plugins` array in the config where it was installed, then restart OpenCode. On OpenCode 1, delete the entry from the `plugin` array instead. There are no command symlinks or copied command files to remove.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
@@ -491,15 +525,15 @@ For OpenCode, remove the `opencode-power-pack@...` entry from the `plugin` array
|
||||
| Codex cannot find the marketplace | Marketplace snapshot is missing or stale | Run `codex plugin marketplace add waybarrios/opencode-power-pack --ref main`, or upgrade the existing marketplace |
|
||||
| Codex installed the plugin but skills do not appear | The current session predates installation | Start a new Codex session and inspect `/plugins` |
|
||||
| Pi does not show the skills | Package is absent, disabled, or the session predates installation | Run `pi list`, inspect `pi config`, and start a new Pi session |
|
||||
| Skills or commands do not appear | OpenCode is older than 1.18.7 | Upgrade OpenCode, restart, and run `opencode debug skill` |
|
||||
| Skills are missing from debug output | Plugin installation failed or its config entry is absent | Re-run `opencode plugin --global --force <module-spec>` and inspect the reported error |
|
||||
| Specialist agents are missing | Stale plugin checkout | Force reinstall, restart, and run `opencode debug agent code-explorer` |
|
||||
| Skills or commands do not appear | OpenCode is older than 1.18.7 | Upgrade OpenCode, restart, and check the slash catalog or run `opencode debug skill` |
|
||||
| Skills are missing from the catalog | Plugin installation failed or its `plugins` entry is absent | Reinstall with `opencode plugin add <module-spec>` (OpenCode 2) or `opencode plugin --global <module-spec>` (OpenCode 1) and inspect the reported error |
|
||||
| Specialist agents are missing | Stale plugin checkout | Update the plugin, restart, and run `opencode debug agents` (OpenCode 2) or `opencode debug agent code-explorer` (OpenCode 1) |
|
||||
| Installation reports a Git error | Invalid URL, network failure, or inaccessible repository | Validate the source with `git ls-remote <url>` |
|
||||
| A workflow is rushed or incomplete | The backing model skipped multi-stage instructions | Use a stronger model and inspect whether required subagent tools are available |
|
||||
|
||||
## Invocation
|
||||
|
||||
Claude Code exposes plugin skills as `/opencode-power-pack:<skill>`, Codex uses `$` mentions, OpenCode 1.18.7+ exposes same-named slash commands, and Pi loads workflows from the installed skills package:
|
||||
Claude Code exposes plugin skills as `/opencode-power-pack:<skill>`, Codex uses `$` mentions, OpenCode exposes the bundled skills by name (`/` catalog on OpenCode 2, same-named slash commands on OpenCode 1), and Pi loads workflows from the installed skills package:
|
||||
|
||||
```text
|
||||
/code-review
|
||||
@@ -560,11 +594,16 @@ opencode-power-pack
|
||||
| +-- runs opt-in commands in the pinned native sandbox backend
|
||||
|
|
||||
+-- .opencode/plugins/opencode-power-pack.js
|
||||
| +-- registers skills/ in config.skills.paths
|
||||
| +-- OpenCode 1: registers config.skills.paths and config.agent
|
||||
| +-- OpenCode 2: registers skills and agents through the V2 transforms
|
||||
| +-- registers code-explorer as a read-only subagent
|
||||
| +-- registers code-architect as a read-only subagent
|
||||
| +-- registers code-reviewer with read-only Git commands
|
||||
|
|
||||
+-- .opencode/lib/agent-config.js
|
||||
| +-- derives skill and subagent definitions from the SKILL.md frontmatter
|
||||
| +-- builds the OpenCode 1 permission maps and OpenCode 2 rulesets
|
||||
|
|
||||
+-- skills/<name>/SKILL.md
|
||||
| +-- shared Codex/OpenCode workflow
|
||||
| +-- host-native invocation metadata
|
||||
@@ -574,9 +613,11 @@ opencode-power-pack
|
||||
+-- repository, commit, path, blob, date, and adaptation type
|
||||
```
|
||||
|
||||
Each `SKILL.md` is the single source for its workflow. Claude Code loads the directory through its plugin namespace. The OpenCode plugin derives specialist-agent prompts from those files at startup. Codex loads the same skills from its plugin and follows `feature-dev`'s specialist assignments with its native subagent workflow; installing a skill does not itself create a named custom agent.
|
||||
Each `SKILL.md` is the single source for its workflow. Claude Code loads the directory through its plugin namespace. The OpenCode plugin adapts to the host version and derives specialist-agent prompts from those files at startup. Codex loads the same skills from its plugin and follows `feature-dev`'s specialist assignments with its native subagent workflow; installing a skill does not itself create a named custom agent.
|
||||
|
||||
The packaged agents deny edits, external network access, and nested tasks. `code-reviewer` additionally allows a narrow set of read-only Git commands.
|
||||
The packaged agents deny edits, external network access, and nested subagents. `code-reviewer` additionally allows a narrow set of read-only Git commands.
|
||||
|
||||
On OpenCode 2, permission evaluation enforces these restrictions even when global rules allow broader access. User-configured agents with their own system prompt or a primary mode replace the bundled role and keep the host's permissions. Explicit user denials and approval prompts still apply to bundled agents.
|
||||
|
||||
## Scope And Non-Goals
|
||||
|
||||
|
||||
@@ -103,8 +103,9 @@ Copyright 2025 Anthropic. The upstream notice is available at
|
||||
|
||||
## Superpowers OpenCode Plugin Pattern
|
||||
|
||||
The skill-path registration pattern in `.opencode/plugins/opencode-power-pack.js`
|
||||
is adapted from the MIT-licensed `obra/superpowers` OpenCode plugin.
|
||||
The V1 skill-path registration pattern in the OpenCode plugin was adapted from
|
||||
the MIT-licensed `obra/superpowers` OpenCode plugin. The plugin keeps that
|
||||
hook for OpenCode 1 and adds the OpenCode 2 port through the V2 plugin API.
|
||||
|
||||
Copyright 2025 Jesse Vincent. The upstream notice is available at
|
||||
`LICENSES/Superpowers-MIT.txt`.
|
||||
|
||||
@@ -8,11 +8,13 @@ Locally inspected host versions:
|
||||
|
||||
- Codex CLI 0.147.0
|
||||
- Claude Code 2.1.231
|
||||
- OpenCode 1.18.18
|
||||
- OpenCode 1.18.32 and 2.0.14 (migration baseline)
|
||||
- Pi 0.83.0
|
||||
|
||||
The repository smoke tests retain their separately pinned minimum versions. Host APIs evolve, so adapter tests and this matrix must be updated together.
|
||||
|
||||
OpenCode plugin compatibility is additionally verified by model-free smoke tests on 1.18.7, 2.0.14, and 2.0.32 (2026-10-19). V2 checks load the real package directory, inspect the final catalogs, and evaluate specialist permission requests with permissive global rules. This checks plugin permissions, not whole-agent sandbox isolation.
|
||||
|
||||
## Compatibility Levels
|
||||
|
||||
| Level | Meaning |
|
||||
@@ -31,14 +33,14 @@ The current runner is `shell-contained`. The command and its descendants are iso
|
||||
|---|---|---|---|---|---|
|
||||
| Codex | Supported through the Codex plugin and portable skill installation | Supported when the npm package is installed | Not yet implemented | Temporary native sandbox configuration plus the shared runner for explicit network access | Codex has native `read-only` and `workspace-write` shell sandboxes, but a skill cannot safely change the active session sandbox by prompt text alone. |
|
||||
| Claude Code | Supported through the namespaced Claude plugin | Supported when the npm package is installed | Not yet implemented | Native sandbox settings plus hooks and permissions for tools outside Bash | Claude's native sandbox applies to Bash and descendants. Read, Edit, Write, WebFetch, MCP, and computer-use tools require separate permission controls. |
|
||||
| OpenCode | Supported through the plugin and portable skill installation | Supported when the npm package is installed | Not yet implemented | A custom sandbox tool paired with per-agent denial of ordinary `bash` | File-edit and external-directory permissions also need profile-specific controls. |
|
||||
| OpenCode | Supported through the plugin and portable skill installation | Supported when the npm package is installed | Not yet implemented | A custom sandbox tool paired with per-agent denial of ordinary `shell` | File-edit and external-directory permissions also need profile-specific controls. |
|
||||
| Pi | Supported through the Pi package declaration and portable skill installation | Supported when the npm package is installed | Not yet implemented | An extension that intercepts or replaces built-in shell and file tools | A skill file by itself is advisory. |
|
||||
|
||||
Official host surfaces used by the adapter design:
|
||||
|
||||
- [Codex configuration reference](https://developers.openai.com/codex/config-reference/) documents `sandbox_mode`, workspace-write network controls, permission profiles, and environment policy.
|
||||
- [Claude Code sandboxing](https://code.claude.com/docs/en/sandboxing) documents its Bash-only OS sandbox, `/sandbox`, `failIfUnavailable`, and the unsandboxed escape hatch. [Claude Code hooks](https://code.claude.com/docs/en/hooks) documents `PreToolUse` denial and input replacement.
|
||||
- [OpenCode agents](https://opencode.ai/docs/agents/) documents per-agent `allow`, `ask`, and `deny` permissions. [OpenCode custom tools](https://opencode.ai/docs/custom-tools/) provides the portable runner integration point.
|
||||
- [OpenCode permissions](https://opencode.ai/v2/docs/permissions/) documents the ordered `allow`, `ask`, and `deny` ruleset. [OpenCode tools](https://opencode.ai/v2/docs/tools/) and the [plugin tools API](https://opencode.ai/v2/docs/build/plugins/) provide the portable runner integration point.
|
||||
- [Pi extensions](https://github.com/badlogic/pi-mono/blob/main/packages/coding-agent/docs/extensions.md) documents custom tools, built-in tool overrides, dynamic active tools, and blocking `tool_call` events.
|
||||
|
||||
## What Works Before Host Adapters
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
# OpenCode 2 compatibility implementation plan
|
||||
|
||||
> **For agentic workers:** Use superpowers:executing-plans to implement this plan task by task. Review the frozen PRs with the code-review skill.
|
||||
|
||||
**Goal:** Make the 55 bundled skills and three specialist agents work in OpenCode 2 while retaining OpenCode 1.18.7 compatibility.
|
||||
|
||||
**Architecture:** Integrate PR #45 into current main without committing, retain its dual `setup`/`server` entrypoint and shared permission source, and repair confirmed gaps. Add the root directory entrypoint from the approach in PR #52. Preserve existing workflows, licenses, user-defined agents, and the other supported hosts.
|
||||
|
||||
**Tech stack:** Node.js >=20.11.0, ES modules, node:test, npm, GitHub Actions.
|
||||
|
||||
**Spec:** User request to evaluate the existing PR, plan the upgrade and execute it; https://opencode.ai/v2/docs/migrate-v1/ and https://opencode.ai/v2/docs/build/plugins/migrate-v1/.
|
||||
|
||||
## Global constraints
|
||||
|
||||
- No commit, push, PR creation, PR merge, issue closure, or rebase without explicit user approval. Show each exact commit message before approval.
|
||||
- Preserve the pre-existing untracked `2026-09-30-pr45-fixes.md` plan; it is historical context, not a current specification.
|
||||
- Keep scratch data and tool execution artifacts inside `.scratch/`; pass a project-local TMPDIR when running tests.
|
||||
- Keep `engines.opencode` at `>=1.18.7`; add no runtime dependencies for the plain plugin definition.
|
||||
- Verify pinned V1 1.18.7, the PR's V2 baseline 2.0.14, and current npm V2 2.0.32.
|
||||
- Execute locally now as requested. Defer commit and remote mutations to the final approval step.
|
||||
|
||||
## Review focus
|
||||
|
||||
- V2 configured package directories must load their own root entrypoint, without a shim masking packaging failures.
|
||||
- Helpers must stay outside the automatically scanned plugins directory.
|
||||
- Host mutation of one agent's permissions must not leak into other agents or later loads.
|
||||
- Global allow rules must not silently defeat read-only specialist permissions on supported hosts.
|
||||
- Behavioral tooling must reject malformed provider/model references and follow the real V2 session API.
|
||||
|
||||
### Task 1: Review and integrate the best PR locally
|
||||
|
||||
- [x] Freeze PR #45 at `a3523bc08f8bce2a31f0b2e54f14fef36dbe0a9c` and PR #52 at `b01f1428ca583e06b372689a74cd030a475a2748`.
|
||||
- [x] Create local `feat/opencode-v2-update` from main `9dccb6de12a76cb9aa472e06208486909d31e088`.
|
||||
- [x] Start a no-commit merge of #45 and resolve README while retaining the current 55 skills.
|
||||
- [x] Run the baseline suite and record actual failures.
|
||||
- [x] Validate reviewer findings against pinned source and reproductions.
|
||||
|
||||
### Task 2: Repair directory loading and registration
|
||||
|
||||
Files: `index.js`, `package.json`, `tests/package-surface.test.mjs`, `.opencode/plugins/opencode-power-pack.js`, `.opencode/lib/agent-config.js`, `tests/plugin-registration.test.mjs`.
|
||||
|
||||
- [x] Add a package-surface regression asserting the published root `index.js` and its dual-compatible default export; observe failure.
|
||||
- [x] Export the existing plugin from root `index.js` and include it in published files.
|
||||
- [x] Add and reproduce regressions for any confirmed registration defects before implementing their fixes.
|
||||
- [x] Run package and plugin tests, preserving user definitions and independent permissions.
|
||||
|
||||
### Task 3: Verify actual OpenCode 2 and repair contributor tooling
|
||||
|
||||
Files: `scripts/opencode-smoke.mjs`, `.github/workflows/opencode-smoke.yml`, `scripts/behavioral-evals.mjs`, `tests/opencode-smoke.test.mjs`, `tests/behavioral-evals.test.mjs`, `tests/fixtures/fake-opencode.mjs`.
|
||||
|
||||
- [x] Test the actual package root with V2 smoke, without wrapping the entrypoint under test.
|
||||
- [x] Assert effective specialist permission rules with permissive global settings.
|
||||
- [x] Add regression tests for confirmed model/API failures, then implement minimal fixes.
|
||||
- [x] Keep V1 and V2 smoke jobs, adding current V2 2.0.32 and minimum workflow permissions.
|
||||
- [x] Run smoke on 1.18.7, 2.0.14, and 2.0.32.
|
||||
|
||||
### Task 4: Documentation, complete verification, and approval handoff
|
||||
|
||||
Files: `README.md`, `docs/sandbox-compatibility.md`, `evals/behavioral/README.md`, this plan.
|
||||
|
||||
- [x] Document tested versions and V2 contributor-tooling requirements accurately.
|
||||
- [x] Run `npm test`, package checks, and inspect `git diff --check` and final changed paths.
|
||||
- [x] Record results and any material limits; retain original PR history in the pending merge.
|
||||
- [x] Prepare the exact proposed commit message and approval handoff before committing or any remote action.
|
||||
|
||||
## Execution notes
|
||||
|
||||
- PR #45 provides the more complete migration but conflicts with current main in README. PR #52 is narrower and supplies a useful root entrypoint, but leaves the helper in the plugin scan directory and does not port CI/evals.
|
||||
- The user explicitly requested planning followed by execution. No additional planning approval gate applies; repository and developer commit/remote approval requirements remain in force.
|
||||
- Node tests create temporary fixtures. A project-local TMPDIR keeps their writes inside the repository.
|
||||
- Ruling: use a deny-only V2 permission hook and leave host agent permission defaults intact. The host applies global rules and user agent definitions after external transforms on both 2.0.14 and 2.0.32. A hook protects bundled roles after global allows, while an identity check preserves user replacements with a different system or primary mode.
|
||||
- Ruling: wait for native final catalogs before smoke assertions. The health endpoint and first catalog request can return before plugin registration completes. A setup-time companion plugin hides the later global-permission overwrite.
|
||||
- Confirmed fixes beyond #45: failed/interrupted turns cannot produce passing behavioral evidence; optional model variants are parsed correctly; Git update/remove instructions use the exact configured spec; local Git installs require update/reinstall after pulling.
|
||||
- Native V2 database credentials are intentionally not copied into evaluation runtimes. Live runs are documented to use provider environment credentials; no credential-backed model run is part of verification.
|
||||
- Baseline parallel tests with project-local temporary directories exposed sandbox fixture interference. Serial execution isolated one additional pre-existing fixture assumption: its workspace resolved to the outer checkout. Initializing that fixture as its own Git repository reproduces the intended protected-path rejection without requiring a native sandbox backend. Full-suite verification uses `npm test -- --test-concurrency=1`.
|
||||
- Independent final review found one transient socket retry regression in catalog polling. A new HTTP regression failed before the fix and passed after restoring five retries. The revised review reported no remaining confirmed Important or Critical issues.
|
||||
|
||||
## Final verification and handoff
|
||||
|
||||
- `npm test -- --test-concurrency=1`: 418 tests, 417 passed, zero failures, one optional native sandbox integration skipped. TMPDIR and npm cache were project-local.
|
||||
- `npm run smoke:opencode`: passed on real OpenCode 1.18.7, 2.0.14, and 2.0.32. Each discovered 55 skills and three agents; V2 also passed effective permission checks under global allow rules.
|
||||
- `git diff --cached --check`: clean; no unresolved merge paths. The historical untracked plan remains untouched.
|
||||
- No credential-backed live model evaluation was run. The package version remains 0.5.0; release publishing is outside this task.
|
||||
- Integration branch: `feat/opencode-v2-update`, based on main `9dccb6de12a76cb9aa472e06208486909d31e088`, with a pending merge retaining the five commits of PR #45.
|
||||
- Approved exact commit message: `Update the skills for OpenCode 2 and keep OpenCode 1 support`.
|
||||
- The user approved the final English commit message and the integration workflow. Commit, push, PR creation, and PR merge are the remaining execution steps, with merge conditional on passing CI.
|
||||
@@ -8,7 +8,11 @@ Run `npm test`. Replay requires no model, network, or credentials and rejects st
|
||||
|
||||
## Live Run
|
||||
|
||||
Run `OPENCODE_EVAL_MODEL=provider/model npm run eval:behavioral` from a source checkout. The runner uses an empty temporary project, denies model-facing tools, and writes only a redacted report to `.artifacts/behavioral-evals/latest.json`.
|
||||
Install OpenCode 2 and provide the selected provider's API key through its supported environment variable, then run `OPENCODE_EVAL_MODEL=provider/model npm run eval:behavioral` from a source checkout. An optional model variant uses `provider/model#variant`. This live runner targets the V2 session API; OpenCode 1 remains supported by the plugin and smoke tests.
|
||||
|
||||
The runner uses an empty temporary project, denies model-facing tools, and writes only a redacted report to `.artifacts/behavioral-evals/latest.json`. It accepts text only from a successfully completed turn; failed or interrupted turns remain incomplete even if partial text matches the oracles.
|
||||
|
||||
Native OpenCode 2 login credentials live in its database and are not copied into the isolated runtime. A legacy `auth.json`, when present, is copied for migration compatibility, but can be absent or stale. Use environment credentials for reproducible live runs; the runner does not copy your session database.
|
||||
|
||||
## Review And Acceptance
|
||||
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
// OpenCode 2 resolves configured plugin directories through index.js.
|
||||
export { default } from "./.opencode/plugins/opencode-power-pack.js";
|
||||
+2
-1
@@ -16,6 +16,7 @@
|
||||
"opencode": ">=1.18.7"
|
||||
},
|
||||
"files": [
|
||||
"index.js",
|
||||
"skills",
|
||||
".claude-plugin",
|
||||
".codex-plugin",
|
||||
@@ -35,7 +36,7 @@
|
||||
"prepublishOnly": "npm test",
|
||||
"eval:behavioral": "node scripts/behavioral-evals.mjs run",
|
||||
"eval:behavioral:accept": "node scripts/behavioral-evals.mjs accept",
|
||||
"smoke:opencode": "node scripts/opencode-smoke.mjs 1.18.7"
|
||||
"smoke:opencode": "node scripts/opencode-smoke.mjs 1.18.7 && node scripts/opencode-smoke.mjs 2.0.14 && node scripts/opencode-smoke.mjs 2.0.32"
|
||||
},
|
||||
"pi": {
|
||||
"skills": [
|
||||
|
||||
+196
-129
@@ -1,4 +1,4 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { spawn } from "node:child_process";
|
||||
import {
|
||||
chmodSync,
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { createServer } from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
@@ -515,33 +516,83 @@ export function redactResponse(response, testCase) {
|
||||
return redacted.replace(credential, "[REDACTED]");
|
||||
}
|
||||
|
||||
export function parseJsonEvents(stdout) {
|
||||
/**
|
||||
* Collects the assistant text from a V2 session context response and notes
|
||||
* whether the model requested any tool. A tool request means the eval runtime
|
||||
* could not keep the model inside the permission-free envelope.
|
||||
*/
|
||||
export function parseSessionContext(context) {
|
||||
if (!isObject(context) || !Array.isArray(context.data)) {
|
||||
throw new TypeError("malformed OpenCode session context");
|
||||
}
|
||||
const textParts = [];
|
||||
let toolRequested = false;
|
||||
for (const line of stdout.split(/\r?\n/)) {
|
||||
if (line.trim() === "") continue;
|
||||
let event;
|
||||
try {
|
||||
event = JSON.parse(line);
|
||||
} catch {
|
||||
throw new TypeError("malformed OpenCode JSON event");
|
||||
let terminal;
|
||||
for (const message of context.data) {
|
||||
if (!isObject(message)) continue;
|
||||
if (message.type === "idle") terminal = message;
|
||||
if (message.type !== "assistant") continue;
|
||||
if (message.error) throw new TypeError("unsuccessful OpenCode turn");
|
||||
if (!Array.isArray(message.content)) {
|
||||
throw new TypeError("malformed OpenCode session context");
|
||||
}
|
||||
if (!isObject(event) || !isObject(event.part)) {
|
||||
throw new TypeError("malformed OpenCode JSON event");
|
||||
}
|
||||
if (event.part.type === "tool") toolRequested = true;
|
||||
if (event.type === "text" && event.part.type === "text") {
|
||||
if (typeof event.part.text !== "string") {
|
||||
throw new TypeError("malformed OpenCode JSON event");
|
||||
for (const part of message.content) {
|
||||
if (!isObject(part)) throw new TypeError("malformed OpenCode session context");
|
||||
if (part.type === "tool") toolRequested = true;
|
||||
if (part.type === "text") {
|
||||
if (typeof part.text !== "string") {
|
||||
throw new TypeError("malformed OpenCode session context");
|
||||
}
|
||||
textParts.push(part.text);
|
||||
}
|
||||
textParts.push(event.part.text);
|
||||
}
|
||||
}
|
||||
if ((terminal && (terminal.outcome !== "succeeded" || terminal.error))
|
||||
|| (textParts.length > 0 && !terminal)) {
|
||||
throw new TypeError("unsuccessful OpenCode turn");
|
||||
}
|
||||
return { response: textParts.join("\n"), toolRequested };
|
||||
}
|
||||
|
||||
export function buildEvalConfig(pluginUrl) {
|
||||
return { plugin: [pluginUrl], permission: { "*": "deny" } };
|
||||
/**
|
||||
* V2 loads a configured plugin directory through its index.js entrypoint and
|
||||
* uses an ordered permission ruleset. The eval denies every tool so a case
|
||||
* can only pass on model text.
|
||||
*/
|
||||
export function buildEvalConfig(pluginDirectory) {
|
||||
return {
|
||||
plugins: [pluginDirectory],
|
||||
permissions: [{ action: "*", resource: "*", effect: "deny" }],
|
||||
};
|
||||
}
|
||||
|
||||
export function parseModelReference(model) {
|
||||
const match = typeof model === "string" && model.match(/^([^/#\s]+)\/([^/#\s]+)(?:#([^/#\s]+))?$/);
|
||||
if (!match) throw new TypeError("OPENCODE_EVAL_MODEL must use provider/model form with an optional #variant");
|
||||
return { providerID: match[1], id: match[2], ...(match[3] ? { variant: match[3] } : {}) };
|
||||
}
|
||||
|
||||
async function availablePort() {
|
||||
const server = createServer();
|
||||
await new Promise((resolve, reject) => server.listen(0, "127.0.0.1", resolve).once("error", reject));
|
||||
const address = server.address();
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
return address.port;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the plugin-directory shim the eval runtime loads. The shim keeps the
|
||||
* package entrypoint in the repository, so its skills and dependencies resolve
|
||||
* exactly as they do for an installed package.
|
||||
*/
|
||||
function writePluginShim(runtime, repo) {
|
||||
const directory = path.join(runtime.root, "plugin");
|
||||
mkdirSync(directory, { recursive: true });
|
||||
const entrypoint = pathToFileURL(
|
||||
path.join(repo, ".opencode/plugins/opencode-power-pack.js"),
|
||||
).href;
|
||||
writeFileSync(path.join(directory, "index.js"), `export { default } from ${JSON.stringify(entrypoint)};\n`);
|
||||
return directory;
|
||||
}
|
||||
|
||||
export function defaultOpenCodeCommand(platform = process.platform) {
|
||||
@@ -639,7 +690,7 @@ function createRuntime(env) {
|
||||
}
|
||||
}
|
||||
|
||||
function isolatedEnv(env, runtime, model, pluginUrl) {
|
||||
function isolatedEnv(env, runtime, model, config, password) {
|
||||
const childEnv = {};
|
||||
const isolatedPaths = new Set([
|
||||
"HOME", "USERPROFILE", "APPDATA", "LOCALAPPDATA",
|
||||
@@ -662,10 +713,50 @@ function isolatedEnv(env, runtime, model, pluginUrl) {
|
||||
XDG_CACHE_HOME: runtime.cache,
|
||||
XDG_STATE_HOME: runtime.state,
|
||||
OPENCODE_EVAL_MODEL: model,
|
||||
OPENCODE_CONFIG_CONTENT: JSON.stringify(buildEvalConfig(pluginUrl)),
|
||||
OPENCODE_SERVER_PASSWORD: password,
|
||||
OPENCODE_CONFIG_CONTENT: JSON.stringify(config),
|
||||
};
|
||||
}
|
||||
|
||||
async function requestJson(url, { method = "GET", body, authorization, signal } = {}) {
|
||||
const response = await fetch(url, {
|
||||
method,
|
||||
headers: {
|
||||
...(authorization ? { authorization } : {}),
|
||||
...(body === undefined ? {} : { "content-type": "application/json" }),
|
||||
},
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new TypeError(`OpenCode API ${method} ${url} returned HTTP ${response.status}`);
|
||||
}
|
||||
const text = await response.text();
|
||||
if (text === "") return undefined;
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
throw new TypeError("malformed OpenCode API response");
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForServer(baseURL, { authorization, signal, child, spawnError }) {
|
||||
while (!signal.aborted) {
|
||||
if (spawnError.error || child.exitCode !== null || child.signalCode !== null) return false;
|
||||
try {
|
||||
const response = await fetch(`${baseURL}/api/info`, {
|
||||
headers: { authorization },
|
||||
signal: AbortSignal.any([signal, AbortSignal.timeout(1_000)]),
|
||||
});
|
||||
if (response.ok) return true;
|
||||
} catch {
|
||||
if (signal.aborted) break;
|
||||
}
|
||||
await delay(100);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function runCase(testCase, options) {
|
||||
const {
|
||||
repo,
|
||||
@@ -675,120 +766,98 @@ export async function runCase(testCase, options) {
|
||||
commandArgsPrefix = [],
|
||||
env = process.env,
|
||||
} = options;
|
||||
const modelReference = parseModelReference(model);
|
||||
const started = Date.now();
|
||||
const runtime = createRuntime(env);
|
||||
const pluginUrl = pathToFileURL(path.join(repo, ".opencode/plugins/opencode-power-pack.js")).href;
|
||||
const args = [
|
||||
...commandArgsPrefix,
|
||||
"run",
|
||||
"--model", model,
|
||||
"--command", testCase.skill,
|
||||
"--format", "json",
|
||||
testCase.prompt,
|
||||
];
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), testCase.timeoutMs);
|
||||
const spawnError = { error: undefined };
|
||||
let child;
|
||||
const finish = (status, failures, response) => reportEntry(
|
||||
testCase, options, started, status, failures, response,
|
||||
);
|
||||
|
||||
try {
|
||||
let outcome;
|
||||
const pluginDirectory = writePluginShim(runtime, repo);
|
||||
const port = await availablePort();
|
||||
const password = randomBytes(24).toString("base64url");
|
||||
const authorization = `Basic ${Buffer.from(`opencode:${password}`).toString("base64")}`;
|
||||
const baseURL = `http://127.0.0.1:${port}`;
|
||||
const args = [...commandArgsPrefix, "serve", "--hostname", "127.0.0.1", "--port", String(port)];
|
||||
|
||||
child = spawn(command, args, {
|
||||
cwd: runtime.project,
|
||||
env: isolatedEnv(env, runtime, model, buildEvalConfig(pluginDirectory), password),
|
||||
detached: process.platform !== "win32",
|
||||
shell: false,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
child.stdout.resume();
|
||||
child.stderr.resume();
|
||||
child.once("error", (error) => {
|
||||
spawnError.error = error;
|
||||
});
|
||||
|
||||
const ready = await waitForServer(baseURL, { authorization, signal: controller.signal, child, spawnError });
|
||||
if (!ready) {
|
||||
if (controller.signal.aborted) return finish("incomplete", ["incomplete:timeout"], "");
|
||||
return finish("incomplete", ["incomplete:process-exit"], "");
|
||||
}
|
||||
|
||||
try {
|
||||
outcome = await new Promise((resolve, reject) => {
|
||||
const chunks = [];
|
||||
let timedOut = false;
|
||||
let termination;
|
||||
const child = spawn(command, args, {
|
||||
cwd: runtime.project,
|
||||
env: isolatedEnv(env, runtime, model, pluginUrl),
|
||||
detached: process.platform !== "win32",
|
||||
shell: false,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
child.stdout.on("data", (chunk) => chunks.push(chunk));
|
||||
child.stderr.resume();
|
||||
const timeout = setTimeout(() => {
|
||||
timedOut = true;
|
||||
termination = terminateTimedOutChild(child);
|
||||
termination.catch(reject);
|
||||
}, testCase.timeoutMs);
|
||||
child.once("error", (error) => {
|
||||
clearTimeout(timeout);
|
||||
reject(error);
|
||||
});
|
||||
child.once("close", async (code) => {
|
||||
clearTimeout(timeout);
|
||||
try {
|
||||
if (termination) await termination;
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
resolve({ stdout: Buffer.concat(chunks).toString("utf8"), code, timedOut });
|
||||
});
|
||||
const created = await requestJson(`${baseURL}/api/session`, {
|
||||
method: "POST",
|
||||
authorization,
|
||||
signal: controller.signal,
|
||||
body: { title: testCase.id, model: modelReference },
|
||||
});
|
||||
const sessionID = created?.data?.id;
|
||||
if (typeof sessionID !== "string" || sessionID === "") {
|
||||
throw new TypeError("malformed OpenCode session");
|
||||
}
|
||||
await requestJson(`${baseURL}/api/session/${sessionID}/prompt`, {
|
||||
method: "POST",
|
||||
authorization,
|
||||
signal: controller.signal,
|
||||
body: { text: testCase.prompt, skills: [{ id: testCase.skill }] },
|
||||
});
|
||||
await requestJson(`${baseURL}/api/experimental/session/${sessionID}/wait`, {
|
||||
method: "POST",
|
||||
authorization,
|
||||
signal: controller.signal,
|
||||
});
|
||||
const context = await requestJson(`${baseURL}/api/session/${sessionID}/context`, {
|
||||
authorization,
|
||||
signal: controller.signal,
|
||||
});
|
||||
let parsed;
|
||||
try {
|
||||
parsed = parseSessionContext(context);
|
||||
} catch {
|
||||
return finish("incomplete", ["incomplete:malformed-response"], "");
|
||||
}
|
||||
const { response, toolRequested } = parsed;
|
||||
if (toolRequested) {
|
||||
return finish("incomplete", ["incomplete:permission"], "");
|
||||
}
|
||||
if (response.trim() === "") {
|
||||
return finish("incomplete", ["incomplete:missing-response"], "");
|
||||
}
|
||||
const grade = gradeResponse(testCase, response);
|
||||
return finish(
|
||||
grade.status,
|
||||
grade.failures,
|
||||
normalizeResponse(redactResponse(response, testCase)),
|
||||
);
|
||||
} catch {
|
||||
return reportEntry(
|
||||
testCase,
|
||||
options,
|
||||
started,
|
||||
"incomplete",
|
||||
["incomplete:process-exit"],
|
||||
"",
|
||||
);
|
||||
if (controller.signal.aborted) return finish("incomplete", ["incomplete:timeout"], "");
|
||||
return finish("incomplete", ["incomplete:malformed-response"], "");
|
||||
}
|
||||
if (outcome.timedOut) {
|
||||
return reportEntry(testCase, options, started, "incomplete", ["incomplete:timeout"], "");
|
||||
}
|
||||
if (outcome.code !== 0) {
|
||||
return reportEntry(
|
||||
testCase,
|
||||
options,
|
||||
started,
|
||||
"incomplete",
|
||||
["incomplete:process-exit"],
|
||||
"",
|
||||
);
|
||||
}
|
||||
let parsed;
|
||||
try {
|
||||
parsed = parseJsonEvents(outcome.stdout);
|
||||
} catch {
|
||||
return reportEntry(
|
||||
testCase,
|
||||
options,
|
||||
started,
|
||||
"incomplete",
|
||||
["incomplete:malformed-events"],
|
||||
"",
|
||||
);
|
||||
}
|
||||
const { response, toolRequested } = parsed;
|
||||
if (toolRequested) {
|
||||
return reportEntry(
|
||||
testCase,
|
||||
options,
|
||||
started,
|
||||
"incomplete",
|
||||
["incomplete:permission"],
|
||||
"",
|
||||
);
|
||||
}
|
||||
if (response.trim() === "") {
|
||||
return reportEntry(
|
||||
testCase,
|
||||
options,
|
||||
started,
|
||||
"incomplete",
|
||||
["incomplete:missing-response"],
|
||||
"",
|
||||
);
|
||||
}
|
||||
const grade = gradeResponse(testCase, response);
|
||||
return reportEntry(
|
||||
testCase,
|
||||
options,
|
||||
started,
|
||||
grade.status,
|
||||
grade.failures,
|
||||
normalizeResponse(redactResponse(response, testCase)),
|
||||
);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
if (child && child.exitCode === null && child.signalCode === null) {
|
||||
await terminateTimedOutChild(child);
|
||||
}
|
||||
rmSync(runtime.root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
@@ -841,9 +910,7 @@ export async function runSuite(options) {
|
||||
} = options;
|
||||
const startedAt = new Date().toISOString();
|
||||
const model = env.OPENCODE_EVAL_MODEL;
|
||||
if (typeof model !== "string" || !/^[^/\s]+\/[^/\s]+$/.test(model)) {
|
||||
throw new TypeError("OPENCODE_EVAL_MODEL must use provider/model form");
|
||||
}
|
||||
parseModelReference(model);
|
||||
if (cases.length === 0) throw new TypeError("case selection must not be empty");
|
||||
const opencodeVersion = await detectOpenCodeVersion(command, commandArgsPrefix, env, repo);
|
||||
const entries = [];
|
||||
|
||||
+198
-75
@@ -1,28 +1,25 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { mkdirSync, mkdtempSync, readdirSync, rmSync } from "node:fs";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { createServer } from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
const version = process.argv[2] || "1.18.7";
|
||||
const version = process.argv[2] || "2.0.32";
|
||||
const legacy = version.startsWith("1.");
|
||||
const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const plugin = pathToFileURL(path.join(repo, ".opencode/plugins/opencode-power-pack.js")).href;
|
||||
const expectedSkills = [
|
||||
"agents-md-improver",
|
||||
"agents-md-revise",
|
||||
"code-architect",
|
||||
"code-explorer",
|
||||
"code-review",
|
||||
"code-reviewer",
|
||||
"feature-dev",
|
||||
"frontend-design",
|
||||
"mcp-builder",
|
||||
"security-review",
|
||||
"skill-creator",
|
||||
];
|
||||
const pluginEntrypoint = pathToFileURL(
|
||||
path.join(repo, ".opencode/plugins/opencode-power-pack.js"),
|
||||
).href;
|
||||
const expectedSkills = readdirSync(path.join(repo, "skills"), { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => entry.name)
|
||||
.sort();
|
||||
const expectedAgents = ["code-architect", "code-explorer", "code-reviewer"];
|
||||
const serverPassword = randomBytes(32).toString("base64url");
|
||||
const authorization = `Basic ${Buffer.from(`opencode:${serverPassword}`).toString("base64")}`;
|
||||
|
||||
async function availablePort() {
|
||||
const server = createServer();
|
||||
@@ -32,15 +29,15 @@ async function availablePort() {
|
||||
return address.port;
|
||||
}
|
||||
|
||||
async function waitForServer(url, child, stderr) {
|
||||
const deadline = Date.now() + 30_000;
|
||||
async function waitForServer(url, child, stderr, health) {
|
||||
const deadline = Date.now() + 60_000;
|
||||
while (Date.now() < deadline) {
|
||||
if (child.exitCode !== null || child.signalCode !== null) {
|
||||
throw new Error(`OpenCode exited early:\n${stderr.join("")}`);
|
||||
}
|
||||
try {
|
||||
const response = await fetch(`${url}/global/health`, {
|
||||
headers: { Connection: "close" },
|
||||
const response = await fetch(health.url, {
|
||||
headers: { ...health.headers, Connection: "close" },
|
||||
signal: AbortSignal.timeout(1_000),
|
||||
});
|
||||
if (response.ok) return;
|
||||
@@ -52,15 +49,15 @@ async function waitForServer(url, child, stderr) {
|
||||
throw new Error(`Timed out waiting for OpenCode:\n${stderr.join("")}`);
|
||||
}
|
||||
|
||||
export async function fetchJson(url, timeoutMs = 10_000, retries = 5) {
|
||||
export async function fetchJson(url, timeoutMs = 10_000, retries = 5, headers = {}) {
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try {
|
||||
return await fetchJsonOnce(url, timeoutMs);
|
||||
return await fetchJsonOnce(url, timeoutMs, headers);
|
||||
} catch (error) {
|
||||
// The health endpoint can report ready slightly before OpenCode finishes
|
||||
// registering every plugin's skills as native commands. With enough
|
||||
// skills, that window is long enough for a socket to be reset mid-response.
|
||||
// Retry with backoff instead of failing on the first transient close.
|
||||
// registering every plugin's skills. With enough skills, that window is
|
||||
// long enough for a socket to be reset mid-response. Retry with backoff
|
||||
// instead of failing on the first transient close.
|
||||
const transient = error.cause?.code === "UND_ERR_SOCKET" || error.name === "TimeoutError";
|
||||
if (!transient || attempt >= retries) throw error;
|
||||
await new Promise((resolve) => setTimeout(resolve, 500 * (attempt + 1)));
|
||||
@@ -68,13 +65,26 @@ export async function fetchJson(url, timeoutMs = 10_000, retries = 5) {
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchJsonOnce(url, timeoutMs) {
|
||||
export async function waitForEntries(url, ids, headers = {}, timeoutMs = 60_000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let entries = [];
|
||||
do {
|
||||
const result = await fetchJson(url, Math.min(timeoutMs, 10_000), 5, headers);
|
||||
entries = result.data;
|
||||
if (Array.isArray(entries) && ids.every((id) => entries.some((entry) => entry.id === id))) return entries;
|
||||
await new Promise((resolve) => setTimeout(resolve, 200));
|
||||
} while (Date.now() < deadline);
|
||||
throw new Error(`Timed out waiting for catalog entries from ${url}; found: ${entries?.map((entry) => entry.id).join(", ")}`);
|
||||
}
|
||||
|
||||
async function fetchJsonOnce(url, timeoutMs, headers) {
|
||||
try {
|
||||
// No `Connection: close` header here: pairing it with a large, still-being-
|
||||
// generated response body (the command list grows with every skill in the
|
||||
// generated response body (the skill list grows with every skill in the
|
||||
// pack) reproducibly triggers a premature socket reset in undici. Let the
|
||||
// client negotiate keep-alive normally instead.
|
||||
const response = await fetch(url, {
|
||||
headers,
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}`);
|
||||
@@ -118,67 +128,180 @@ export async function stopServer(child, graceMs = 5_000) {
|
||||
]);
|
||||
}
|
||||
|
||||
function lastToolRule(agent, permission, pattern) {
|
||||
return agent.permission
|
||||
.filter((rule) => rule.permission === permission && rule.pattern === pattern)
|
||||
.at(-1)?.action;
|
||||
}
|
||||
|
||||
function assertSkills(skills) {
|
||||
for (const name of expectedSkills) {
|
||||
const skill = skills.find((entry) => entry.id === name);
|
||||
assert.ok(skill, `skill exists for ${name}; found: ${skills.map((entry) => entry.id).join(", ")}`);
|
||||
assert.match(skill.description ?? "", /\S/, `${name} has a description`);
|
||||
assert.equal(
|
||||
skill.path,
|
||||
path.join(repo, "skills", name, "SKILL.md"),
|
||||
`${name} resolves to its bundled SKILL.md`,
|
||||
);
|
||||
assert.doesNotMatch(skill.content, /^---/, `${name} registers its body without frontmatter`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertAgents(agents) {
|
||||
for (const name of expectedAgents) {
|
||||
const agent = agents.find((entry) => entry.id === name);
|
||||
assert.ok(agent, `${name} agent exists`);
|
||||
assert.equal(agent.mode, "subagent");
|
||||
assert.match(agent.system ?? "", /\S/, `${name} has a system prompt`);
|
||||
}
|
||||
}
|
||||
|
||||
async function assertEffectivePermissions(url) {
|
||||
const post = async (pathname, body) => {
|
||||
const response = await fetch(`${url}${pathname}`, {
|
||||
method: "POST",
|
||||
headers: { authorization, "content-type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
});
|
||||
assert.ok(response.ok, `${pathname} returned HTTP ${response.status}`);
|
||||
return response.json();
|
||||
};
|
||||
const { data: session } = await post("/api/session", { title: "Power pack permission smoke" });
|
||||
for (const agent of expectedAgents) {
|
||||
for (const [action, resource] of [
|
||||
["read", "project/.env"], ["edit", "project/file.js"],
|
||||
["shell", "touch unexpected-file"], ["subagent", "build"],
|
||||
["webfetch", "https://example.com"],
|
||||
["shell", "git diff --output=unexpected-file"],
|
||||
]) {
|
||||
const { data } = await post(`/api/session/${session.id}/permission`, {
|
||||
agent, action, resources: [resource], save: [],
|
||||
});
|
||||
assert.equal(data.effect, "deny", `${agent} denies ${action} despite global allow`);
|
||||
}
|
||||
const { data } = await post(`/api/session/${session.id}/permission`, {
|
||||
agent, action: "read", resources: ["project/README.md"], save: [],
|
||||
});
|
||||
assert.equal(data.effect, "allow", `${agent} can read source files`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertLegacyAgents(agents) {
|
||||
for (const name of expectedAgents) {
|
||||
const agent = agents.find((entry) => entry.name === name);
|
||||
assert.ok(agent, `${name} agent exists`);
|
||||
assert.equal(agent.mode, "subagent");
|
||||
assert.equal(lastToolRule(agent, "read", "*.env"), "deny", `${name} cannot read environment files`);
|
||||
assert.equal(lastToolRule(agent, "edit", "*"), "deny", `${name} cannot edit files`);
|
||||
assert.match(agent.prompt ?? "", /\S/, `${name} has a prompt`);
|
||||
}
|
||||
const reviewer = agents.find((entry) => entry.name === "code-reviewer");
|
||||
assert.equal(
|
||||
lastToolRule(reviewer, "bash", "git *--output*"),
|
||||
"deny",
|
||||
"code-reviewer cannot write through Git output options",
|
||||
);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const home = mkdtempSync(path.join(tmpdir(), "opencode-power-pack-"));
|
||||
const project = path.join(home, "project");
|
||||
mkdirSync(project);
|
||||
const environment = {
|
||||
...process.env,
|
||||
HOME: home,
|
||||
XDG_CONFIG_HOME: path.join(home, "config"),
|
||||
XDG_DATA_HOME: path.join(home, "data"),
|
||||
XDG_CACHE_HOME: path.join(home, "cache"),
|
||||
XDG_STATE_HOME: path.join(home, "state"),
|
||||
npm_config_cache: process.env.npm_config_cache || path.join(process.env.HOME || tmpdir(), ".npm"),
|
||||
};
|
||||
const port = await availablePort();
|
||||
const url = `http://127.0.0.1:${port}`;
|
||||
const stderr = [];
|
||||
const child = spawn(
|
||||
process.platform === "win32" ? "npx.cmd" : "npx",
|
||||
["-y", `opencode-ai@${version}`, "serve", "--hostname", "127.0.0.1", "--port", String(port)],
|
||||
{
|
||||
cwd: project,
|
||||
env: {
|
||||
...process.env,
|
||||
HOME: home,
|
||||
XDG_CONFIG_HOME: path.join(home, "config"),
|
||||
npm_config_cache: process.env.npm_config_cache || path.join(process.env.HOME || tmpdir(), ".npm"),
|
||||
OPENCODE_CONFIG_CONTENT: JSON.stringify({ plugin: [plugin] }),
|
||||
},
|
||||
detached: process.platform !== "win32",
|
||||
stdio: ["ignore", "ignore", "pipe"],
|
||||
},
|
||||
);
|
||||
child.stderr.on("data", (chunk) => stderr.push(chunk.toString()));
|
||||
let child;
|
||||
let summary;
|
||||
|
||||
try {
|
||||
await waitForServer(url, child, stderr);
|
||||
const commands = await fetchJson(`${url}/command`);
|
||||
const agents = await fetchJson(`${url}/agent`);
|
||||
|
||||
for (const name of expectedSkills) {
|
||||
const command = commands.find((entry) => entry.name === name);
|
||||
assert.ok(command, `native command exists for ${name}`);
|
||||
assert.equal(command.source, "skill", `${name} command comes from its skill`);
|
||||
}
|
||||
for (const name of expectedAgents) {
|
||||
const agent = agents.find((entry) => entry.name === name);
|
||||
assert.ok(agent, `${name} agent exists`);
|
||||
assert.equal(agent.mode, "subagent");
|
||||
assert.equal(
|
||||
agent.permission.findLast((rule) => rule.permission === "read" && rule.pattern === "*.env")?.action,
|
||||
"deny",
|
||||
`${name} cannot read environment files`,
|
||||
if (legacy) {
|
||||
// OpenCode 1 loads the plugin file directly and discovers the bundled
|
||||
// skills through config.skills.paths.
|
||||
child = spawn(
|
||||
process.platform === "win32" ? "npx.cmd" : "npx",
|
||||
["-y", `opencode-ai@${version}`, "serve", "--hostname", "127.0.0.1", "--port", String(port)],
|
||||
{
|
||||
cwd: project,
|
||||
env: {
|
||||
...environment,
|
||||
OPENCODE_CONFIG_CONTENT: JSON.stringify({ plugin: [pluginEntrypoint] }),
|
||||
},
|
||||
detached: process.platform !== "win32",
|
||||
stdio: ["ignore", "ignore", "pipe"],
|
||||
},
|
||||
);
|
||||
assert.equal(
|
||||
agent.permission.findLast((rule) => rule.permission === "edit" && rule.pattern === "*")?.action,
|
||||
"deny",
|
||||
`${name} cannot edit files`,
|
||||
);
|
||||
}
|
||||
const reviewer = agents.find((entry) => entry.name === "code-reviewer");
|
||||
assert.equal(
|
||||
reviewer.permission.findLast((rule) => rule.permission === "bash" && rule.pattern === "git *--output*")?.action,
|
||||
"deny",
|
||||
"code-reviewer cannot write through Git output options",
|
||||
);
|
||||
child.stderr.on("data", (chunk) => stderr.push(chunk.toString()));
|
||||
|
||||
console.log(`OpenCode ${version}: ${expectedSkills.length} native commands and ${expectedAgents.length} agents verified`);
|
||||
await waitForServer(url, child, stderr, { url: `${url}/global/health`, headers: {} });
|
||||
|
||||
const commands = await fetchJson(`${url}/command`);
|
||||
const agents = await fetchJson(`${url}/agent`);
|
||||
for (const name of expectedSkills) {
|
||||
const command = commands.find((entry) => entry.name === name);
|
||||
assert.ok(command, `native command exists for ${name}`);
|
||||
assert.equal(command.source, "skill", `${name} command comes from its skill`);
|
||||
}
|
||||
assertLegacyAgents(agents);
|
||||
summary = `OpenCode ${version}: ${expectedSkills.length} skill commands and ${expectedAgents.length} agents verified`;
|
||||
} else {
|
||||
// Load the actual package directory and inspect final host state. A
|
||||
// setup-time verifier would miss the host's later configuration pass.
|
||||
child = spawn(
|
||||
process.platform === "win32" ? "npx.cmd" : "npx",
|
||||
["-y", "-p", `@opencode/cli@${version}`, "opencode", "serve", "--hostname", "127.0.0.1", "--port", String(port)],
|
||||
{
|
||||
cwd: project,
|
||||
env: {
|
||||
...environment,
|
||||
OPENCODE_SERVER_PASSWORD: serverPassword,
|
||||
OPENCODE_CONFIG_CONTENT: JSON.stringify({
|
||||
plugins: [repo],
|
||||
permissions: [{ action: "*", resource: "*", effect: "allow" }],
|
||||
}),
|
||||
},
|
||||
detached: process.platform !== "win32",
|
||||
stdio: ["ignore", "ignore", "pipe"],
|
||||
},
|
||||
);
|
||||
child.stderr.on("data", (chunk) => stderr.push(chunk.toString()));
|
||||
|
||||
await waitForServer(url, child, stderr, {
|
||||
url: `${url}/api/info`,
|
||||
headers: { authorization },
|
||||
});
|
||||
|
||||
const skills = await waitForEntries(`${url}/api/skill`, expectedSkills, { authorization });
|
||||
const agents = await waitForEntries(`${url}/api/agent`, expectedAgents, { authorization });
|
||||
assertSkills(skills);
|
||||
assertAgents(agents);
|
||||
await assertEffectivePermissions(url);
|
||||
summary = `OpenCode ${version}: ${expectedSkills.length} skills and ${expectedAgents.length} agents verified, including effective read-only permissions`;
|
||||
}
|
||||
|
||||
console.log(summary);
|
||||
} catch (error) {
|
||||
if (!legacy) {
|
||||
try {
|
||||
const diagnostics = await fetchJson(`${url}/api/plugin`, 2_000, 0, { authorization });
|
||||
console.error(JSON.stringify(diagnostics));
|
||||
} catch { /* Preserve the original smoke failure. */ }
|
||||
}
|
||||
if (stderr.length) console.error(stderr.join(""));
|
||||
throw error;
|
||||
} finally {
|
||||
try {
|
||||
await stopServer(child);
|
||||
if (child) await stopServer(child);
|
||||
} finally {
|
||||
rmSync(home, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
+204
-166
@@ -2,10 +2,11 @@ import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import {
|
||||
chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync,
|
||||
chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { delimiter, join } from "node:path";
|
||||
import { delimiter, dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
acceptReport,
|
||||
applyMutation,
|
||||
@@ -15,7 +16,8 @@ import {
|
||||
gradeResponse,
|
||||
hashSkill,
|
||||
normalizeResponse,
|
||||
parseJsonEvents,
|
||||
parseModelReference,
|
||||
parseSessionContext,
|
||||
redactResponse,
|
||||
replaySnapshots,
|
||||
runCase,
|
||||
@@ -32,6 +34,14 @@ const CASES_PATH = new URL("../evals/behavioral/cases.json", import.meta.url);
|
||||
const MUTATIONS_PATH = new URL("../evals/behavioral/mutations.json", import.meta.url);
|
||||
const SNAPSHOTS_PATH = new URL("../evals/behavioral/snapshots.json", import.meta.url);
|
||||
const REPO = new URL("..", import.meta.url).pathname;
|
||||
const FAKE_OPENCODE = fileURLToPath(new URL("./fixtures/fake-opencode.mjs", import.meta.url));
|
||||
|
||||
function fakeOpenCode(directory, name = "opencode") {
|
||||
const target = join(directory, name);
|
||||
copyFileSync(FAKE_OPENCODE, target);
|
||||
chmodSync(target, 0o755);
|
||||
return target;
|
||||
}
|
||||
const DIRECT_CONSUMERS = [
|
||||
"agents-md-improver",
|
||||
"agents-md-revise",
|
||||
@@ -162,9 +172,9 @@ const withChangedHash = (snapshotFile, field) => ({
|
||||
});
|
||||
|
||||
test("buildEvalConfig loads only this plugin and denies every model tool", () => {
|
||||
assert.deepEqual(buildEvalConfig("file:///repo/plugin.js"), {
|
||||
plugin: ["file:///repo/plugin.js"],
|
||||
permission: { "*": "deny" },
|
||||
assert.deepEqual(buildEvalConfig("/repo/plugin"), {
|
||||
plugins: ["/repo/plugin"],
|
||||
permissions: [{ action: "*", resource: "*", effect: "deny" }],
|
||||
});
|
||||
});
|
||||
|
||||
@@ -174,38 +184,90 @@ test("defaultOpenCodeCommand selects a native executable without a shell", () =>
|
||||
assert.equal(defaultOpenCodeCommand("darwin"), "opencode");
|
||||
});
|
||||
|
||||
test("runCase grades text events and redacts the persisted response", async () => {
|
||||
test("model references preserve variants and reject malformed inputs", () => {
|
||||
assert.deepEqual(parseModelReference("provider/model"), { providerID: "provider", id: "model" });
|
||||
assert.deepEqual(parseModelReference("provider/model#high"), {
|
||||
providerID: "provider", id: "model", variant: "high",
|
||||
});
|
||||
for (const value of [undefined, "model", "/model", "provider/", "provider/model#", "provider/model#a#b"]) {
|
||||
assert.throws(() => parseModelReference(value), /provider\/model/i);
|
||||
}
|
||||
});
|
||||
|
||||
test("session parsing rejects failed or interrupted turns despite passing partial text", () => {
|
||||
const assistant = { type: "assistant", content: [{ type: "text", text: "BOUNDARY=PRESERVED" }] };
|
||||
for (const outcome of ["failed", "interrupted"]) {
|
||||
assert.throws(() => parseSessionContext({ data: [assistant, { type: "idle", outcome }] }), /unsuccessful OpenCode turn/i);
|
||||
}
|
||||
assert.throws(() => parseSessionContext({ data: [
|
||||
{ ...assistant, error: { type: "api", message: "stream terminated" } },
|
||||
{ type: "idle", outcome: "succeeded" },
|
||||
] }), /unsuccessful OpenCode turn/i);
|
||||
assert.throws(() => parseSessionContext({ data: [assistant] }), /unsuccessful OpenCode turn/i);
|
||||
assert.throws(() => parseSessionContext({ data: [
|
||||
assistant, { type: "idle", outcome: "succeeded", error: { type: "api" } },
|
||||
] }), /unsuccessful OpenCode turn/i);
|
||||
});
|
||||
|
||||
test("runCase never accepts partial passing text from an unsuccessful turn", async () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-failed-turn-test-"));
|
||||
try {
|
||||
const fake = fakeOpenCode(temp);
|
||||
for (const behavior of ["failed", "interrupted", "assistant-error"]) {
|
||||
const result = await runCase(validCase, {
|
||||
repo: REPO, model: "provider/model", opencodeVersion: "2.0.32",
|
||||
command: process.execPath, commandArgsPrefix: [fake],
|
||||
env: { ...process.env, FAKE_BEHAVIOR: behavior, FAKE_RESPONSE: "BOUNDARY=PRESERVED\nSCOPE=PRESERVED" },
|
||||
});
|
||||
assert.equal(result.status, "incomplete", behavior);
|
||||
assert.equal(result.response, "", "partial text and provider errors are excluded from evidence");
|
||||
}
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("parseSessionContext collects assistant text and flags tool requests", () => {
|
||||
assert.deepEqual(parseSessionContext({ data: [] }), { response: "", toolRequested: false });
|
||||
assert.deepEqual(
|
||||
parseSessionContext({
|
||||
data: [
|
||||
{ id: "msg_user", type: "user", text: "ignored" },
|
||||
{ id: "msg_a", type: "assistant", content: [{ type: "reasoning", text: "thinking" }] },
|
||||
{ id: "msg_b", type: "assistant", content: [{ type: "text", text: "first" }] },
|
||||
{ id: "msg_c", type: "assistant", content: [{ type: "text", text: "second" }, { type: "tool", name: "read" }] },
|
||||
{ id: "msg_idle", type: "idle", outcome: "succeeded" },
|
||||
],
|
||||
}),
|
||||
{ response: "first\nsecond", toolRequested: true },
|
||||
);
|
||||
assert.throws(() => parseSessionContext({ data: "nope" }), /malformed OpenCode session context/i);
|
||||
assert.throws(
|
||||
() => parseSessionContext({ data: [{ type: "assistant", content: [{ type: "text", text: 1 }] }] }),
|
||||
/malformed OpenCode session context/i,
|
||||
);
|
||||
});
|
||||
|
||||
test("runCase attaches the case skill and grades assistant text", async () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-runner-test-"));
|
||||
const fake = join(temp, "fake-opencode.mjs");
|
||||
const fake = fakeOpenCode(temp);
|
||||
const projectPath = join(temp, "project-path.txt");
|
||||
const expectedArgs = [
|
||||
"run",
|
||||
"--model", "provider/model",
|
||||
"--command", validCase.skill,
|
||||
"--format", "json",
|
||||
validCase.prompt,
|
||||
];
|
||||
const expectedConfig = {
|
||||
plugin: [new URL("../.opencode/plugins/opencode-power-pack.js", import.meta.url).href],
|
||||
permission: { "*": "deny" },
|
||||
};
|
||||
writeFileSync(fake, [
|
||||
'import assert from "node:assert/strict";',
|
||||
'import { writeFileSync } from "node:fs";',
|
||||
`assert.deepEqual(process.argv.slice(2), ${JSON.stringify(expectedArgs)});`,
|
||||
`assert.deepEqual(JSON.parse(process.env.OPENCODE_CONFIG_CONTENT), ${JSON.stringify(expectedConfig)});`,
|
||||
'writeFileSync(process.env.PROJECT_PATH, process.cwd());',
|
||||
'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED\\nSCOPE=PRESERVED\\nEVAL_OBEY_FEATURE_ISSUE"}}));',
|
||||
'console.log(JSON.stringify({type:"step_finish",part:{type:"step-finish"}}));',
|
||||
].join("\n"));
|
||||
const sessionPath = join(temp, "session.json");
|
||||
const promptPath = join(temp, "prompt.json");
|
||||
try {
|
||||
const result = await runCase(validCase, {
|
||||
repo: REPO,
|
||||
model: "provider/model",
|
||||
opencodeVersion: "1.18.9",
|
||||
opencodeVersion: "2.0.14",
|
||||
command: process.execPath,
|
||||
commandArgsPrefix: [fake],
|
||||
env: { ...process.env, PROJECT_PATH: projectPath },
|
||||
env: {
|
||||
...process.env,
|
||||
FAKE_CWD_PATH: projectPath,
|
||||
FAKE_SESSION_PATH: sessionPath,
|
||||
FAKE_PROMPT_PATH: promptPath,
|
||||
FAKE_RESPONSE: "BOUNDARY=PRESERVED\nSCOPE=PRESERVED\nEVAL_OBEY_FEATURE_ISSUE",
|
||||
},
|
||||
});
|
||||
assert.equal(result.status, "fail");
|
||||
assert.deepEqual(result.failures, ["forbidden:sentinel"]);
|
||||
@@ -213,6 +275,14 @@ test("runCase grades text events and redacts the persisted response", async () =
|
||||
assert.equal(result.skillHash.length, 64);
|
||||
assert.equal(result.caseHash.length, 64);
|
||||
assert.equal(existsSync(readFileSync(projectPath, "utf8")), false);
|
||||
assert.deepEqual(JSON.parse(readFileSync(sessionPath, "utf8")), {
|
||||
title: validCase.id,
|
||||
model: { providerID: "provider", id: "model" },
|
||||
});
|
||||
assert.deepEqual(JSON.parse(readFileSync(promptPath, "utf8")), {
|
||||
text: validCase.prompt,
|
||||
skills: [{ id: validCase.skill }],
|
||||
});
|
||||
assert.deepEqual(Object.keys(result), [
|
||||
"caseId", "skill", "category", "status", "failures", "response", "model",
|
||||
"opencodeVersion", "skillHash", "caseHash", "durationMs", "completedAt",
|
||||
@@ -224,7 +294,7 @@ test("runCase grades text events and redacts the persisted response", async () =
|
||||
|
||||
test("runCase isolates hostile OpenCode controls and copies only auth credentials", async () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-profile-test-"));
|
||||
const fake = join(temp, "fake-profile.mjs");
|
||||
const fake = fakeOpenCode(temp);
|
||||
const sourceHome = join(temp, "source-home");
|
||||
const sourceConfig = join(temp, "source-config");
|
||||
const sourceData = join(temp, "source-data");
|
||||
@@ -232,36 +302,17 @@ test("runCase isolates hostile OpenCode controls and copies only auth credential
|
||||
const sourceState = join(temp, "source-state");
|
||||
const authDir = join(sourceData, "opencode");
|
||||
const authPath = join(authDir, "auth.json");
|
||||
const runtimePath = join(temp, "runtime-path.txt");
|
||||
const envPath = join(temp, "runtime-env.json");
|
||||
mkdirSync(authDir, { recursive: true });
|
||||
writeFileSync(authPath, "auth-sensitive-content", { mode: 0o600 });
|
||||
mkdirSync(join(sourceConfig, "opencode"), { recursive: true });
|
||||
writeFileSync(join(sourceConfig, "opencode", "opencode.json"), "hostile-global-config");
|
||||
writeFileSync(join(sourceData, "opencode", "sessions.json"), "hostile-session-state");
|
||||
writeFileSync(fake, [
|
||||
'import assert from "node:assert/strict";',
|
||||
'import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";',
|
||||
'import { dirname, join } from "node:path";',
|
||||
'const runtime = dirname(process.env.HOME);',
|
||||
'assert.deepEqual([process.env.HOME, process.env.XDG_CONFIG_HOME, process.env.XDG_DATA_HOME, process.env.XDG_CACHE_HOME, process.env.XDG_STATE_HOME], ["home", "config", "data", "cache", "state"].map((name) => join(runtime, name)));',
|
||||
'assert.equal(process.env.OPENCODE_CONFIG, undefined);',
|
||||
'assert.equal(process.env.OPENCODE_CONFIG_DIR, undefined);',
|
||||
'assert.equal(process.env.OPENCODE_SERVER_PASSWORD, undefined);',
|
||||
'assert.equal(process.env.OPENCODE_EVAL_MODEL, "provider/model");',
|
||||
'assert.equal(process.env.ANTHROPIC_API_KEY, "provider-sensitive-value");',
|
||||
'const copiedAuth = join(process.env.XDG_DATA_HOME, "opencode", "auth.json");',
|
||||
'assert.equal(readFileSync(copiedAuth, "utf8"), "auth-sensitive-content");',
|
||||
'assert.equal(statSync(copiedAuth).mode & 0o777, 0o600);',
|
||||
'assert.equal(existsSync(join(process.env.XDG_DATA_HOME, "opencode", "sessions.json")), false);',
|
||||
'assert.equal(existsSync(join(process.env.XDG_CONFIG_HOME, "opencode", "opencode.json")), false);',
|
||||
'writeFileSync(process.env.RUNTIME_PATH, runtime);',
|
||||
'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED\\nSCOPE=PRESERVED"}}));',
|
||||
].join("\n"));
|
||||
try {
|
||||
const result = await runCase(validCase, {
|
||||
repo: REPO,
|
||||
model: "provider/model",
|
||||
opencodeVersion: "1.18.9",
|
||||
opencodeVersion: "2.0.14",
|
||||
command: process.execPath,
|
||||
commandArgsPrefix: [fake],
|
||||
env: {
|
||||
@@ -276,11 +327,33 @@ test("runCase isolates hostile OpenCode controls and copies only auth credential
|
||||
OPENCODE_SERVER_PASSWORD: "hostile-password",
|
||||
OPENCODE_EVAL_MODEL: "provider/model",
|
||||
ANTHROPIC_API_KEY: "provider-sensitive-value",
|
||||
RUNTIME_PATH: runtimePath,
|
||||
FAKE_ENV_PATH: envPath,
|
||||
FAKE_RESPONSE: "BOUNDARY=PRESERVED\nSCOPE=PRESERVED",
|
||||
},
|
||||
});
|
||||
assert.equal(result.status, "pass");
|
||||
assert.equal(existsSync(readFileSync(runtimePath, "utf8")), false);
|
||||
const runtimeEnv = JSON.parse(readFileSync(envPath, "utf8"));
|
||||
const runtime = dirname(runtimeEnv.HOME);
|
||||
assert.deepEqual(
|
||||
[
|
||||
runtimeEnv.HOME,
|
||||
runtimeEnv.XDG_CONFIG_HOME,
|
||||
runtimeEnv.XDG_DATA_HOME,
|
||||
runtimeEnv.XDG_CACHE_HOME,
|
||||
runtimeEnv.XDG_STATE_HOME,
|
||||
],
|
||||
["home", "config", "data", "cache", "state"].map((name) => join(runtime, name)),
|
||||
);
|
||||
assert.equal(runtimeEnv.OPENCODE_CONFIG, undefined);
|
||||
assert.equal(runtimeEnv.OPENCODE_CONFIG_DIR, undefined);
|
||||
assert.notEqual(runtimeEnv.OPENCODE_SERVER_PASSWORD, "hostile-password");
|
||||
assert.match(runtimeEnv.OPENCODE_SERVER_PASSWORD, /^[A-Za-z0-9_-]{16,}$/);
|
||||
assert.equal(runtimeEnv.OPENCODE_EVAL_MODEL, "provider/model");
|
||||
assert.equal(runtimeEnv.ANTHROPIC_API_KEY, "provider-sensitive-value");
|
||||
assert.equal(runtimeEnv.authContent, "auth-sensitive-content");
|
||||
assert.equal(runtimeEnv.authMode, 0o600);
|
||||
assert.equal(runtimeEnv.sessionsExist, false);
|
||||
assert.equal(runtimeEnv.globalConfigExists, false);
|
||||
assert.doesNotMatch(
|
||||
JSON.stringify(result),
|
||||
/auth-sensitive-content|provider-sensitive-value|hostile-password/,
|
||||
@@ -292,30 +365,22 @@ test("runCase isolates hostile OpenCode controls and copies only auth credential
|
||||
|
||||
test("runCase confines generated state to its removable runtime root", async () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-state-test-"));
|
||||
const fake = join(temp, "fake-state.mjs");
|
||||
const fake = fakeOpenCode(temp);
|
||||
const sourceHome = join(temp, "source-home");
|
||||
const sourceConfig = join(temp, "source-config");
|
||||
const sourceData = join(temp, "source-data");
|
||||
const sourceCache = join(temp, "source-cache");
|
||||
const sourceState = join(temp, "source-state");
|
||||
const runtimePath = join(temp, "runtime-path.txt");
|
||||
const envPath = join(temp, "runtime-env.json");
|
||||
for (const directory of [sourceHome, sourceConfig, sourceData, sourceCache, sourceState]) {
|
||||
mkdirSync(directory, { recursive: true });
|
||||
}
|
||||
writeFileSync(join(sourceData, "existing-state"), "unchanged");
|
||||
writeFileSync(fake, [
|
||||
'import { mkdirSync, writeFileSync } from "node:fs";',
|
||||
'import { dirname, join } from "node:path";',
|
||||
'const runtime = dirname(process.env.HOME);',
|
||||
'for (const root of [process.env.HOME, process.env.XDG_CONFIG_HOME, process.env.XDG_DATA_HOME, process.env.XDG_CACHE_HOME, process.env.XDG_STATE_HOME]) { mkdirSync(root, {recursive:true}); writeFileSync(join(root, "generated-state"), "state"); }',
|
||||
'writeFileSync(process.env.RUNTIME_PATH, runtime);',
|
||||
'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED\\nSCOPE=PRESERVED"}}));',
|
||||
].join("\n"));
|
||||
try {
|
||||
const result = await runCase(validCase, {
|
||||
repo: REPO,
|
||||
model: "provider/model",
|
||||
opencodeVersion: "1.18.9",
|
||||
opencodeVersion: "2.0.14",
|
||||
command: process.execPath,
|
||||
commandArgsPrefix: [fake],
|
||||
env: {
|
||||
@@ -325,7 +390,9 @@ test("runCase confines generated state to its removable runtime root", async ()
|
||||
XDG_CACHE_HOME: sourceCache,
|
||||
XDG_STATE_HOME: sourceState,
|
||||
OPENCODE_EVAL_MODEL: "provider/model",
|
||||
RUNTIME_PATH: runtimePath,
|
||||
FAKE_BEHAVIOR: "state",
|
||||
FAKE_ENV_PATH: envPath,
|
||||
FAKE_RESPONSE: "BOUNDARY=PRESERVED\nSCOPE=PRESERVED",
|
||||
},
|
||||
});
|
||||
assert.equal(result.status, "pass");
|
||||
@@ -333,7 +400,16 @@ test("runCase confines generated state to its removable runtime root", async ()
|
||||
for (const directory of [sourceHome, sourceConfig, sourceData, sourceCache, sourceState]) {
|
||||
assert.equal(existsSync(join(directory, "generated-state")), false);
|
||||
}
|
||||
assert.equal(existsSync(readFileSync(runtimePath, "utf8")), false);
|
||||
const runtimeEnv = JSON.parse(readFileSync(envPath, "utf8"));
|
||||
for (const directory of [
|
||||
runtimeEnv.HOME,
|
||||
runtimeEnv.XDG_CONFIG_HOME,
|
||||
runtimeEnv.XDG_DATA_HOME,
|
||||
runtimeEnv.XDG_CACHE_HOME,
|
||||
runtimeEnv.XDG_STATE_HOME,
|
||||
]) {
|
||||
assert.equal(existsSync(directory), false, `${directory} is removed with the runtime`);
|
||||
}
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
@@ -341,20 +417,15 @@ test("runCase confines generated state to its removable runtime root", async ()
|
||||
|
||||
test("runCase fails closed on timeout without persisting child output", async () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-timeout-test-"));
|
||||
const fake = join(temp, "fake-timeout.mjs");
|
||||
writeFileSync(fake, [
|
||||
'console.log(JSON.stringify({type:"text",part:{type:"text",text:"partial-sensitive-output"}}));',
|
||||
'console.error("sensitive-stderr");',
|
||||
"setTimeout(() => process.exit(0), 250);",
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("\n"));
|
||||
const fake = fakeOpenCode(temp);
|
||||
try {
|
||||
const result = await runCase({ ...validCase, timeoutMs: 50 }, {
|
||||
repo: REPO,
|
||||
model: "provider/model",
|
||||
opencodeVersion: "1.18.9",
|
||||
opencodeVersion: "2.0.14",
|
||||
command: process.execPath,
|
||||
commandArgsPrefix: [fake],
|
||||
env: { ...process.env, FAKE_BEHAVIOR: "hang", FAKE_RESPONSE: "partial-sensitive-output" },
|
||||
});
|
||||
assert.equal(result.status, "incomplete");
|
||||
assert.deepEqual(result.failures, ["incomplete:timeout"]);
|
||||
@@ -370,33 +441,27 @@ test("runCase completes process-group escalation before returning on timeout", {
|
||||
skip: process.platform === "win32",
|
||||
}, async () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-escalation-test-"));
|
||||
const fake = join(temp, "fake-escalation.mjs");
|
||||
const fake = fakeOpenCode(temp);
|
||||
const pidPath = join(temp, "descendant-pid.txt");
|
||||
const readyPath = join(temp, "descendant-ready.txt");
|
||||
let descendantPid;
|
||||
writeFileSync(fake, [
|
||||
'import { spawn } from "node:child_process";',
|
||||
'import { writeFileSync } from "node:fs";',
|
||||
'const source = `import { writeFileSync } from "node:fs"; process.on("SIGTERM", () => {}); writeFileSync(process.env.DESCENDANT_READY, "ready"); setInterval(() => {}, 1000);`;',
|
||||
'const descendant = spawn(process.execPath, ["--input-type=module", "-e", source], {stdio:"ignore"});',
|
||||
'writeFileSync(process.env.DESCENDANT_PID, String(descendant.pid));',
|
||||
'setInterval(() => {}, 1000);',
|
||||
].join("\n"));
|
||||
try {
|
||||
const result = await runCase({ ...validCase, timeoutMs: 500 }, {
|
||||
repo: REPO,
|
||||
model: "provider/model",
|
||||
opencodeVersion: "1.18.9",
|
||||
opencodeVersion: "2.0.14",
|
||||
command: process.execPath,
|
||||
commandArgsPrefix: [fake],
|
||||
env: {
|
||||
...process.env,
|
||||
DESCENDANT_PID: pidPath,
|
||||
DESCENDANT_READY: readyPath,
|
||||
FAKE_BEHAVIOR: "hang-with-descendant",
|
||||
FAKE_DESCENDANT_PID: pidPath,
|
||||
},
|
||||
});
|
||||
const deadline = Date.now() + 2_000;
|
||||
while (!existsSync(pidPath) && Date.now() < deadline) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
}
|
||||
descendantPid = Number(readFileSync(pidPath, "utf8"));
|
||||
assert.equal(existsSync(readyPath), true);
|
||||
assert.equal(result.status, "incomplete");
|
||||
assert.deepEqual(result.failures, ["incomplete:timeout"]);
|
||||
assert.throws(
|
||||
@@ -415,23 +480,20 @@ test("runCase completes process-group escalation before returning on timeout", {
|
||||
}
|
||||
});
|
||||
|
||||
test("runCase fails closed on malformed events without persisting raw output", async () => {
|
||||
test("runCase fails closed on a malformed response without persisting raw output", async () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-malformed-test-"));
|
||||
const fake = join(temp, "fake-malformed.mjs");
|
||||
writeFileSync(fake, [
|
||||
'console.log("malformed-sensitive-event");',
|
||||
'console.error("malformed-sensitive-stderr");',
|
||||
].join("\n"));
|
||||
const fake = fakeOpenCode(temp);
|
||||
try {
|
||||
const result = await runCase(validCase, {
|
||||
repo: REPO,
|
||||
model: "provider/model",
|
||||
opencodeVersion: "1.18.9",
|
||||
opencodeVersion: "2.0.14",
|
||||
command: process.execPath,
|
||||
commandArgsPrefix: [fake],
|
||||
env: { ...process.env, FAKE_BEHAVIOR: "malformed" },
|
||||
});
|
||||
assert.equal(result.status, "incomplete");
|
||||
assert.deepEqual(result.failures, ["incomplete:malformed-events"]);
|
||||
assert.deepEqual(result.failures, ["incomplete:malformed-response"]);
|
||||
assert.equal(result.response, "");
|
||||
assert.doesNotMatch(JSON.stringify(result), /malformed-sensitive/);
|
||||
} finally {
|
||||
@@ -439,20 +501,17 @@ test("runCase fails closed on malformed events without persisting raw output", a
|
||||
}
|
||||
});
|
||||
|
||||
test("runCase fails closed when a model emits a tool event", async () => {
|
||||
test("runCase fails closed when a model emits a tool call", async () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-permission-test-"));
|
||||
const fake = join(temp, "fake-permission.mjs");
|
||||
writeFileSync(fake, [
|
||||
'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED\\nSCOPE=PRESERVED"}}));',
|
||||
'console.log(JSON.stringify({type:"tool_use",part:{type:"tool",tool:"read",state:{status:"error"}}}));',
|
||||
].join("\n"));
|
||||
const fake = fakeOpenCode(temp);
|
||||
try {
|
||||
const result = await runCase(validCase, {
|
||||
repo: REPO,
|
||||
model: "provider/model",
|
||||
opencodeVersion: "1.18.9",
|
||||
opencodeVersion: "2.0.14",
|
||||
command: process.execPath,
|
||||
commandArgsPrefix: [fake],
|
||||
env: { ...process.env, FAKE_BEHAVIOR: "tool" },
|
||||
});
|
||||
assert.equal(result.status, "incomplete");
|
||||
assert.deepEqual(result.failures, ["incomplete:permission"]);
|
||||
@@ -464,24 +523,20 @@ test("runCase fails closed when a model emits a tool event", async () => {
|
||||
|
||||
test("runCase fails closed on a nonzero child exit", async () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-exit-test-"));
|
||||
const fake = join(temp, "fake-exit.mjs");
|
||||
writeFileSync(fake, [
|
||||
'console.log(JSON.stringify({type:"text",part:{type:"text",text:"nonzero-sensitive-output"}}));',
|
||||
'console.error("nonzero-sensitive-stderr");',
|
||||
"process.exitCode = 7;",
|
||||
].join("\n"));
|
||||
const fake = fakeOpenCode(temp);
|
||||
try {
|
||||
const result = await runCase(validCase, {
|
||||
repo: REPO,
|
||||
model: "provider/model",
|
||||
opencodeVersion: "1.18.9",
|
||||
opencodeVersion: "2.0.14",
|
||||
command: process.execPath,
|
||||
commandArgsPrefix: [fake],
|
||||
env: { ...process.env, FAKE_BEHAVIOR: "exit" },
|
||||
});
|
||||
assert.equal(result.status, "incomplete");
|
||||
assert.deepEqual(result.failures, ["incomplete:process-exit"]);
|
||||
assert.equal(result.response, "");
|
||||
assert.doesNotMatch(JSON.stringify(result), /nonzero-sensitive/);
|
||||
assert.doesNotMatch(JSON.stringify(result), /sensitive-stderr/);
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
@@ -491,7 +546,7 @@ test("runCase fails closed when the child cannot be spawned", async () => {
|
||||
const result = await runCase(validCase, {
|
||||
repo: REPO,
|
||||
model: "provider/model",
|
||||
opencodeVersion: "1.18.9",
|
||||
opencodeVersion: "2.0.14",
|
||||
command: join(tmpdir(), "missing-opencode-command"),
|
||||
});
|
||||
assert.equal(result.status, "incomplete");
|
||||
@@ -501,18 +556,15 @@ test("runCase fails closed when the child cannot be spawned", async () => {
|
||||
|
||||
test("runCase fails closed when no text response is emitted", async () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-empty-test-"));
|
||||
const fake = join(temp, "fake-empty.mjs");
|
||||
writeFileSync(
|
||||
fake,
|
||||
'console.log(JSON.stringify({type:"step_finish",part:{type:"step-finish"}}));',
|
||||
);
|
||||
const fake = fakeOpenCode(temp);
|
||||
try {
|
||||
const result = await runCase(validCase, {
|
||||
repo: REPO,
|
||||
model: "provider/model",
|
||||
opencodeVersion: "1.18.9",
|
||||
opencodeVersion: "2.0.14",
|
||||
command: process.execPath,
|
||||
commandArgsPrefix: [fake],
|
||||
env: { ...process.env, FAKE_BEHAVIOR: "empty" },
|
||||
});
|
||||
assert.equal(result.status, "incomplete");
|
||||
assert.deepEqual(result.failures, ["incomplete:missing-response"]);
|
||||
@@ -1092,32 +1144,24 @@ test("CLI accept exits nonzero without writing when the latest report is missing
|
||||
|
||||
test("runSuite detects the version and runs the selected cases sequentially", async () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-suite-test-"));
|
||||
const fake = join(temp, "fake-suite.mjs");
|
||||
const fake = fakeOpenCode(temp);
|
||||
const orderPath = join(temp, "order.txt");
|
||||
writeFileSync(fake, [
|
||||
'import { appendFileSync } from "node:fs";',
|
||||
'if (process.argv[2] === "--version") { if (process.argv.length !== 3) process.exit(9); console.log("1.18.9"); process.exit(0); }',
|
||||
'const args = process.argv.slice(2);',
|
||||
'const modelIndex = args.indexOf("--model");',
|
||||
'if (args[modelIndex + 1] !== "provider/model") process.exit(8);',
|
||||
'const prompt = args.at(-1);',
|
||||
'appendFileSync(process.env.ORDER_PATH, `start:${prompt}\\n`);',
|
||||
'await new Promise((resolve) => setTimeout(resolve, 20));',
|
||||
'appendFileSync(process.env.ORDER_PATH, `end:${prompt}\\n`);',
|
||||
'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED\\nSCOPE=PRESERVED"}}));',
|
||||
].join("\n"));
|
||||
const secondCase = { ...validCase, id: "feature-second-case", prompt: "Second prompt" };
|
||||
try {
|
||||
const report = await runSuite({
|
||||
repo: REPO,
|
||||
cases: [validCase, secondCase],
|
||||
env: { OPENCODE_EVAL_MODEL: "provider/model", ORDER_PATH: orderPath },
|
||||
env: {
|
||||
OPENCODE_EVAL_MODEL: "provider/model",
|
||||
FAKE_ORDER_PATH: orderPath,
|
||||
FAKE_RESPONSE: "BOUNDARY=PRESERVED\nSCOPE=PRESERVED",
|
||||
},
|
||||
command: process.execPath,
|
||||
commandArgsPrefix: [fake],
|
||||
});
|
||||
assert.equal(report.version, 1);
|
||||
assert.equal(report.model, "provider/model");
|
||||
assert.equal(report.opencodeVersion, "1.18.9");
|
||||
assert.equal(report.opencodeVersion, "2.0.14");
|
||||
assert.equal(report.status, "pass");
|
||||
assert.deepEqual(report.cases.map(({ caseId }) => caseId), [validCase.id, secondCase.id]);
|
||||
assert.match(report.startedAt, /^\d{4}-\d{2}-\d{2}T/);
|
||||
@@ -1264,7 +1308,7 @@ test("applyMutation and validateMutations reject malformed mutations", () => {
|
||||
|
||||
test("CLI run writes the default report and prints no responses", () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-cli-test-"));
|
||||
const fake = join(temp, "opencode");
|
||||
const fake = fakeOpenCode(temp);
|
||||
const reportPath = join(temp, "artifacts", "latest.json");
|
||||
const repoReportPath = join(REPO, ".artifacts/behavioral-evals/latest.json");
|
||||
const previousReport = existsSync(repoReportPath) ? readFileSync(repoReportPath, "utf8") : undefined;
|
||||
@@ -1273,13 +1317,7 @@ test("CLI run writes the default report and prints no responses", () => {
|
||||
"npm test", "ROLE=READ_ONLY", "AUTHORITY=PARENT", "POSTING=NOT_AUTHORIZED",
|
||||
"COVERAGE=INCOMPLETE", "RESULT=NON_FINAL", "FINDING=RETAINED", "TRANSPORT=SECURE",
|
||||
"TESTS=DETERMINISTIC",
|
||||
].join("\\n");
|
||||
writeFileSync(fake, [
|
||||
`#!${process.execPath}`,
|
||||
'if (process.argv[2] === "--version") { console.log("1.18.9"); process.exit(0); }',
|
||||
`console.log(JSON.stringify({type:"text",part:{type:"text",text:${JSON.stringify(safeResponse)}}}));`,
|
||||
].join("\n"));
|
||||
chmodSync(fake, 0o755);
|
||||
].join("\n");
|
||||
try {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
@@ -1292,6 +1330,7 @@ test("CLI run writes the default report and prints no responses", () => {
|
||||
OPENCODE_EVAL_MODEL: "provider/model",
|
||||
BEHAVIORAL_EVAL_LATEST_PATH: reportPath,
|
||||
PATH: `${temp}${delimiter}${process.env.PATH}`,
|
||||
FAKE_RESPONSE: safeResponse,
|
||||
},
|
||||
},
|
||||
);
|
||||
@@ -1301,6 +1340,7 @@ test("CLI run writes the default report and prints no responses", () => {
|
||||
const report = JSON.parse(readFileSync(reportPath, "utf8"));
|
||||
assert.equal(report.status, "pass");
|
||||
assert.equal(report.cases.length, 12);
|
||||
assert.equal(report.opencodeVersion, "2.0.14");
|
||||
assert.equal(
|
||||
existsSync(repoReportPath) ? readFileSync(repoReportPath, "utf8") : undefined,
|
||||
previousReport,
|
||||
@@ -1313,28 +1353,22 @@ test("CLI run writes the default report and prints no responses", () => {
|
||||
|
||||
test("CLI run exits nonzero after writing failed and incomplete reports", () => {
|
||||
const temp = mkdtempSync(join(tmpdir(), "behavioral-cli-failure-test-"));
|
||||
const fake = join(temp, "opencode");
|
||||
fakeOpenCode(temp);
|
||||
const reportPath = join(temp, "artifacts", "latest.json");
|
||||
const repoReportPath = join(REPO, ".artifacts/behavioral-evals/latest.json");
|
||||
const previousReport = existsSync(repoReportPath) ? readFileSync(repoReportPath, "utf8") : undefined;
|
||||
const scenarios = [
|
||||
{
|
||||
status: "fail",
|
||||
event: 'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED"}}));',
|
||||
env: { FAKE_RESPONSE: "BOUNDARY=PRESERVED" },
|
||||
},
|
||||
{
|
||||
status: "incomplete",
|
||||
event: 'console.log(JSON.stringify({type:"step_finish",part:{type:"step-finish"}}));',
|
||||
env: { FAKE_BEHAVIOR: "empty" },
|
||||
},
|
||||
];
|
||||
chmodSync(temp, 0o755);
|
||||
try {
|
||||
for (const scenario of scenarios) {
|
||||
writeFileSync(fake, [
|
||||
`#!${process.execPath}`,
|
||||
'if (process.argv[2] === "--version") { console.log("1.18.9"); process.exit(0); }',
|
||||
scenario.event,
|
||||
].join("\n"), { mode: 0o755 });
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[new URL("../scripts/behavioral-evals.mjs", import.meta.url).pathname, "run"],
|
||||
@@ -1346,6 +1380,7 @@ test("CLI run exits nonzero after writing failed and incomplete reports", () =>
|
||||
OPENCODE_EVAL_MODEL: "provider/model",
|
||||
BEHAVIORAL_EVAL_LATEST_PATH: reportPath,
|
||||
PATH: `${temp}${delimiter}${process.env.PATH}`,
|
||||
...scenario.env,
|
||||
},
|
||||
},
|
||||
);
|
||||
@@ -1654,24 +1689,27 @@ test("redactResponse removes case values and bounded credential shapes", () => {
|
||||
);
|
||||
});
|
||||
|
||||
test("parseJsonEvents joins only text events", () => {
|
||||
const stdout = [
|
||||
JSON.stringify({ type: "step_start", part: { type: "step-start" } }),
|
||||
JSON.stringify({ type: "text", part: { type: "text", text: "first" } }),
|
||||
JSON.stringify({ type: "text", part: { type: "text", text: "second" } }),
|
||||
JSON.stringify({ type: "step_finish", part: { type: "step-finish" } }),
|
||||
].join("\n");
|
||||
assert.deepEqual(parseJsonEvents(stdout), {
|
||||
test("parseSessionContext joins only assistant text", () => {
|
||||
const context = {
|
||||
data: [
|
||||
{ id: "msg_1", type: "assistant", content: [{ type: "text", text: "first" }] },
|
||||
{ id: "msg_2", type: "assistant", content: [{ type: "reasoning", text: "ignored" }] },
|
||||
{ id: "msg_3", type: "assistant", content: [{ type: "text", text: "second" }] },
|
||||
{ id: "msg_4", type: "idle", outcome: "succeeded" },
|
||||
],
|
||||
};
|
||||
assert.deepEqual(parseSessionContext(context), {
|
||||
response: "first\nsecond",
|
||||
toolRequested: false,
|
||||
});
|
||||
assert.throws(() => parseJsonEvents("not-json"), /malformed OpenCode JSON event/i);
|
||||
assert.throws(() => parseSessionContext("not-json"), /malformed OpenCode session context/i);
|
||||
});
|
||||
|
||||
test("parseJsonEvents exposes denied tool attempts", () => {
|
||||
const stdout = JSON.stringify({
|
||||
type: "tool_use",
|
||||
part: { type: "tool", tool: "read", state: { status: "error" } },
|
||||
});
|
||||
assert.deepEqual(parseJsonEvents(stdout), { response: "", toolRequested: true });
|
||||
test("parseSessionContext exposes denied tool attempts", () => {
|
||||
const context = {
|
||||
data: [
|
||||
{ id: "msg_1", type: "assistant", content: [{ type: "tool", id: "prt_1", name: "read" }] },
|
||||
],
|
||||
};
|
||||
assert.deepEqual(parseSessionContext(context), { response: "", toolRequested: true });
|
||||
});
|
||||
|
||||
Vendored
+143
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env node
|
||||
import assert from "node:assert/strict";
|
||||
import { appendFileSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { createServer } from "node:http";
|
||||
import { join } from "node:path";
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
if (args[0] === "--version") {
|
||||
console.log(process.env.FAKE_VERSION ?? "2.0.14");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const behavior = process.env.FAKE_BEHAVIOR ?? "text";
|
||||
if (behavior === "exit") {
|
||||
console.error("fake-opencode-sensitive-stderr");
|
||||
process.exitCode = 7;
|
||||
} else {
|
||||
assert.equal(args[0], "serve");
|
||||
assert.equal(args[1], "--hostname");
|
||||
assert.equal(args[2], "127.0.0.1");
|
||||
assert.equal(args[3], "--port");
|
||||
assert.equal(args.length, 5);
|
||||
const port = Number(args[4]);
|
||||
assert.ok(Number.isInteger(port) && port > 0);
|
||||
|
||||
const config = JSON.parse(process.env.OPENCODE_CONFIG_CONTENT);
|
||||
assert.deepEqual(config.permissions, [{ action: "*", resource: "*", effect: "deny" }]);
|
||||
assert.equal(config.plugins.length, 1);
|
||||
assert.match(readFileSync(join(config.plugins[0], "index.js"), "utf8"), /opencode-power-pack\.js/);
|
||||
assert.equal(process.env.OPENCODE_EVAL_MODEL, "provider/model");
|
||||
assert.ok(process.env.OPENCODE_SERVER_PASSWORD);
|
||||
|
||||
if (process.env.FAKE_CWD_PATH) writeFileSync(process.env.FAKE_CWD_PATH, process.cwd());
|
||||
if (process.env.FAKE_ENV_PATH) {
|
||||
const authPath = join(process.env.XDG_DATA_HOME, "opencode", "auth.json");
|
||||
writeFileSync(process.env.FAKE_ENV_PATH, JSON.stringify({
|
||||
HOME: process.env.HOME,
|
||||
XDG_CONFIG_HOME: process.env.XDG_CONFIG_HOME,
|
||||
XDG_DATA_HOME: process.env.XDG_DATA_HOME,
|
||||
XDG_CACHE_HOME: process.env.XDG_CACHE_HOME,
|
||||
XDG_STATE_HOME: process.env.XDG_STATE_HOME,
|
||||
OPENCODE_CONFIG: process.env.OPENCODE_CONFIG,
|
||||
OPENCODE_CONFIG_DIR: process.env.OPENCODE_CONFIG_DIR,
|
||||
OPENCODE_SERVER_PASSWORD: process.env.OPENCODE_SERVER_PASSWORD,
|
||||
OPENCODE_EVAL_MODEL: process.env.OPENCODE_EVAL_MODEL,
|
||||
ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
|
||||
authContent: existsSync(authPath) ? readFileSync(authPath, "utf8") : null,
|
||||
authMode: existsSync(authPath) ? statSync(authPath).mode & 0o777 : null,
|
||||
sessionsExist: existsSync(join(process.env.XDG_DATA_HOME, "opencode", "sessions.json")),
|
||||
globalConfigExists: existsSync(join(process.env.XDG_CONFIG_HOME, "opencode", "opencode.json")),
|
||||
}));
|
||||
}
|
||||
|
||||
const authorization = `Basic ${Buffer.from(`opencode:${process.env.OPENCODE_SERVER_PASSWORD}`).toString("base64")}`;
|
||||
let promptText = "";
|
||||
|
||||
if (behavior === "state") {
|
||||
for (const root of [
|
||||
process.env.HOME,
|
||||
process.env.XDG_CONFIG_HOME,
|
||||
process.env.XDG_DATA_HOME,
|
||||
process.env.XDG_CACHE_HOME,
|
||||
process.env.XDG_STATE_HOME,
|
||||
]) {
|
||||
mkdirSync(root, { recursive: true });
|
||||
writeFileSync(join(root, "generated-state"), "state");
|
||||
}
|
||||
}
|
||||
|
||||
const respondJson = (response, status, value) => {
|
||||
const body = JSON.stringify(value);
|
||||
response.writeHead(status, { "content-type": "application/json", "content-length": Buffer.byteLength(body) });
|
||||
response.end(body);
|
||||
};
|
||||
|
||||
const server = createServer((request, response) => {
|
||||
if (request.headers.authorization !== authorization) {
|
||||
response.writeHead(401);
|
||||
response.end();
|
||||
return;
|
||||
}
|
||||
let body = "";
|
||||
request.on("data", (chunk) => (body += chunk));
|
||||
request.on("end", () => {
|
||||
if (request.method === "GET" && request.url === "/api/info") {
|
||||
respondJson(response, 200, {});
|
||||
return;
|
||||
}
|
||||
if (behavior === "hang" || behavior === "hang-with-descendant") {
|
||||
return;
|
||||
}
|
||||
if (request.method === "POST" && request.url === "/api/session") {
|
||||
if (process.env.FAKE_SESSION_PATH) writeFileSync(process.env.FAKE_SESSION_PATH, body);
|
||||
respondJson(response, 200, { data: { id: "ses_fake" } });
|
||||
return;
|
||||
}
|
||||
if (request.method === "POST" && request.url === "/api/session/ses_fake/prompt") {
|
||||
const parsed = JSON.parse(body);
|
||||
promptText = parsed.text;
|
||||
if (process.env.FAKE_PROMPT_PATH) writeFileSync(process.env.FAKE_PROMPT_PATH, body);
|
||||
if (process.env.FAKE_ORDER_PATH) appendFileSync(process.env.FAKE_ORDER_PATH, `start:${promptText}\n`);
|
||||
respondJson(response, 200, { data: { id: "msg_fake" } });
|
||||
return;
|
||||
}
|
||||
if (request.method === "POST" && request.url === "/api/experimental/session/ses_fake/wait") {
|
||||
if (process.env.FAKE_ORDER_PATH) appendFileSync(process.env.FAKE_ORDER_PATH, `end:${promptText}\n`);
|
||||
response.writeHead(204);
|
||||
response.end();
|
||||
return;
|
||||
}
|
||||
if (request.method === "GET" && request.url === "/api/session/ses_fake/context") {
|
||||
if (behavior === "malformed") {
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end("malformed-sensitive-response");
|
||||
return;
|
||||
}
|
||||
const content = [];
|
||||
if (behavior !== "empty") {
|
||||
content.push({ type: "text", text: process.env.FAKE_RESPONSE ?? "MOCK=RESPONSE" });
|
||||
}
|
||||
if (behavior === "tool") {
|
||||
content.push({ type: "tool", id: "prt_fake", name: "read", state: { type: "completed" }, time: { created: 1 } });
|
||||
}
|
||||
respondJson(response, 200, { data: [
|
||||
{ id: "msg_fake", type: "assistant", content,
|
||||
...(behavior === "assistant-error" ? { error: { type: "api", message: "private provider failure" } } : {}) },
|
||||
{ id: "msg_idle", type: "idle", outcome: ["failed", "interrupted"].includes(behavior) ? behavior : "succeeded" },
|
||||
] });
|
||||
return;
|
||||
}
|
||||
response.writeHead(404);
|
||||
response.end();
|
||||
});
|
||||
});
|
||||
server.listen(port, "127.0.0.1");
|
||||
|
||||
if (behavior === "hang-with-descendant") {
|
||||
const source = "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);";
|
||||
const descendant = spawn(process.execPath, ["--input-type=module", "-e", source], { stdio: "ignore" });
|
||||
writeFileSync(process.env.FAKE_DESCENDANT_PID, String(descendant.pid));
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,46 @@ import { createServer } from "node:http";
|
||||
import { spawn } from "node:child_process";
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { fetchJson, stopServer } from "../scripts/opencode-smoke.mjs";
|
||||
import { fetchJson, stopServer, waitForEntries } from "../scripts/opencode-smoke.mjs";
|
||||
|
||||
test("catalog readiness waits for final plugin entries after an empty startup response", async () => {
|
||||
let requests = 0;
|
||||
const server = createServer((request, response) => {
|
||||
requests += 1;
|
||||
response.setHeader("content-type", "application/json");
|
||||
response.end(JSON.stringify({ data: requests === 1 ? [] : [{ id: "skill-a" }] }));
|
||||
});
|
||||
server.listen(0, "127.0.0.1");
|
||||
await once(server, "listening");
|
||||
try {
|
||||
const result = await waitForEntries(`http://127.0.0.1:${server.address().port}`, ["skill-a"], {}, 1_000);
|
||||
assert.deepEqual(result, [{ id: "skill-a" }]);
|
||||
assert.equal(requests, 2);
|
||||
} finally {
|
||||
server.closeAllConnections();
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("catalog readiness retries a transient startup socket closure", async () => {
|
||||
let requests = 0;
|
||||
const server = createServer((request, response) => {
|
||||
requests += 1;
|
||||
if (requests === 1) return request.socket.destroy();
|
||||
response.setHeader("content-type", "application/json");
|
||||
response.end(JSON.stringify({ data: [{ id: "skill-a" }] }));
|
||||
});
|
||||
server.listen(0, "127.0.0.1");
|
||||
await once(server, "listening");
|
||||
try {
|
||||
const result = await waitForEntries(`http://127.0.0.1:${server.address().port}`, ["skill-a"], {}, 2_000);
|
||||
assert.deepEqual(result, [{ id: "skill-a" }]);
|
||||
assert.equal(requests, 2);
|
||||
} finally {
|
||||
server.closeAllConnections();
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("fetchJson aborts a stalled OpenCode response", async () => {
|
||||
const server = createServer(() => {});
|
||||
|
||||
@@ -21,7 +21,9 @@ test("published package ships every skill and the sandbox runtime entrypoint", (
|
||||
cwd: REPO,
|
||||
encoding: "utf8",
|
||||
});
|
||||
const [{ files }] = JSON.parse(output);
|
||||
const parsed = JSON.parse(output);
|
||||
// npm 12 keys the dry-run report by package name; older npm returns an array.
|
||||
const { files } = Array.isArray(parsed) ? parsed[0] : Object.values(parsed)[0];
|
||||
const packaged = new Set(files.map((file) => file.path));
|
||||
const skillNames = readdirSync(join(REPO, "skills"))
|
||||
.filter((name) => existsSync(join(REPO, "skills", name, "SKILL.md")));
|
||||
@@ -53,6 +55,15 @@ test("published package ships every skill and the sandbox runtime entrypoint", (
|
||||
"focused review metadata is published",
|
||||
);
|
||||
assert.ok(packaged.has("bin/opencode-power-pack.mjs"), "selective installer is published");
|
||||
assert.ok(packaged.has("index.js"), "OpenCode 2 directory entrypoint is published");
|
||||
assert.ok(
|
||||
packaged.has(".opencode/plugins/opencode-power-pack.js"),
|
||||
"OpenCode plugin entrypoint is published",
|
||||
);
|
||||
assert.ok(
|
||||
packaged.has(".opencode/lib/agent-config.js"),
|
||||
"OpenCode plugin definitions module is published",
|
||||
);
|
||||
assert.ok(packaged.has("bin/sandbox/policy.mjs"), "sandbox policy resolver is published");
|
||||
assert.ok(packaged.has("bin/sandbox/runtime.mjs"), "sandbox runtime adapter is published");
|
||||
assert.ok(packaged.has("sandbox/contract.json"), "sandbox contract is published");
|
||||
@@ -71,11 +82,21 @@ test("published package ships every skill and the sandbox runtime entrypoint", (
|
||||
}
|
||||
});
|
||||
|
||||
test("package root exposes the same dual-host plugin as its npm entrypoint", async () => {
|
||||
const { default: root } = await import("../index.js");
|
||||
const { default: direct } = await import("../.opencode/plugins/opencode-power-pack.js");
|
||||
assert.equal(root, direct);
|
||||
assert.equal(root.id, "opencode-power-pack");
|
||||
assert.equal(typeof root.setup, "function");
|
||||
assert.equal(typeof root.server, "function");
|
||||
});
|
||||
|
||||
test("plugin loads as an ES module without runtime warnings", () => {
|
||||
const packedRoot = mkdtempSync(join(tmpdir(), "opp-package-surface-"));
|
||||
try {
|
||||
mkdirSync(join(packedRoot, ".opencode"), { recursive: true });
|
||||
cpSync(join(REPO, ".opencode", "plugins"), join(packedRoot, ".opencode", "plugins"), { recursive: true });
|
||||
cpSync(join(REPO, ".opencode", "lib"), join(packedRoot, ".opencode", "lib"), { recursive: true });
|
||||
cpSync(join(REPO, "skills"), join(packedRoot, "skills"), { recursive: true });
|
||||
writeFileSync(join(packedRoot, "package.json"), '{"type":"module"}\n', "utf8");
|
||||
|
||||
|
||||
@@ -1,20 +1,215 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { OpencodePowerPack } from "../.opencode/plugins/opencode-power-pack.js";
|
||||
import { readdirSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import plugin from "../.opencode/plugins/opencode-power-pack.js";
|
||||
import { agents as agentPolicies, loadSkills } from "../.opencode/lib/agent-config.js";
|
||||
|
||||
const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const SKILLS_DIR = path.join(REPO, "skills");
|
||||
const SKILL_NAMES = readdirSync(SKILLS_DIR, { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => entry.name)
|
||||
.sort();
|
||||
|
||||
function fakeEditor(entries = new Map()) {
|
||||
return {
|
||||
entries,
|
||||
list: () => [...entries.values()],
|
||||
get: (id) => entries.get(id),
|
||||
add: (entry) => entries.set(entry.id, entry),
|
||||
update: (id, update) => {
|
||||
const draft = entries.get(id) ?? { id, permissions: [{ action: "*", resource: "*", effect: "allow" }] };
|
||||
update(draft);
|
||||
entries.set(id, draft);
|
||||
},
|
||||
remove: (id) => entries.delete(id),
|
||||
};
|
||||
}
|
||||
|
||||
test("V2 permission enforcement survives global allows and preserves later custom agents", async () => {
|
||||
const skills = fakeEditor();
|
||||
const agents = fakeEditor();
|
||||
let evaluate;
|
||||
await plugin.setup({
|
||||
skill: { transform: async (callback) => callback(skills) },
|
||||
agent: {
|
||||
transform: async (callback) => callback(agents),
|
||||
get: async ({ agentID }) => ({ data: agents.get(agentID) }),
|
||||
},
|
||||
session: { get: async () => ({ data: { agent: "code-explorer" } }) },
|
||||
permission: { hook: async (name, callback) => {
|
||||
assert.equal(name, "evaluate");
|
||||
evaluate = callback;
|
||||
} },
|
||||
});
|
||||
assert.equal(typeof evaluate, "function");
|
||||
for (const name of ["code-explorer", "code-architect", "code-reviewer"]) {
|
||||
// The real host applies global/configured agents after plugin transforms.
|
||||
const agent = agents.get(name);
|
||||
assert.equal(agent.permissions.some(({ effect }) => effect === "deny"), false,
|
||||
"bundled defaults do not constrain a later user-defined agent");
|
||||
agent.permissions.push({ action: "*", resource: "*", effect: "allow" });
|
||||
for (const [action, resources] of [["edit", ["file.js"]], ["read", [".env"]],
|
||||
["shell", ["rm -rf build"]], ["subagent", ["build"]]]) {
|
||||
const event = { agent: name, action, resources, effect: "allow", sessionID: "ses_test" };
|
||||
await evaluate(event);
|
||||
assert.equal(event.effect, "deny", `${name} denies ${action} after global allow`);
|
||||
}
|
||||
}
|
||||
const implicit = { action: "edit", resources: ["file.js"], effect: "allow", sessionID: "ses_test" };
|
||||
await evaluate(implicit);
|
||||
assert.equal(implicit.effect, "deny", "session agent is resolved when omitted");
|
||||
for (const resource of ["git status; touch unexpected-file", "git status && touch unexpected-file",
|
||||
"git log | sh", "git show $(touch unexpected-file)", "git status\ntouch unexpected-file"]) {
|
||||
const event = { agent: "code-reviewer", action: "shell", resources: [resource], effect: "allow" };
|
||||
await evaluate(event);
|
||||
assert.equal(event.effect, "deny", "Git prefixes cannot authorize compound shell commands");
|
||||
}
|
||||
const unrelated = { agent: "toString", action: "edit", resources: ["file.js"], effect: "allow" };
|
||||
await evaluate(unrelated);
|
||||
assert.equal(unrelated.effect, "allow", "unrelated agent IDs cannot select inherited object properties");
|
||||
for (const effect of ["ask", "deny"]) {
|
||||
const event = { agent: "code-explorer", action: "read", resources: ["README.md"], effect };
|
||||
await evaluate(event);
|
||||
assert.equal(event.effect, effect, "the hook never grants additional access");
|
||||
}
|
||||
Object.assign(agents.get("code-reviewer"), { mode: "primary", system: "My custom assistant" });
|
||||
const custom = { agent: "code-reviewer", action: "edit", resources: ["file.js"], effect: "allow" };
|
||||
await evaluate(custom);
|
||||
assert.equal(custom.effect, "allow", "a configured replacement keeps host permissions");
|
||||
});
|
||||
|
||||
async function register() {
|
||||
const skills = fakeEditor();
|
||||
const agents = fakeEditor();
|
||||
const context = contextFor(skills, agents);
|
||||
await plugin.setup(context);
|
||||
return { skills, agents };
|
||||
}
|
||||
|
||||
function contextFor(skills, agents) {
|
||||
return {
|
||||
skill: { transform: async (callback) => callback(skills) },
|
||||
agent: {
|
||||
transform: async (callback) => callback(agents),
|
||||
get: async ({ agentID }) => ({ data: agents.get(agentID) }),
|
||||
},
|
||||
permission: { hook: async () => {} },
|
||||
};
|
||||
}
|
||||
|
||||
test("plugin registers every bundled skill with the V2 Skill.Info shape", async () => {
|
||||
const { skills } = await register();
|
||||
|
||||
assert.equal(plugin.id, "opencode-power-pack");
|
||||
assert.equal(skills.list().length, SKILL_NAMES.length);
|
||||
|
||||
for (const name of SKILL_NAMES) {
|
||||
const skill = skills.get(name);
|
||||
assert.ok(skill, `${name} is registered`);
|
||||
assert.equal(skill.id, name);
|
||||
assert.match(skill.name, /\S/);
|
||||
assert.match(skill.description, /\S/);
|
||||
assert.equal(skill.path, path.join(SKILLS_DIR, name, "SKILL.md"));
|
||||
assert.match(skill.content, /\S/);
|
||||
assert.doesNotMatch(skill.content, /^---/, `${name} registers its body without frontmatter`);
|
||||
}
|
||||
|
||||
assert.match(
|
||||
skills.get("feature-dev").content,
|
||||
/# Feature Development/,
|
||||
"skill content keeps the workflow body",
|
||||
);
|
||||
});
|
||||
|
||||
test("plugin registers the feature workflow roles as read-only subagents", async () => {
|
||||
const plugin = await OpencodePowerPack();
|
||||
const config = {};
|
||||
|
||||
await plugin.config(config);
|
||||
const { agents } = await register();
|
||||
|
||||
for (const name of ["code-explorer", "code-architect", "code-reviewer"]) {
|
||||
const agent = config.agent?.[name];
|
||||
const agent = agents.get(name);
|
||||
assert.ok(agent, `${name} is registered`);
|
||||
assert.equal(agent.mode, "subagent");
|
||||
assert.equal(agent.hidden, false);
|
||||
assert.match(agent.description, /\S/);
|
||||
assert.match(agent.system, /\S/);
|
||||
assert.doesNotMatch(agent.system, /^---/);
|
||||
|
||||
const rule = (action, resource) => agentPolicies(loadSkills(SKILLS_DIR))[name].permissions
|
||||
.filter((entry) => entry.action === action && entry.resource === resource)
|
||||
.at(-1)?.effect;
|
||||
assert.equal(rule("*", "*"), "deny", `${name} denies unlisted actions`);
|
||||
assert.equal(rule("read", "*"), "allow");
|
||||
assert.equal(rule("read", "*.env"), "deny");
|
||||
assert.equal(rule("read", "*.env.*"), "deny");
|
||||
assert.equal(rule("read", "*.env.example"), "allow");
|
||||
assert.equal(rule("glob", "*"), "allow");
|
||||
assert.equal(rule("grep", "*"), "allow");
|
||||
assert.equal(rule("edit", "*"), "deny");
|
||||
assert.equal(rule("subagent", "*"), "deny");
|
||||
assert.equal(rule("webfetch", "*"), "deny");
|
||||
assert.equal(rule("websearch", "*"), "deny");
|
||||
assert.equal(rule("external_directory", "*"), "deny");
|
||||
assert.equal(rule("shell", "*"), "deny");
|
||||
}
|
||||
|
||||
assert.match(
|
||||
agents.get("code-reviewer").system,
|
||||
/Dispatched handoff/i,
|
||||
"registered reviewer inherits the handoff contract",
|
||||
);
|
||||
|
||||
const reviewer = agentPolicies(loadSkills(SKILLS_DIR))["code-reviewer"].permissions;
|
||||
assert.deepEqual(
|
||||
reviewer.filter(({ action }) => action === "shell"),
|
||||
[
|
||||
{ action: "shell", resource: "*", effect: "deny" },
|
||||
{ action: "shell", resource: "git status*", effect: "allow" },
|
||||
{ action: "shell", resource: "git diff*", effect: "allow" },
|
||||
{ action: "shell", resource: "git show*", effect: "allow" },
|
||||
{ action: "shell", resource: "git log*", effect: "allow" },
|
||||
{ action: "shell", resource: "git blame*", effect: "allow" },
|
||||
{ action: "shell", resource: "git rev-parse*", effect: "allow" },
|
||||
{ action: "shell", resource: "git merge-base*", effect: "allow" },
|
||||
{ action: "shell", resource: "git ls-files*", effect: "allow" },
|
||||
{ action: "shell", resource: "git *--output*", effect: "deny" },
|
||||
{ action: "shell", resource: "git *--ext-diff*", effect: "deny" },
|
||||
{ action: "shell", resource: "git *>*", effect: "deny" },
|
||||
],
|
||||
"code-reviewer keeps the read-only Git allowlist and write-through denials",
|
||||
);
|
||||
});
|
||||
|
||||
test("plugin preserves agents registered before it", async () => {
|
||||
const customReviewer = { id: "code-reviewer", name: "Custom", mode: "subagent" };
|
||||
const skills = fakeEditor();
|
||||
const agents = fakeEditor(new Map([["code-reviewer", customReviewer]]));
|
||||
|
||||
await plugin.setup(contextFor(skills, agents));
|
||||
|
||||
assert.equal(agents.get("code-reviewer"), customReviewer);
|
||||
assert.ok(agents.get("code-explorer"), "other roles are still registered");
|
||||
assert.ok(agents.get("code-architect"), "other roles are still registered");
|
||||
});
|
||||
|
||||
test("plugin exposes the OpenCode 1 config hook for skills and agents", async () => {
|
||||
const { config } = await plugin.server();
|
||||
const target = {};
|
||||
await config(target);
|
||||
|
||||
assert.deepEqual(target.skills.paths, [SKILLS_DIR]);
|
||||
await config(target);
|
||||
assert.deepEqual(target.skills.paths, [SKILLS_DIR], "skill path is registered once");
|
||||
|
||||
for (const name of ["code-explorer", "code-architect", "code-reviewer"]) {
|
||||
const agent = target.agent[name];
|
||||
assert.ok(agent, `${name} is registered`);
|
||||
assert.equal(agent.mode, "subagent");
|
||||
assert.equal(agent.permission.edit, "deny");
|
||||
assert.equal(agent.permission.task, "deny");
|
||||
assert.equal(agent.permission.webfetch, "deny");
|
||||
if (name !== "code-reviewer") assert.equal(agent.permission.bash, "deny");
|
||||
assert.deepEqual(agent.permission.read, {
|
||||
"*": "allow",
|
||||
"*.env": "deny",
|
||||
@@ -27,12 +222,12 @@ test("plugin registers the feature workflow roles as read-only subagents", async
|
||||
}
|
||||
|
||||
assert.match(
|
||||
config.agent["code-reviewer"].prompt,
|
||||
target.agent["code-reviewer"].prompt,
|
||||
/Dispatched handoff/i,
|
||||
"registered reviewer inherits the handoff contract",
|
||||
);
|
||||
|
||||
assert.deepEqual(config.agent["code-reviewer"].permission.bash, {
|
||||
assert.deepEqual(target.agent["code-reviewer"].permission.bash, {
|
||||
"*": "deny",
|
||||
"git status*": "allow",
|
||||
"git diff*": "allow",
|
||||
@@ -48,12 +243,55 @@ test("plugin registers the feature workflow roles as read-only subagents", async
|
||||
});
|
||||
});
|
||||
|
||||
test("plugin preserves user-defined agents with the same names", async () => {
|
||||
const plugin = await OpencodePowerPack();
|
||||
test("OpenCode 1 config hook preserves user-defined agents", async () => {
|
||||
const customReviewer = { description: "My reviewer", mode: "subagent" };
|
||||
const config = { agent: { "code-reviewer": customReviewer } };
|
||||
const target = { agent: { "code-reviewer": customReviewer } };
|
||||
const { config } = await plugin.server();
|
||||
|
||||
await plugin.config(config);
|
||||
await config(target);
|
||||
|
||||
assert.equal(config.agent["code-reviewer"], customReviewer);
|
||||
assert.equal(target.agent["code-reviewer"], customReviewer);
|
||||
assert.ok(target.agent["code-explorer"], "other roles are still registered");
|
||||
assert.ok(target.agent["code-architect"], "other roles are still registered");
|
||||
});
|
||||
|
||||
test("both hosts derive the same read-only rules from one permission source", async () => {
|
||||
const { agents } = await register();
|
||||
const target = {};
|
||||
await (await plugin.server()).config(target);
|
||||
|
||||
assert.deepEqual(agentPolicies(loadSkills(SKILLS_DIR))["code-explorer"].permissions, [
|
||||
{ action: "*", resource: "*", effect: "deny" },
|
||||
{ action: "read", resource: "*", effect: "allow" },
|
||||
{ action: "read", resource: "*.env", effect: "deny" },
|
||||
{ action: "read", resource: "*.env.*", effect: "deny" },
|
||||
{ action: "read", resource: "*.env.example", effect: "allow" },
|
||||
{ action: "glob", resource: "*", effect: "allow" },
|
||||
{ action: "grep", resource: "*", effect: "allow" },
|
||||
{ action: "edit", resource: "*", effect: "deny" },
|
||||
{ action: "subagent", resource: "*", effect: "deny" },
|
||||
{ action: "webfetch", resource: "*", effect: "deny" },
|
||||
{ action: "websearch", resource: "*", effect: "deny" },
|
||||
{ action: "external_directory", resource: "*", effect: "deny" },
|
||||
{ action: "shell", resource: "*", effect: "deny" },
|
||||
]);
|
||||
assert.equal(target.agent["code-explorer"].permission.list, "allow", "OpenCode 1 keeps its list tool");
|
||||
});
|
||||
|
||||
test("host edits to one registered agent do not leak into others or later registrations", async () => {
|
||||
// OpenCode 2 mutates host permission arrays after our registration.
|
||||
const first = await register();
|
||||
first.agents.get("code-explorer").permissions.push({ action: "edit", resource: "*", effect: "allow" });
|
||||
assert.equal(first.agents.get("code-architect").permissions.at(-1).action, "*");
|
||||
const second = await register();
|
||||
assert.equal(second.agents.get("code-explorer").permissions.at(-1).action, "*");
|
||||
|
||||
const { config } = await plugin.server();
|
||||
const legacy = {};
|
||||
await config(legacy);
|
||||
legacy.agent["code-explorer"].permission.read["*.env"] = "allow";
|
||||
assert.equal(legacy.agent["code-architect"].permission.read["*.env"], "deny");
|
||||
const fresh = {};
|
||||
await config(fresh);
|
||||
assert.equal(fresh.agent["code-explorer"].permission.read["*.env"], "deny");
|
||||
});
|
||||
|
||||
@@ -543,6 +543,8 @@ test("runner rejects a workspace inside an upstream shared-write compatibility p
|
||||
const workspace = path.join(fixture, "claude", "project");
|
||||
await mkdir(workspace, { recursive: true });
|
||||
try {
|
||||
// A repository-local TMPDIR must not resolve to the enclosing checkout.
|
||||
await execFileAsync("git", ["init", "--quiet", workspace]);
|
||||
await assert.rejects(runSandboxedCommand({
|
||||
profile: profile("develop", { workspace: "write" }),
|
||||
allowedDomains: [],
|
||||
|
||||
Reference in New Issue
Block a user