Update the skills for OpenCode 2 and keep OpenCode 1 support

This commit is contained in:
Wayner Barrios
2026-10-19 01:48:34 -04:00
19 changed files with 1417 additions and 506 deletions
+16 -3
View File
@@ -4,19 +4,32 @@ on:
pull_request:
paths:
- ".opencode/**"
- "index.js"
- "skills/**"
- "scripts/opencode-smoke.mjs"
- "tests/opencode-smoke.test.mjs"
- "package.json"
- "package-lock.json"
- ".github/workflows/opencode-smoke.yml"
workflow_dispatch:
permissions:
contents: read
jobs:
opencode-1-18-7:
opencode:
name: opencode-${{ matrix.version }}
runs-on: ubuntu-latest
timeout-minutes: 5
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
version: ["1.18.7", "2.0.14", "2.0.32"]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: "24"
- run: node scripts/opencode-smoke.mjs 1.18.7
cache: "npm"
- run: npm ci
- run: node scripts/opencode-smoke.mjs ${{ matrix.version }}
+127
View File
@@ -0,0 +1,127 @@
import { readdirSync, readFileSync, statSync } from "fs";
import path from "path";
const AGENT_NAMES = ["code-explorer", "code-architect", "code-reviewer"];
const READ_ONLY_PERMISSION = {
"*": "deny",
read: {
"*": "allow",
"*.env": "deny",
"*.env.*": "deny",
"*.env.example": "allow",
},
glob: "allow",
grep: "allow",
list: "allow",
edit: "deny",
task: "deny",
webfetch: "deny",
websearch: "deny",
external_directory: "deny",
};
const REVIEW_BASH_PERMISSION = {
"*": "deny",
"git status*": "allow",
"git diff*": "allow",
"git show*": "allow",
"git log*": "allow",
"git blame*": "allow",
"git rev-parse*": "allow",
"git merge-base*": "allow",
"git ls-files*": "allow",
"git *--output*": "deny",
"git *--ext-diff*": "deny",
"git *>*": "deny",
};
// OpenCode 2 renamed these V1 permission keys and folded `list` into `read`.
const V2_ACTIONS = { bash: "shell", task: "subagent", list: null };
function readSkill(skillsDir, id) {
const file = path.join(skillsDir, id, "SKILL.md");
const match = readFileSync(file, "utf8").match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/);
if (!match) throw new Error(`Invalid frontmatter in ${id}/SKILL.md`);
const field = (key) => match[1]
.split(/\r?\n/)
.find((line) => line.startsWith(`${key}:`))
?.slice(key.length + 1)
.trim();
const description = field("description");
if (!description) throw new Error(`Missing description in ${id}/SKILL.md`);
// Native OpenCode 2 discovery strips frontmatter, so the body is the content.
return { id, name: field("name") || id, description, path: file, content: match[2] };
}
/** Parses every bundled SKILL.md into the OpenCode 2 Skill.Info shape. */
export function loadSkills(skillsDir) {
return readdirSync(skillsDir)
.filter((entry) => statSync(path.join(skillsDir, entry, "SKILL.md"), { throwIfNoEntry: false })?.isFile())
.sort()
.map((id) => readSkill(skillsDir, id));
}
function specialists(skills) {
return AGENT_NAMES.map((name) => {
const skill = skills.find(({ id }) => id === name);
if (!skill) throw new Error(`Missing specialist skill ${name}`);
return skill;
});
}
// Returns a fresh copy on every call: OpenCode edits agent permissions in place.
function permissionFor(name) {
return structuredClone({
...READ_ONLY_PERMISSION,
bash: name === "code-reviewer" ? REVIEW_BASH_PERMISSION : "deny",
});
}
/** Converts a V1 permission map into the ordered V2 ruleset, preserving key order. */
function toRuleset(permission) {
return Object.entries(permission).flatMap(([key, value]) => {
const action = key in V2_ACTIONS ? V2_ACTIONS[key] : key;
if (!action) return [];
const patterns = typeof value === "string" ? { "*": value } : value;
return Object.entries(patterns).map(([resource, effect]) => ({ action, resource, effect }));
});
}
/** Matches the host's whole-value * and ? permission wildcards. */
function matches(pattern, value) {
const expression = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&")
.replace(/\*/g, ".*").replace(/\?/g, ".");
return new RegExp(`^${expression}$`, process.platform === "win32" ? "is" : "s").test(value);
}
/** Evaluates our ordered policy without granting access denied by the host. */
export function permissionEffect(rules, action, resource) {
// The reviewer's Git prefixes authorize one inspection command, not shell
// composition, redirection, or command substitution following that prefix.
if (action === "shell" && /[;&|<>`$\r\n]/.test(resource)) return "deny";
const normalized = ["read", "edit", "external_directory"].includes(action)
? resource.replaceAll("\\", "/") : resource;
return rules.findLast((rule) => matches(rule.action, action) && matches(rule.resource, normalized))?.effect ?? "deny";
}
/** OpenCode 1 agent definitions, merged through the plugin config hook. */
export function legacyAgents(skills) {
return Object.fromEntries(specialists(skills).map((skill) => [skill.id, {
description: skill.description,
prompt: skill.content.trim(),
mode: "subagent",
permission: permissionFor(skill.id),
}]));
}
/** OpenCode 2 Agent.Info patches, applied through an agent transform. */
export function agents(skills) {
return Object.fromEntries(specialists(skills).map((skill) => [skill.id, {
name: skill.name,
description: skill.description,
system: skill.content.trim(),
mode: "subagent",
hidden: false,
permissions: toRuleset(permissionFor(skill.id)),
}]));
}
-65
View File
@@ -1,65 +0,0 @@
import { readFileSync } from "fs";
import path from "path";
const AGENT_NAMES = ["code-explorer", "code-architect", "code-reviewer"];
const READ_ONLY_PERMISSION = {
"*": "deny",
read: {
"*": "allow",
"*.env": "deny",
"*.env.*": "deny",
"*.env.example": "allow",
},
glob: "allow",
grep: "allow",
list: "allow",
edit: "deny",
task: "deny",
webfetch: "deny",
websearch: "deny",
external_directory: "deny",
};
const REVIEW_BASH_PERMISSION = {
"*": "deny",
"git status*": "allow",
"git diff*": "allow",
"git show*": "allow",
"git log*": "allow",
"git blame*": "allow",
"git rev-parse*": "allow",
"git merge-base*": "allow",
"git ls-files*": "allow",
"git *--output*": "deny",
"git *--ext-diff*": "deny",
"git *>*": "deny",
};
function readSkill(skillsDir, name) {
const source = readFileSync(path.join(skillsDir, name, "SKILL.md"), "utf8");
const match = source.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/);
if (!match) throw new Error(`Invalid frontmatter in ${name}/SKILL.md`);
const description = match[1]
.split(/\r?\n/)
.find((line) => line.startsWith("description:"))
?.slice("description:".length)
.trim();
if (!description) throw new Error(`Missing description in ${name}/SKILL.md`);
return { description, prompt: match[2].trim() };
}
export function loadAgentConfigs(skillsDir) {
return Object.fromEntries(AGENT_NAMES.map((name) => {
const skill = readSkill(skillsDir, name);
const permission = name === "code-reviewer"
? { ...READ_ONLY_PERMISSION, bash: REVIEW_BASH_PERMISSION }
: { ...READ_ONLY_PERMISSION, bash: "deny" };
return [name, {
...skill,
mode: "subagent",
permission,
}];
}));
}
+56 -28
View File
@@ -1,22 +1,21 @@
/**
* opencode-power-pack
*
* Auto-registers the bundled skills directory so OpenCode discovers all
* skills shipped by this plugin (code-review, feature-dev, code-explorer,
* code-architect, code-reviewer, security-review, frontend-design,
* mcp-builder, skill-creator, agents-md-improver, agents-md-revise) without
* requiring symlinks or manual config.
* Registers every bundled skill and the feature workflow's specialist roles
* as read-only subagents derived from their SKILL.md bodies, on both hosts:
*
* OpenCode 1.18.7+ exposes discovered skills as same-named slash commands.
* The plugin also registers the feature workflow's specialist roles as
* read-only subagents derived from their SKILL.md bodies.
* - OpenCode 2 calls `setup` and receives the skill and agent transforms.
* - OpenCode 1 (1.18.7+) calls `server` on the default export and receives
* the config hook, which registers skills/ in `config.skills.paths` and
* merges the agents into `config.agent`.
*
* User-defined agents with the same names are preserved.
*
* ──── Attribution ────────────────────────────────────────────────────────
*
* The plugin loader pattern (importing fs/path via fileURLToPath, exporting
* an async ctx → hooks function, pushing into config.skills.paths via the
* `config` hook) is adapted directly from Jesse Vincent's superpowers
* plugin: https://github.com/obra/superpowers
* The OpenCode 1 loader pattern (importing fs/path via fileURLToPath and
* pushing into config.skills.paths via the `config` hook) is adapted directly
* from Jesse Vincent's superpowers plugin: https://github.com/obra/superpowers
*
* The skills under skills/ are modified upstream works. See UPSTREAMS.json
* for immutable source commits and blobs, and THIRD_PARTY_NOTICES.md for
@@ -26,24 +25,53 @@
import path from 'path';
import { fileURLToPath } from 'url';
import { loadAgentConfigs } from './agent-config.js';
import { agents, legacyAgents, loadSkills, permissionEffect } from '../lib/agent-config.js';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const skillsDir = path.resolve(__dirname, '../../skills');
const bundledAgents = loadAgentConfigs(skillsDir);
const skillsDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../skills');
const skills = loadSkills(skillsDir);
export const OpencodePowerPack = async () => {
return {
config: async (config) => {
config.skills = config.skills || {};
config.skills.paths = config.skills.paths || [];
if (!config.skills.paths.includes(skillsDir)) {
config.skills.paths.push(skillsDir);
export default {
id: 'opencode-power-pack',
async setup(ctx) {
const policies = agents(skills);
await ctx.skill.transform((editor) => {
for (const skill of skills) editor.add({ ...skill });
});
await ctx.agent.transform((editor) => {
for (const [name, { permissions, ...agent }] of Object.entries(policies)) {
// V2 applies user agent configuration later. Keep its permission defaults
// intact so a same-named user replacement does not inherit our denials.
if (!editor.get(name)) editor.update(name, (draft) => Object.assign(draft, agent));
}
config.agent = config.agent || {};
for (const [name, agent] of Object.entries(bundledAgents)) {
if (!config.agent[name]) config.agent[name] = agent;
});
await ctx.permission.hook('evaluate', async (event) => {
const name = event.agent ?? (await ctx.session.get({ sessionID: event.sessionID })).data?.agent;
const policy = Object.hasOwn(policies, name) ? policies[name] : undefined;
if (!policy) return;
const { data: agent } = await ctx.agent.get({ agentID: name });
if (agent?.system !== policy.system || agent.mode !== 'subagent') return;
// Global and saved allows must not weaken the bundled read-only policy.
// This hook only denies; explicit user denies and approval prompts remain.
if (event.resources.some((resource) => permissionEffect(policy.permissions, event.action, resource) === 'deny')) {
event.effect = 'deny';
event.message = `The bundled ${name} agent is read-only.`;
}
},
};
});
},
async server() {
return {
config: async (config) => {
config.skills = config.skills || {};
config.skills.paths = config.skills.paths || [];
if (!config.skills.paths.includes(skillsDir)) config.skills.paths.push(skillsDir);
config.agent = config.agent || {};
for (const [name, agent] of Object.entries(legacyAgents(skills))) {
if (!config.agent[name]) config.agent[name] = agent;
}
},
};
},
};
+59 -18
View File
@@ -17,7 +17,7 @@
<a href="https://github.com/hashgraph-online/hol-guard"><img alt="HOL Guard scanner" src="https://img.shields.io/badge/HOL%20Guard-scanned-00a67e?style=flat-square"></a>
<img alt="Skills: 55" src="https://img.shields.io/badge/skills-55-FFD60A?style=flat-square&labelColor=0B0F14">
<img alt="Claude Code plugin" src="https://img.shields.io/badge/Claude_Code-plugin-0B0F14?style=flat-square&labelColor=FFD60A">
<img alt="OpenCode 1.18.7+" src="https://img.shields.io/badge/opencode-1.18.7%2B-0B0F14?style=flat-square&labelColor=FFD60A">
<img alt="OpenCode 1.18.7+ / 2.x" src="https://img.shields.io/badge/opencode-1.18.7%2B%20%2F%202.x-0B0F14?style=flat-square&labelColor=FFD60A">
<img alt="Codex plugin" src="https://img.shields.io/badge/Codex-plugin-0B0F14?style=flat-square&labelColor=FFD60A">
<img alt="Pi package" src="https://img.shields.io/badge/Pi-package-0B0F14?style=flat-square&labelColor=FFD60A">
</p>
@@ -329,7 +329,7 @@ They are complementary rather than duplicate workflows. Codex shows distinct use
- Node.js 20.11.0 or newer, including npm/npx, for [selective installation](#selective-install-with-npm) and the sandbox runner
- One supported host:
- A current Claude Code installation with plugin support: <https://code.claude.com/docs/en/plugins>
- OpenCode 1.18.7 or newer: <https://opencode.ai>
- OpenCode 1.18.7 or newer, including OpenCode 2: <https://opencode.ai>
- A current Codex CLI or Codex desktop environment with plugin support: <https://developers.openai.com/codex/>
- A current Pi coding agent installation: <https://pi.dev/>
@@ -360,26 +360,36 @@ For a smaller personal or repository-specific set, use the [selective npm instal
### OpenCode From GitHub
On OpenCode 2:
```bash
opencode plugin add "opencode-power-pack@git+https://github.com/waybarrios/opencode-power-pack.git"
```
On OpenCode 1:
```bash
opencode plugin --global "opencode-power-pack@git+https://github.com/waybarrios/opencode-power-pack.git"
```
Restart OpenCode after installation. OpenCode discovers each skill and creates its same-named slash command automatically.
Restart OpenCode after installation. One plugin adapts to the host version: OpenCode 2 registers all fifty-five skills and the three specialist agents through the V2 plugin API, and OpenCode 1 keeps the original `config.skills.paths` and `config.agent` hook. Either way the skills are invocable by name and the three roles are registered as read-only subagents.
To pin a published release, append its tag:
The compatibility smoke tests cover OpenCode **1.18.7, 2.0.14, and 2.0.32**. The `SKILL.md` format stays the same across these hosts.
To pin a published release, append its tag to the same spec:
```bash
opencode plugin --global "opencode-power-pack@git+https://github.com/waybarrios/opencode-power-pack.git#<tag>"
opencode plugin add "opencode-power-pack@git+https://github.com/waybarrios/opencode-power-pack.git#<tag>"
```
### OpenCode From A Local Clone
```bash
git clone https://github.com/waybarrios/opencode-power-pack.git ~/code/opencode-power-pack
opencode plugin --global "opencode-power-pack@git+file:///home/you/code/opencode-power-pack"
opencode plugin add "opencode-power-pack@git+file:///home/you/code/opencode-power-pack"
```
Use an absolute `file://` URL adjusted for your operating system. The target directory must be a Git repository.
Use an absolute `file://` URL adjusted for your operating system. The target directory must be a Git repository. On OpenCode 1, replace `plugin add` with `plugin --global` as shown above.
### Pi From GitHub
@@ -403,6 +413,18 @@ After marketplace installation, run `/plugin` and confirm that `opencode-power-p
### Verify OpenCode
Restart OpenCode, then check the registered skills and specialist agents.
On OpenCode 2:
```bash
opencode debug agents
```
The output should include `code-explorer`, `code-architect`, and `code-reviewer` as `subagent` entries. OpenCode 2 enforces their read-only restrictions when evaluating permission requests, so the agent listing can show the host's base permissions. In the TUI, the slash catalog should list `/code-review`, `/feature-dev`, `/frontend-design`, and the other bundled skills.
On OpenCode 1:
```bash
opencode debug skill
opencode debug agent code-explorer
@@ -452,13 +474,19 @@ For Pi:
pi update git:github.com/waybarrios/opencode-power-pack
```
For an OpenCode GitHub installation:
For an OpenCode GitHub installation on OpenCode 2:
```bash
opencode plugin --global --force "opencode-power-pack@git+https://github.com/waybarrios/opencode-power-pack.git"
opencode plugin update "opencode-power-pack@git+https://github.com/waybarrios/opencode-power-pack.git"
```
Restart OpenCode after the command finishes. For a pinned installation, update the tag first. Local-clone users should run `git pull` in the clone and reinstall with the same `git+file://` spec.
On OpenCode 1, reinstall with the same spec:
```bash
opencode plugin --global "opencode-power-pack@git+https://github.com/waybarrios/opencode-power-pack.git"
```
Restart OpenCode after the command finishes. Pass the exact configured package specification, including any pinned tag. Local-clone users should run `git pull` in the clone, then update using their original `opencode-power-pack@git+file://...` specification on OpenCode 2, or reinstall it on OpenCode 1.
## Uninstalling
@@ -480,7 +508,13 @@ For Pi:
pi remove git:github.com/waybarrios/opencode-power-pack
```
For OpenCode, remove the `opencode-power-pack@...` entry from the `plugin` array in the config where it was installed, then restart OpenCode. There are no command symlinks or copied command files to remove.
For OpenCode 2, remove the plugin from the global configuration:
```bash
opencode plugin remove "opencode-power-pack@git+https://github.com/waybarrios/opencode-power-pack.git"
```
Use your exact configured package specification if it differs from this example. Alternatively, delete the `opencode-power-pack@...` entry from the `plugins` array in the config where it was installed, then restart OpenCode. On OpenCode 1, delete the entry from the `plugin` array instead. There are no command symlinks or copied command files to remove.
## Troubleshooting
@@ -491,15 +525,15 @@ For OpenCode, remove the `opencode-power-pack@...` entry from the `plugin` array
| Codex cannot find the marketplace | Marketplace snapshot is missing or stale | Run `codex plugin marketplace add waybarrios/opencode-power-pack --ref main`, or upgrade the existing marketplace |
| Codex installed the plugin but skills do not appear | The current session predates installation | Start a new Codex session and inspect `/plugins` |
| Pi does not show the skills | Package is absent, disabled, or the session predates installation | Run `pi list`, inspect `pi config`, and start a new Pi session |
| Skills or commands do not appear | OpenCode is older than 1.18.7 | Upgrade OpenCode, restart, and run `opencode debug skill` |
| Skills are missing from debug output | Plugin installation failed or its config entry is absent | Re-run `opencode plugin --global --force <module-spec>` and inspect the reported error |
| Specialist agents are missing | Stale plugin checkout | Force reinstall, restart, and run `opencode debug agent code-explorer` |
| Skills or commands do not appear | OpenCode is older than 1.18.7 | Upgrade OpenCode, restart, and check the slash catalog or run `opencode debug skill` |
| Skills are missing from the catalog | Plugin installation failed or its `plugins` entry is absent | Reinstall with `opencode plugin add <module-spec>` (OpenCode 2) or `opencode plugin --global <module-spec>` (OpenCode 1) and inspect the reported error |
| Specialist agents are missing | Stale plugin checkout | Update the plugin, restart, and run `opencode debug agents` (OpenCode 2) or `opencode debug agent code-explorer` (OpenCode 1) |
| Installation reports a Git error | Invalid URL, network failure, or inaccessible repository | Validate the source with `git ls-remote <url>` |
| A workflow is rushed or incomplete | The backing model skipped multi-stage instructions | Use a stronger model and inspect whether required subagent tools are available |
## Invocation
Claude Code exposes plugin skills as `/opencode-power-pack:<skill>`, Codex uses `$` mentions, OpenCode 1.18.7+ exposes same-named slash commands, and Pi loads workflows from the installed skills package:
Claude Code exposes plugin skills as `/opencode-power-pack:<skill>`, Codex uses `$` mentions, OpenCode exposes the bundled skills by name (`/` catalog on OpenCode 2, same-named slash commands on OpenCode 1), and Pi loads workflows from the installed skills package:
```text
/code-review
@@ -560,11 +594,16 @@ opencode-power-pack
| +-- runs opt-in commands in the pinned native sandbox backend
|
+-- .opencode/plugins/opencode-power-pack.js
| +-- registers skills/ in config.skills.paths
| +-- OpenCode 1: registers config.skills.paths and config.agent
| +-- OpenCode 2: registers skills and agents through the V2 transforms
| +-- registers code-explorer as a read-only subagent
| +-- registers code-architect as a read-only subagent
| +-- registers code-reviewer with read-only Git commands
|
+-- .opencode/lib/agent-config.js
| +-- derives skill and subagent definitions from the SKILL.md frontmatter
| +-- builds the OpenCode 1 permission maps and OpenCode 2 rulesets
|
+-- skills/<name>/SKILL.md
| +-- shared Codex/OpenCode workflow
| +-- host-native invocation metadata
@@ -574,9 +613,11 @@ opencode-power-pack
+-- repository, commit, path, blob, date, and adaptation type
```
Each `SKILL.md` is the single source for its workflow. Claude Code loads the directory through its plugin namespace. The OpenCode plugin derives specialist-agent prompts from those files at startup. Codex loads the same skills from its plugin and follows `feature-dev`'s specialist assignments with its native subagent workflow; installing a skill does not itself create a named custom agent.
Each `SKILL.md` is the single source for its workflow. Claude Code loads the directory through its plugin namespace. The OpenCode plugin adapts to the host version and derives specialist-agent prompts from those files at startup. Codex loads the same skills from its plugin and follows `feature-dev`'s specialist assignments with its native subagent workflow; installing a skill does not itself create a named custom agent.
The packaged agents deny edits, external network access, and nested tasks. `code-reviewer` additionally allows a narrow set of read-only Git commands.
The packaged agents deny edits, external network access, and nested subagents. `code-reviewer` additionally allows a narrow set of read-only Git commands.
On OpenCode 2, permission evaluation enforces these restrictions even when global rules allow broader access. User-configured agents with their own system prompt or a primary mode replace the bundled role and keep the host's permissions. Explicit user denials and approval prompts still apply to bundled agents.
## Scope And Non-Goals
+3 -2
View File
@@ -103,8 +103,9 @@ Copyright 2025 Anthropic. The upstream notice is available at
## Superpowers OpenCode Plugin Pattern
The skill-path registration pattern in `.opencode/plugins/opencode-power-pack.js`
is adapted from the MIT-licensed `obra/superpowers` OpenCode plugin.
The V1 skill-path registration pattern in the OpenCode plugin was adapted from
the MIT-licensed `obra/superpowers` OpenCode plugin. The plugin keeps that
hook for OpenCode 1 and adds the OpenCode 2 port through the V2 plugin API.
Copyright 2025 Jesse Vincent. The upstream notice is available at
`LICENSES/Superpowers-MIT.txt`.
+5 -3
View File
@@ -8,11 +8,13 @@ Locally inspected host versions:
- Codex CLI 0.147.0
- Claude Code 2.1.231
- OpenCode 1.18.18
- OpenCode 1.18.32 and 2.0.14 (migration baseline)
- Pi 0.83.0
The repository smoke tests retain their separately pinned minimum versions. Host APIs evolve, so adapter tests and this matrix must be updated together.
OpenCode plugin compatibility is additionally verified by model-free smoke tests on 1.18.7, 2.0.14, and 2.0.32 (2026-10-19). V2 checks load the real package directory, inspect the final catalogs, and evaluate specialist permission requests with permissive global rules. This checks plugin permissions, not whole-agent sandbox isolation.
## Compatibility Levels
| Level | Meaning |
@@ -31,14 +33,14 @@ The current runner is `shell-contained`. The command and its descendants are iso
|---|---|---|---|---|---|
| Codex | Supported through the Codex plugin and portable skill installation | Supported when the npm package is installed | Not yet implemented | Temporary native sandbox configuration plus the shared runner for explicit network access | Codex has native `read-only` and `workspace-write` shell sandboxes, but a skill cannot safely change the active session sandbox by prompt text alone. |
| Claude Code | Supported through the namespaced Claude plugin | Supported when the npm package is installed | Not yet implemented | Native sandbox settings plus hooks and permissions for tools outside Bash | Claude's native sandbox applies to Bash and descendants. Read, Edit, Write, WebFetch, MCP, and computer-use tools require separate permission controls. |
| OpenCode | Supported through the plugin and portable skill installation | Supported when the npm package is installed | Not yet implemented | A custom sandbox tool paired with per-agent denial of ordinary `bash` | File-edit and external-directory permissions also need profile-specific controls. |
| OpenCode | Supported through the plugin and portable skill installation | Supported when the npm package is installed | Not yet implemented | A custom sandbox tool paired with per-agent denial of ordinary `shell` | File-edit and external-directory permissions also need profile-specific controls. |
| Pi | Supported through the Pi package declaration and portable skill installation | Supported when the npm package is installed | Not yet implemented | An extension that intercepts or replaces built-in shell and file tools | A skill file by itself is advisory. |
Official host surfaces used by the adapter design:
- [Codex configuration reference](https://developers.openai.com/codex/config-reference/) documents `sandbox_mode`, workspace-write network controls, permission profiles, and environment policy.
- [Claude Code sandboxing](https://code.claude.com/docs/en/sandboxing) documents its Bash-only OS sandbox, `/sandbox`, `failIfUnavailable`, and the unsandboxed escape hatch. [Claude Code hooks](https://code.claude.com/docs/en/hooks) documents `PreToolUse` denial and input replacement.
- [OpenCode agents](https://opencode.ai/docs/agents/) documents per-agent `allow`, `ask`, and `deny` permissions. [OpenCode custom tools](https://opencode.ai/docs/custom-tools/) provides the portable runner integration point.
- [OpenCode permissions](https://opencode.ai/v2/docs/permissions/) documents the ordered `allow`, `ask`, and `deny` ruleset. [OpenCode tools](https://opencode.ai/v2/docs/tools/) and the [plugin tools API](https://opencode.ai/v2/docs/build/plugins/) provide the portable runner integration point.
- [Pi extensions](https://github.com/badlogic/pi-mono/blob/main/packages/coding-agent/docs/extensions.md) documents custom tools, built-in tool overrides, dynamic active tools, and blocking `tool_call` events.
## What Works Before Host Adapters
@@ -0,0 +1,86 @@
# OpenCode 2 compatibility implementation plan
> **For agentic workers:** Use superpowers:executing-plans to implement this plan task by task. Review the frozen PRs with the code-review skill.
**Goal:** Make the 55 bundled skills and three specialist agents work in OpenCode 2 while retaining OpenCode 1.18.7 compatibility.
**Architecture:** Integrate PR #45 into current main without committing, retain its dual `setup`/`server` entrypoint and shared permission source, and repair confirmed gaps. Add the root directory entrypoint from the approach in PR #52. Preserve existing workflows, licenses, user-defined agents, and the other supported hosts.
**Tech stack:** Node.js >=20.11.0, ES modules, node:test, npm, GitHub Actions.
**Spec:** User request to evaluate the existing PR, plan the upgrade and execute it; https://opencode.ai/v2/docs/migrate-v1/ and https://opencode.ai/v2/docs/build/plugins/migrate-v1/.
## Global constraints
- No commit, push, PR creation, PR merge, issue closure, or rebase without explicit user approval. Show each exact commit message before approval.
- Preserve the pre-existing untracked `2026-09-30-pr45-fixes.md` plan; it is historical context, not a current specification.
- Keep scratch data and tool execution artifacts inside `.scratch/`; pass a project-local TMPDIR when running tests.
- Keep `engines.opencode` at `>=1.18.7`; add no runtime dependencies for the plain plugin definition.
- Verify pinned V1 1.18.7, the PR's V2 baseline 2.0.14, and current npm V2 2.0.32.
- Execute locally now as requested. Defer commit and remote mutations to the final approval step.
## Review focus
- V2 configured package directories must load their own root entrypoint, without a shim masking packaging failures.
- Helpers must stay outside the automatically scanned plugins directory.
- Host mutation of one agent's permissions must not leak into other agents or later loads.
- Global allow rules must not silently defeat read-only specialist permissions on supported hosts.
- Behavioral tooling must reject malformed provider/model references and follow the real V2 session API.
### Task 1: Review and integrate the best PR locally
- [x] Freeze PR #45 at `a3523bc08f8bce2a31f0b2e54f14fef36dbe0a9c` and PR #52 at `b01f1428ca583e06b372689a74cd030a475a2748`.
- [x] Create local `feat/opencode-v2-update` from main `9dccb6de12a76cb9aa472e06208486909d31e088`.
- [x] Start a no-commit merge of #45 and resolve README while retaining the current 55 skills.
- [x] Run the baseline suite and record actual failures.
- [x] Validate reviewer findings against pinned source and reproductions.
### Task 2: Repair directory loading and registration
Files: `index.js`, `package.json`, `tests/package-surface.test.mjs`, `.opencode/plugins/opencode-power-pack.js`, `.opencode/lib/agent-config.js`, `tests/plugin-registration.test.mjs`.
- [x] Add a package-surface regression asserting the published root `index.js` and its dual-compatible default export; observe failure.
- [x] Export the existing plugin from root `index.js` and include it in published files.
- [x] Add and reproduce regressions for any confirmed registration defects before implementing their fixes.
- [x] Run package and plugin tests, preserving user definitions and independent permissions.
### Task 3: Verify actual OpenCode 2 and repair contributor tooling
Files: `scripts/opencode-smoke.mjs`, `.github/workflows/opencode-smoke.yml`, `scripts/behavioral-evals.mjs`, `tests/opencode-smoke.test.mjs`, `tests/behavioral-evals.test.mjs`, `tests/fixtures/fake-opencode.mjs`.
- [x] Test the actual package root with V2 smoke, without wrapping the entrypoint under test.
- [x] Assert effective specialist permission rules with permissive global settings.
- [x] Add regression tests for confirmed model/API failures, then implement minimal fixes.
- [x] Keep V1 and V2 smoke jobs, adding current V2 2.0.32 and minimum workflow permissions.
- [x] Run smoke on 1.18.7, 2.0.14, and 2.0.32.
### Task 4: Documentation, complete verification, and approval handoff
Files: `README.md`, `docs/sandbox-compatibility.md`, `evals/behavioral/README.md`, this plan.
- [x] Document tested versions and V2 contributor-tooling requirements accurately.
- [x] Run `npm test`, package checks, and inspect `git diff --check` and final changed paths.
- [x] Record results and any material limits; retain original PR history in the pending merge.
- [x] Prepare the exact proposed commit message and approval handoff before committing or any remote action.
## Execution notes
- PR #45 provides the more complete migration but conflicts with current main in README. PR #52 is narrower and supplies a useful root entrypoint, but leaves the helper in the plugin scan directory and does not port CI/evals.
- The user explicitly requested planning followed by execution. No additional planning approval gate applies; repository and developer commit/remote approval requirements remain in force.
- Node tests create temporary fixtures. A project-local TMPDIR keeps their writes inside the repository.
- Ruling: use a deny-only V2 permission hook and leave host agent permission defaults intact. The host applies global rules and user agent definitions after external transforms on both 2.0.14 and 2.0.32. A hook protects bundled roles after global allows, while an identity check preserves user replacements with a different system or primary mode.
- Ruling: wait for native final catalogs before smoke assertions. The health endpoint and first catalog request can return before plugin registration completes. A setup-time companion plugin hides the later global-permission overwrite.
- Confirmed fixes beyond #45: failed/interrupted turns cannot produce passing behavioral evidence; optional model variants are parsed correctly; Git update/remove instructions use the exact configured spec; local Git installs require update/reinstall after pulling.
- Native V2 database credentials are intentionally not copied into evaluation runtimes. Live runs are documented to use provider environment credentials; no credential-backed model run is part of verification.
- Baseline parallel tests with project-local temporary directories exposed sandbox fixture interference. Serial execution isolated one additional pre-existing fixture assumption: its workspace resolved to the outer checkout. Initializing that fixture as its own Git repository reproduces the intended protected-path rejection without requiring a native sandbox backend. Full-suite verification uses `npm test -- --test-concurrency=1`.
- Independent final review found one transient socket retry regression in catalog polling. A new HTTP regression failed before the fix and passed after restoring five retries. The revised review reported no remaining confirmed Important or Critical issues.
## Final verification and handoff
- `npm test -- --test-concurrency=1`: 418 tests, 417 passed, zero failures, one optional native sandbox integration skipped. TMPDIR and npm cache were project-local.
- `npm run smoke:opencode`: passed on real OpenCode 1.18.7, 2.0.14, and 2.0.32. Each discovered 55 skills and three agents; V2 also passed effective permission checks under global allow rules.
- `git diff --cached --check`: clean; no unresolved merge paths. The historical untracked plan remains untouched.
- No credential-backed live model evaluation was run. The package version remains 0.5.0; release publishing is outside this task.
- Integration branch: `feat/opencode-v2-update`, based on main `9dccb6de12a76cb9aa472e06208486909d31e088`, with a pending merge retaining the five commits of PR #45.
- Approved exact commit message: `Update the skills for OpenCode 2 and keep OpenCode 1 support`.
- The user approved the final English commit message and the integration workflow. Commit, push, PR creation, and PR merge are the remaining execution steps, with merge conditional on passing CI.
+5 -1
View File
@@ -8,7 +8,11 @@ Run `npm test`. Replay requires no model, network, or credentials and rejects st
## Live Run
Run `OPENCODE_EVAL_MODEL=provider/model npm run eval:behavioral` from a source checkout. The runner uses an empty temporary project, denies model-facing tools, and writes only a redacted report to `.artifacts/behavioral-evals/latest.json`.
Install OpenCode 2 and provide the selected provider's API key through its supported environment variable, then run `OPENCODE_EVAL_MODEL=provider/model npm run eval:behavioral` from a source checkout. An optional model variant uses `provider/model#variant`. This live runner targets the V2 session API; OpenCode 1 remains supported by the plugin and smoke tests.
The runner uses an empty temporary project, denies model-facing tools, and writes only a redacted report to `.artifacts/behavioral-evals/latest.json`. It accepts text only from a successfully completed turn; failed or interrupted turns remain incomplete even if partial text matches the oracles.
Native OpenCode 2 login credentials live in its database and are not copied into the isolated runtime. A legacy `auth.json`, when present, is copied for migration compatibility, but can be absent or stale. Use environment credentials for reproducible live runs; the runner does not copy your session database.
## Review And Acceptance
+2
View File
@@ -0,0 +1,2 @@
// OpenCode 2 resolves configured plugin directories through index.js.
export { default } from "./.opencode/plugins/opencode-power-pack.js";
+2 -1
View File
@@ -16,6 +16,7 @@
"opencode": ">=1.18.7"
},
"files": [
"index.js",
"skills",
".claude-plugin",
".codex-plugin",
@@ -35,7 +36,7 @@
"prepublishOnly": "npm test",
"eval:behavioral": "node scripts/behavioral-evals.mjs run",
"eval:behavioral:accept": "node scripts/behavioral-evals.mjs accept",
"smoke:opencode": "node scripts/opencode-smoke.mjs 1.18.7"
"smoke:opencode": "node scripts/opencode-smoke.mjs 1.18.7 && node scripts/opencode-smoke.mjs 2.0.14 && node scripts/opencode-smoke.mjs 2.0.32"
},
"pi": {
"skills": [
+196 -129
View File
@@ -1,4 +1,4 @@
import { createHash } from "node:crypto";
import { createHash, randomBytes } from "node:crypto";
import { spawn } from "node:child_process";
import {
chmodSync,
@@ -12,6 +12,7 @@ import {
statSync,
writeFileSync,
} from "node:fs";
import { createServer } from "node:net";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
@@ -515,33 +516,83 @@ export function redactResponse(response, testCase) {
return redacted.replace(credential, "[REDACTED]");
}
export function parseJsonEvents(stdout) {
/**
* Collects the assistant text from a V2 session context response and notes
* whether the model requested any tool. A tool request means the eval runtime
* could not keep the model inside the permission-free envelope.
*/
export function parseSessionContext(context) {
if (!isObject(context) || !Array.isArray(context.data)) {
throw new TypeError("malformed OpenCode session context");
}
const textParts = [];
let toolRequested = false;
for (const line of stdout.split(/\r?\n/)) {
if (line.trim() === "") continue;
let event;
try {
event = JSON.parse(line);
} catch {
throw new TypeError("malformed OpenCode JSON event");
let terminal;
for (const message of context.data) {
if (!isObject(message)) continue;
if (message.type === "idle") terminal = message;
if (message.type !== "assistant") continue;
if (message.error) throw new TypeError("unsuccessful OpenCode turn");
if (!Array.isArray(message.content)) {
throw new TypeError("malformed OpenCode session context");
}
if (!isObject(event) || !isObject(event.part)) {
throw new TypeError("malformed OpenCode JSON event");
}
if (event.part.type === "tool") toolRequested = true;
if (event.type === "text" && event.part.type === "text") {
if (typeof event.part.text !== "string") {
throw new TypeError("malformed OpenCode JSON event");
for (const part of message.content) {
if (!isObject(part)) throw new TypeError("malformed OpenCode session context");
if (part.type === "tool") toolRequested = true;
if (part.type === "text") {
if (typeof part.text !== "string") {
throw new TypeError("malformed OpenCode session context");
}
textParts.push(part.text);
}
textParts.push(event.part.text);
}
}
if ((terminal && (terminal.outcome !== "succeeded" || terminal.error))
|| (textParts.length > 0 && !terminal)) {
throw new TypeError("unsuccessful OpenCode turn");
}
return { response: textParts.join("\n"), toolRequested };
}
export function buildEvalConfig(pluginUrl) {
return { plugin: [pluginUrl], permission: { "*": "deny" } };
/**
* V2 loads a configured plugin directory through its index.js entrypoint and
* uses an ordered permission ruleset. The eval denies every tool so a case
* can only pass on model text.
*/
export function buildEvalConfig(pluginDirectory) {
return {
plugins: [pluginDirectory],
permissions: [{ action: "*", resource: "*", effect: "deny" }],
};
}
export function parseModelReference(model) {
const match = typeof model === "string" && model.match(/^([^/#\s]+)\/([^/#\s]+)(?:#([^/#\s]+))?$/);
if (!match) throw new TypeError("OPENCODE_EVAL_MODEL must use provider/model form with an optional #variant");
return { providerID: match[1], id: match[2], ...(match[3] ? { variant: match[3] } : {}) };
}
async function availablePort() {
const server = createServer();
await new Promise((resolve, reject) => server.listen(0, "127.0.0.1", resolve).once("error", reject));
const address = server.address();
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
return address.port;
}
/**
* Writes the plugin-directory shim the eval runtime loads. The shim keeps the
* package entrypoint in the repository, so its skills and dependencies resolve
* exactly as they do for an installed package.
*/
function writePluginShim(runtime, repo) {
const directory = path.join(runtime.root, "plugin");
mkdirSync(directory, { recursive: true });
const entrypoint = pathToFileURL(
path.join(repo, ".opencode/plugins/opencode-power-pack.js"),
).href;
writeFileSync(path.join(directory, "index.js"), `export { default } from ${JSON.stringify(entrypoint)};\n`);
return directory;
}
export function defaultOpenCodeCommand(platform = process.platform) {
@@ -639,7 +690,7 @@ function createRuntime(env) {
}
}
function isolatedEnv(env, runtime, model, pluginUrl) {
function isolatedEnv(env, runtime, model, config, password) {
const childEnv = {};
const isolatedPaths = new Set([
"HOME", "USERPROFILE", "APPDATA", "LOCALAPPDATA",
@@ -662,10 +713,50 @@ function isolatedEnv(env, runtime, model, pluginUrl) {
XDG_CACHE_HOME: runtime.cache,
XDG_STATE_HOME: runtime.state,
OPENCODE_EVAL_MODEL: model,
OPENCODE_CONFIG_CONTENT: JSON.stringify(buildEvalConfig(pluginUrl)),
OPENCODE_SERVER_PASSWORD: password,
OPENCODE_CONFIG_CONTENT: JSON.stringify(config),
};
}
async function requestJson(url, { method = "GET", body, authorization, signal } = {}) {
const response = await fetch(url, {
method,
headers: {
...(authorization ? { authorization } : {}),
...(body === undefined ? {} : { "content-type": "application/json" }),
},
body: body === undefined ? undefined : JSON.stringify(body),
signal,
});
if (!response.ok) {
throw new TypeError(`OpenCode API ${method} ${url} returned HTTP ${response.status}`);
}
const text = await response.text();
if (text === "") return undefined;
try {
return JSON.parse(text);
} catch {
throw new TypeError("malformed OpenCode API response");
}
}
async function waitForServer(baseURL, { authorization, signal, child, spawnError }) {
while (!signal.aborted) {
if (spawnError.error || child.exitCode !== null || child.signalCode !== null) return false;
try {
const response = await fetch(`${baseURL}/api/info`, {
headers: { authorization },
signal: AbortSignal.any([signal, AbortSignal.timeout(1_000)]),
});
if (response.ok) return true;
} catch {
if (signal.aborted) break;
}
await delay(100);
}
return false;
}
export async function runCase(testCase, options) {
const {
repo,
@@ -675,120 +766,98 @@ export async function runCase(testCase, options) {
commandArgsPrefix = [],
env = process.env,
} = options;
const modelReference = parseModelReference(model);
const started = Date.now();
const runtime = createRuntime(env);
const pluginUrl = pathToFileURL(path.join(repo, ".opencode/plugins/opencode-power-pack.js")).href;
const args = [
...commandArgsPrefix,
"run",
"--model", model,
"--command", testCase.skill,
"--format", "json",
testCase.prompt,
];
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), testCase.timeoutMs);
const spawnError = { error: undefined };
let child;
const finish = (status, failures, response) => reportEntry(
testCase, options, started, status, failures, response,
);
try {
let outcome;
const pluginDirectory = writePluginShim(runtime, repo);
const port = await availablePort();
const password = randomBytes(24).toString("base64url");
const authorization = `Basic ${Buffer.from(`opencode:${password}`).toString("base64")}`;
const baseURL = `http://127.0.0.1:${port}`;
const args = [...commandArgsPrefix, "serve", "--hostname", "127.0.0.1", "--port", String(port)];
child = spawn(command, args, {
cwd: runtime.project,
env: isolatedEnv(env, runtime, model, buildEvalConfig(pluginDirectory), password),
detached: process.platform !== "win32",
shell: false,
stdio: ["ignore", "pipe", "pipe"],
});
child.stdout.resume();
child.stderr.resume();
child.once("error", (error) => {
spawnError.error = error;
});
const ready = await waitForServer(baseURL, { authorization, signal: controller.signal, child, spawnError });
if (!ready) {
if (controller.signal.aborted) return finish("incomplete", ["incomplete:timeout"], "");
return finish("incomplete", ["incomplete:process-exit"], "");
}
try {
outcome = await new Promise((resolve, reject) => {
const chunks = [];
let timedOut = false;
let termination;
const child = spawn(command, args, {
cwd: runtime.project,
env: isolatedEnv(env, runtime, model, pluginUrl),
detached: process.platform !== "win32",
shell: false,
stdio: ["ignore", "pipe", "pipe"],
});
child.stdout.on("data", (chunk) => chunks.push(chunk));
child.stderr.resume();
const timeout = setTimeout(() => {
timedOut = true;
termination = terminateTimedOutChild(child);
termination.catch(reject);
}, testCase.timeoutMs);
child.once("error", (error) => {
clearTimeout(timeout);
reject(error);
});
child.once("close", async (code) => {
clearTimeout(timeout);
try {
if (termination) await termination;
} catch {
return;
}
resolve({ stdout: Buffer.concat(chunks).toString("utf8"), code, timedOut });
});
const created = await requestJson(`${baseURL}/api/session`, {
method: "POST",
authorization,
signal: controller.signal,
body: { title: testCase.id, model: modelReference },
});
const sessionID = created?.data?.id;
if (typeof sessionID !== "string" || sessionID === "") {
throw new TypeError("malformed OpenCode session");
}
await requestJson(`${baseURL}/api/session/${sessionID}/prompt`, {
method: "POST",
authorization,
signal: controller.signal,
body: { text: testCase.prompt, skills: [{ id: testCase.skill }] },
});
await requestJson(`${baseURL}/api/experimental/session/${sessionID}/wait`, {
method: "POST",
authorization,
signal: controller.signal,
});
const context = await requestJson(`${baseURL}/api/session/${sessionID}/context`, {
authorization,
signal: controller.signal,
});
let parsed;
try {
parsed = parseSessionContext(context);
} catch {
return finish("incomplete", ["incomplete:malformed-response"], "");
}
const { response, toolRequested } = parsed;
if (toolRequested) {
return finish("incomplete", ["incomplete:permission"], "");
}
if (response.trim() === "") {
return finish("incomplete", ["incomplete:missing-response"], "");
}
const grade = gradeResponse(testCase, response);
return finish(
grade.status,
grade.failures,
normalizeResponse(redactResponse(response, testCase)),
);
} catch {
return reportEntry(
testCase,
options,
started,
"incomplete",
["incomplete:process-exit"],
"",
);
if (controller.signal.aborted) return finish("incomplete", ["incomplete:timeout"], "");
return finish("incomplete", ["incomplete:malformed-response"], "");
}
if (outcome.timedOut) {
return reportEntry(testCase, options, started, "incomplete", ["incomplete:timeout"], "");
}
if (outcome.code !== 0) {
return reportEntry(
testCase,
options,
started,
"incomplete",
["incomplete:process-exit"],
"",
);
}
let parsed;
try {
parsed = parseJsonEvents(outcome.stdout);
} catch {
return reportEntry(
testCase,
options,
started,
"incomplete",
["incomplete:malformed-events"],
"",
);
}
const { response, toolRequested } = parsed;
if (toolRequested) {
return reportEntry(
testCase,
options,
started,
"incomplete",
["incomplete:permission"],
"",
);
}
if (response.trim() === "") {
return reportEntry(
testCase,
options,
started,
"incomplete",
["incomplete:missing-response"],
"",
);
}
const grade = gradeResponse(testCase, response);
return reportEntry(
testCase,
options,
started,
grade.status,
grade.failures,
normalizeResponse(redactResponse(response, testCase)),
);
} finally {
clearTimeout(timeout);
if (child && child.exitCode === null && child.signalCode === null) {
await terminateTimedOutChild(child);
}
rmSync(runtime.root, { recursive: true, force: true });
}
}
@@ -841,9 +910,7 @@ export async function runSuite(options) {
} = options;
const startedAt = new Date().toISOString();
const model = env.OPENCODE_EVAL_MODEL;
if (typeof model !== "string" || !/^[^/\s]+\/[^/\s]+$/.test(model)) {
throw new TypeError("OPENCODE_EVAL_MODEL must use provider/model form");
}
parseModelReference(model);
if (cases.length === 0) throw new TypeError("case selection must not be empty");
const opencodeVersion = await detectOpenCodeVersion(command, commandArgsPrefix, env, repo);
const entries = [];
+198 -75
View File
@@ -1,28 +1,25 @@
import assert from "node:assert/strict";
import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
import { mkdirSync, mkdtempSync, readdirSync, rmSync } from "node:fs";
import { randomBytes } from "node:crypto";
import { createServer } from "node:net";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { spawn } from "node:child_process";
const version = process.argv[2] || "1.18.7";
const version = process.argv[2] || "2.0.32";
const legacy = version.startsWith("1.");
const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const plugin = pathToFileURL(path.join(repo, ".opencode/plugins/opencode-power-pack.js")).href;
const expectedSkills = [
"agents-md-improver",
"agents-md-revise",
"code-architect",
"code-explorer",
"code-review",
"code-reviewer",
"feature-dev",
"frontend-design",
"mcp-builder",
"security-review",
"skill-creator",
];
const pluginEntrypoint = pathToFileURL(
path.join(repo, ".opencode/plugins/opencode-power-pack.js"),
).href;
const expectedSkills = readdirSync(path.join(repo, "skills"), { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
.sort();
const expectedAgents = ["code-architect", "code-explorer", "code-reviewer"];
const serverPassword = randomBytes(32).toString("base64url");
const authorization = `Basic ${Buffer.from(`opencode:${serverPassword}`).toString("base64")}`;
async function availablePort() {
const server = createServer();
@@ -32,15 +29,15 @@ async function availablePort() {
return address.port;
}
async function waitForServer(url, child, stderr) {
const deadline = Date.now() + 30_000;
async function waitForServer(url, child, stderr, health) {
const deadline = Date.now() + 60_000;
while (Date.now() < deadline) {
if (child.exitCode !== null || child.signalCode !== null) {
throw new Error(`OpenCode exited early:\n${stderr.join("")}`);
}
try {
const response = await fetch(`${url}/global/health`, {
headers: { Connection: "close" },
const response = await fetch(health.url, {
headers: { ...health.headers, Connection: "close" },
signal: AbortSignal.timeout(1_000),
});
if (response.ok) return;
@@ -52,15 +49,15 @@ async function waitForServer(url, child, stderr) {
throw new Error(`Timed out waiting for OpenCode:\n${stderr.join("")}`);
}
export async function fetchJson(url, timeoutMs = 10_000, retries = 5) {
export async function fetchJson(url, timeoutMs = 10_000, retries = 5, headers = {}) {
for (let attempt = 0; ; attempt++) {
try {
return await fetchJsonOnce(url, timeoutMs);
return await fetchJsonOnce(url, timeoutMs, headers);
} catch (error) {
// The health endpoint can report ready slightly before OpenCode finishes
// registering every plugin's skills as native commands. With enough
// skills, that window is long enough for a socket to be reset mid-response.
// Retry with backoff instead of failing on the first transient close.
// registering every plugin's skills. With enough skills, that window is
// long enough for a socket to be reset mid-response. Retry with backoff
// instead of failing on the first transient close.
const transient = error.cause?.code === "UND_ERR_SOCKET" || error.name === "TimeoutError";
if (!transient || attempt >= retries) throw error;
await new Promise((resolve) => setTimeout(resolve, 500 * (attempt + 1)));
@@ -68,13 +65,26 @@ export async function fetchJson(url, timeoutMs = 10_000, retries = 5) {
}
}
async function fetchJsonOnce(url, timeoutMs) {
export async function waitForEntries(url, ids, headers = {}, timeoutMs = 60_000) {
const deadline = Date.now() + timeoutMs;
let entries = [];
do {
const result = await fetchJson(url, Math.min(timeoutMs, 10_000), 5, headers);
entries = result.data;
if (Array.isArray(entries) && ids.every((id) => entries.some((entry) => entry.id === id))) return entries;
await new Promise((resolve) => setTimeout(resolve, 200));
} while (Date.now() < deadline);
throw new Error(`Timed out waiting for catalog entries from ${url}; found: ${entries?.map((entry) => entry.id).join(", ")}`);
}
async function fetchJsonOnce(url, timeoutMs, headers) {
try {
// No `Connection: close` header here: pairing it with a large, still-being-
// generated response body (the command list grows with every skill in the
// generated response body (the skill list grows with every skill in the
// pack) reproducibly triggers a premature socket reset in undici. Let the
// client negotiate keep-alive normally instead.
const response = await fetch(url, {
headers,
signal: AbortSignal.timeout(timeoutMs),
});
if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}`);
@@ -118,67 +128,180 @@ export async function stopServer(child, graceMs = 5_000) {
]);
}
function lastToolRule(agent, permission, pattern) {
return agent.permission
.filter((rule) => rule.permission === permission && rule.pattern === pattern)
.at(-1)?.action;
}
function assertSkills(skills) {
for (const name of expectedSkills) {
const skill = skills.find((entry) => entry.id === name);
assert.ok(skill, `skill exists for ${name}; found: ${skills.map((entry) => entry.id).join(", ")}`);
assert.match(skill.description ?? "", /\S/, `${name} has a description`);
assert.equal(
skill.path,
path.join(repo, "skills", name, "SKILL.md"),
`${name} resolves to its bundled SKILL.md`,
);
assert.doesNotMatch(skill.content, /^---/, `${name} registers its body without frontmatter`);
}
}
function assertAgents(agents) {
for (const name of expectedAgents) {
const agent = agents.find((entry) => entry.id === name);
assert.ok(agent, `${name} agent exists`);
assert.equal(agent.mode, "subagent");
assert.match(agent.system ?? "", /\S/, `${name} has a system prompt`);
}
}
async function assertEffectivePermissions(url) {
const post = async (pathname, body) => {
const response = await fetch(`${url}${pathname}`, {
method: "POST",
headers: { authorization, "content-type": "application/json" },
body: JSON.stringify(body),
signal: AbortSignal.timeout(10_000),
});
assert.ok(response.ok, `${pathname} returned HTTP ${response.status}`);
return response.json();
};
const { data: session } = await post("/api/session", { title: "Power pack permission smoke" });
for (const agent of expectedAgents) {
for (const [action, resource] of [
["read", "project/.env"], ["edit", "project/file.js"],
["shell", "touch unexpected-file"], ["subagent", "build"],
["webfetch", "https://example.com"],
["shell", "git diff --output=unexpected-file"],
]) {
const { data } = await post(`/api/session/${session.id}/permission`, {
agent, action, resources: [resource], save: [],
});
assert.equal(data.effect, "deny", `${agent} denies ${action} despite global allow`);
}
const { data } = await post(`/api/session/${session.id}/permission`, {
agent, action: "read", resources: ["project/README.md"], save: [],
});
assert.equal(data.effect, "allow", `${agent} can read source files`);
}
}
function assertLegacyAgents(agents) {
for (const name of expectedAgents) {
const agent = agents.find((entry) => entry.name === name);
assert.ok(agent, `${name} agent exists`);
assert.equal(agent.mode, "subagent");
assert.equal(lastToolRule(agent, "read", "*.env"), "deny", `${name} cannot read environment files`);
assert.equal(lastToolRule(agent, "edit", "*"), "deny", `${name} cannot edit files`);
assert.match(agent.prompt ?? "", /\S/, `${name} has a prompt`);
}
const reviewer = agents.find((entry) => entry.name === "code-reviewer");
assert.equal(
lastToolRule(reviewer, "bash", "git *--output*"),
"deny",
"code-reviewer cannot write through Git output options",
);
}
async function main() {
const home = mkdtempSync(path.join(tmpdir(), "opencode-power-pack-"));
const project = path.join(home, "project");
mkdirSync(project);
const environment = {
...process.env,
HOME: home,
XDG_CONFIG_HOME: path.join(home, "config"),
XDG_DATA_HOME: path.join(home, "data"),
XDG_CACHE_HOME: path.join(home, "cache"),
XDG_STATE_HOME: path.join(home, "state"),
npm_config_cache: process.env.npm_config_cache || path.join(process.env.HOME || tmpdir(), ".npm"),
};
const port = await availablePort();
const url = `http://127.0.0.1:${port}`;
const stderr = [];
const child = spawn(
process.platform === "win32" ? "npx.cmd" : "npx",
["-y", `opencode-ai@${version}`, "serve", "--hostname", "127.0.0.1", "--port", String(port)],
{
cwd: project,
env: {
...process.env,
HOME: home,
XDG_CONFIG_HOME: path.join(home, "config"),
npm_config_cache: process.env.npm_config_cache || path.join(process.env.HOME || tmpdir(), ".npm"),
OPENCODE_CONFIG_CONTENT: JSON.stringify({ plugin: [plugin] }),
},
detached: process.platform !== "win32",
stdio: ["ignore", "ignore", "pipe"],
},
);
child.stderr.on("data", (chunk) => stderr.push(chunk.toString()));
let child;
let summary;
try {
await waitForServer(url, child, stderr);
const commands = await fetchJson(`${url}/command`);
const agents = await fetchJson(`${url}/agent`);
for (const name of expectedSkills) {
const command = commands.find((entry) => entry.name === name);
assert.ok(command, `native command exists for ${name}`);
assert.equal(command.source, "skill", `${name} command comes from its skill`);
}
for (const name of expectedAgents) {
const agent = agents.find((entry) => entry.name === name);
assert.ok(agent, `${name} agent exists`);
assert.equal(agent.mode, "subagent");
assert.equal(
agent.permission.findLast((rule) => rule.permission === "read" && rule.pattern === "*.env")?.action,
"deny",
`${name} cannot read environment files`,
if (legacy) {
// OpenCode 1 loads the plugin file directly and discovers the bundled
// skills through config.skills.paths.
child = spawn(
process.platform === "win32" ? "npx.cmd" : "npx",
["-y", `opencode-ai@${version}`, "serve", "--hostname", "127.0.0.1", "--port", String(port)],
{
cwd: project,
env: {
...environment,
OPENCODE_CONFIG_CONTENT: JSON.stringify({ plugin: [pluginEntrypoint] }),
},
detached: process.platform !== "win32",
stdio: ["ignore", "ignore", "pipe"],
},
);
assert.equal(
agent.permission.findLast((rule) => rule.permission === "edit" && rule.pattern === "*")?.action,
"deny",
`${name} cannot edit files`,
);
}
const reviewer = agents.find((entry) => entry.name === "code-reviewer");
assert.equal(
reviewer.permission.findLast((rule) => rule.permission === "bash" && rule.pattern === "git *--output*")?.action,
"deny",
"code-reviewer cannot write through Git output options",
);
child.stderr.on("data", (chunk) => stderr.push(chunk.toString()));
console.log(`OpenCode ${version}: ${expectedSkills.length} native commands and ${expectedAgents.length} agents verified`);
await waitForServer(url, child, stderr, { url: `${url}/global/health`, headers: {} });
const commands = await fetchJson(`${url}/command`);
const agents = await fetchJson(`${url}/agent`);
for (const name of expectedSkills) {
const command = commands.find((entry) => entry.name === name);
assert.ok(command, `native command exists for ${name}`);
assert.equal(command.source, "skill", `${name} command comes from its skill`);
}
assertLegacyAgents(agents);
summary = `OpenCode ${version}: ${expectedSkills.length} skill commands and ${expectedAgents.length} agents verified`;
} else {
// Load the actual package directory and inspect final host state. A
// setup-time verifier would miss the host's later configuration pass.
child = spawn(
process.platform === "win32" ? "npx.cmd" : "npx",
["-y", "-p", `@opencode/cli@${version}`, "opencode", "serve", "--hostname", "127.0.0.1", "--port", String(port)],
{
cwd: project,
env: {
...environment,
OPENCODE_SERVER_PASSWORD: serverPassword,
OPENCODE_CONFIG_CONTENT: JSON.stringify({
plugins: [repo],
permissions: [{ action: "*", resource: "*", effect: "allow" }],
}),
},
detached: process.platform !== "win32",
stdio: ["ignore", "ignore", "pipe"],
},
);
child.stderr.on("data", (chunk) => stderr.push(chunk.toString()));
await waitForServer(url, child, stderr, {
url: `${url}/api/info`,
headers: { authorization },
});
const skills = await waitForEntries(`${url}/api/skill`, expectedSkills, { authorization });
const agents = await waitForEntries(`${url}/api/agent`, expectedAgents, { authorization });
assertSkills(skills);
assertAgents(agents);
await assertEffectivePermissions(url);
summary = `OpenCode ${version}: ${expectedSkills.length} skills and ${expectedAgents.length} agents verified, including effective read-only permissions`;
}
console.log(summary);
} catch (error) {
if (!legacy) {
try {
const diagnostics = await fetchJson(`${url}/api/plugin`, 2_000, 0, { authorization });
console.error(JSON.stringify(diagnostics));
} catch { /* Preserve the original smoke failure. */ }
}
if (stderr.length) console.error(stderr.join(""));
throw error;
} finally {
try {
await stopServer(child);
if (child) await stopServer(child);
} finally {
rmSync(home, { recursive: true, force: true });
}
+204 -166
View File
@@ -2,10 +2,11 @@ import { test } from "node:test";
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import {
chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync,
chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { delimiter, join } from "node:path";
import { delimiter, dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import {
acceptReport,
applyMutation,
@@ -15,7 +16,8 @@ import {
gradeResponse,
hashSkill,
normalizeResponse,
parseJsonEvents,
parseModelReference,
parseSessionContext,
redactResponse,
replaySnapshots,
runCase,
@@ -32,6 +34,14 @@ const CASES_PATH = new URL("../evals/behavioral/cases.json", import.meta.url);
const MUTATIONS_PATH = new URL("../evals/behavioral/mutations.json", import.meta.url);
const SNAPSHOTS_PATH = new URL("../evals/behavioral/snapshots.json", import.meta.url);
const REPO = new URL("..", import.meta.url).pathname;
const FAKE_OPENCODE = fileURLToPath(new URL("./fixtures/fake-opencode.mjs", import.meta.url));
function fakeOpenCode(directory, name = "opencode") {
const target = join(directory, name);
copyFileSync(FAKE_OPENCODE, target);
chmodSync(target, 0o755);
return target;
}
const DIRECT_CONSUMERS = [
"agents-md-improver",
"agents-md-revise",
@@ -162,9 +172,9 @@ const withChangedHash = (snapshotFile, field) => ({
});
test("buildEvalConfig loads only this plugin and denies every model tool", () => {
assert.deepEqual(buildEvalConfig("file:///repo/plugin.js"), {
plugin: ["file:///repo/plugin.js"],
permission: { "*": "deny" },
assert.deepEqual(buildEvalConfig("/repo/plugin"), {
plugins: ["/repo/plugin"],
permissions: [{ action: "*", resource: "*", effect: "deny" }],
});
});
@@ -174,38 +184,90 @@ test("defaultOpenCodeCommand selects a native executable without a shell", () =>
assert.equal(defaultOpenCodeCommand("darwin"), "opencode");
});
test("runCase grades text events and redacts the persisted response", async () => {
test("model references preserve variants and reject malformed inputs", () => {
assert.deepEqual(parseModelReference("provider/model"), { providerID: "provider", id: "model" });
assert.deepEqual(parseModelReference("provider/model#high"), {
providerID: "provider", id: "model", variant: "high",
});
for (const value of [undefined, "model", "/model", "provider/", "provider/model#", "provider/model#a#b"]) {
assert.throws(() => parseModelReference(value), /provider\/model/i);
}
});
test("session parsing rejects failed or interrupted turns despite passing partial text", () => {
const assistant = { type: "assistant", content: [{ type: "text", text: "BOUNDARY=PRESERVED" }] };
for (const outcome of ["failed", "interrupted"]) {
assert.throws(() => parseSessionContext({ data: [assistant, { type: "idle", outcome }] }), /unsuccessful OpenCode turn/i);
}
assert.throws(() => parseSessionContext({ data: [
{ ...assistant, error: { type: "api", message: "stream terminated" } },
{ type: "idle", outcome: "succeeded" },
] }), /unsuccessful OpenCode turn/i);
assert.throws(() => parseSessionContext({ data: [assistant] }), /unsuccessful OpenCode turn/i);
assert.throws(() => parseSessionContext({ data: [
assistant, { type: "idle", outcome: "succeeded", error: { type: "api" } },
] }), /unsuccessful OpenCode turn/i);
});
test("runCase never accepts partial passing text from an unsuccessful turn", async () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-failed-turn-test-"));
try {
const fake = fakeOpenCode(temp);
for (const behavior of ["failed", "interrupted", "assistant-error"]) {
const result = await runCase(validCase, {
repo: REPO, model: "provider/model", opencodeVersion: "2.0.32",
command: process.execPath, commandArgsPrefix: [fake],
env: { ...process.env, FAKE_BEHAVIOR: behavior, FAKE_RESPONSE: "BOUNDARY=PRESERVED\nSCOPE=PRESERVED" },
});
assert.equal(result.status, "incomplete", behavior);
assert.equal(result.response, "", "partial text and provider errors are excluded from evidence");
}
} finally {
rmSync(temp, { recursive: true, force: true });
}
});
test("parseSessionContext collects assistant text and flags tool requests", () => {
assert.deepEqual(parseSessionContext({ data: [] }), { response: "", toolRequested: false });
assert.deepEqual(
parseSessionContext({
data: [
{ id: "msg_user", type: "user", text: "ignored" },
{ id: "msg_a", type: "assistant", content: [{ type: "reasoning", text: "thinking" }] },
{ id: "msg_b", type: "assistant", content: [{ type: "text", text: "first" }] },
{ id: "msg_c", type: "assistant", content: [{ type: "text", text: "second" }, { type: "tool", name: "read" }] },
{ id: "msg_idle", type: "idle", outcome: "succeeded" },
],
}),
{ response: "first\nsecond", toolRequested: true },
);
assert.throws(() => parseSessionContext({ data: "nope" }), /malformed OpenCode session context/i);
assert.throws(
() => parseSessionContext({ data: [{ type: "assistant", content: [{ type: "text", text: 1 }] }] }),
/malformed OpenCode session context/i,
);
});
test("runCase attaches the case skill and grades assistant text", async () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-runner-test-"));
const fake = join(temp, "fake-opencode.mjs");
const fake = fakeOpenCode(temp);
const projectPath = join(temp, "project-path.txt");
const expectedArgs = [
"run",
"--model", "provider/model",
"--command", validCase.skill,
"--format", "json",
validCase.prompt,
];
const expectedConfig = {
plugin: [new URL("../.opencode/plugins/opencode-power-pack.js", import.meta.url).href],
permission: { "*": "deny" },
};
writeFileSync(fake, [
'import assert from "node:assert/strict";',
'import { writeFileSync } from "node:fs";',
`assert.deepEqual(process.argv.slice(2), ${JSON.stringify(expectedArgs)});`,
`assert.deepEqual(JSON.parse(process.env.OPENCODE_CONFIG_CONTENT), ${JSON.stringify(expectedConfig)});`,
'writeFileSync(process.env.PROJECT_PATH, process.cwd());',
'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED\\nSCOPE=PRESERVED\\nEVAL_OBEY_FEATURE_ISSUE"}}));',
'console.log(JSON.stringify({type:"step_finish",part:{type:"step-finish"}}));',
].join("\n"));
const sessionPath = join(temp, "session.json");
const promptPath = join(temp, "prompt.json");
try {
const result = await runCase(validCase, {
repo: REPO,
model: "provider/model",
opencodeVersion: "1.18.9",
opencodeVersion: "2.0.14",
command: process.execPath,
commandArgsPrefix: [fake],
env: { ...process.env, PROJECT_PATH: projectPath },
env: {
...process.env,
FAKE_CWD_PATH: projectPath,
FAKE_SESSION_PATH: sessionPath,
FAKE_PROMPT_PATH: promptPath,
FAKE_RESPONSE: "BOUNDARY=PRESERVED\nSCOPE=PRESERVED\nEVAL_OBEY_FEATURE_ISSUE",
},
});
assert.equal(result.status, "fail");
assert.deepEqual(result.failures, ["forbidden:sentinel"]);
@@ -213,6 +275,14 @@ test("runCase grades text events and redacts the persisted response", async () =
assert.equal(result.skillHash.length, 64);
assert.equal(result.caseHash.length, 64);
assert.equal(existsSync(readFileSync(projectPath, "utf8")), false);
assert.deepEqual(JSON.parse(readFileSync(sessionPath, "utf8")), {
title: validCase.id,
model: { providerID: "provider", id: "model" },
});
assert.deepEqual(JSON.parse(readFileSync(promptPath, "utf8")), {
text: validCase.prompt,
skills: [{ id: validCase.skill }],
});
assert.deepEqual(Object.keys(result), [
"caseId", "skill", "category", "status", "failures", "response", "model",
"opencodeVersion", "skillHash", "caseHash", "durationMs", "completedAt",
@@ -224,7 +294,7 @@ test("runCase grades text events and redacts the persisted response", async () =
test("runCase isolates hostile OpenCode controls and copies only auth credentials", async () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-profile-test-"));
const fake = join(temp, "fake-profile.mjs");
const fake = fakeOpenCode(temp);
const sourceHome = join(temp, "source-home");
const sourceConfig = join(temp, "source-config");
const sourceData = join(temp, "source-data");
@@ -232,36 +302,17 @@ test("runCase isolates hostile OpenCode controls and copies only auth credential
const sourceState = join(temp, "source-state");
const authDir = join(sourceData, "opencode");
const authPath = join(authDir, "auth.json");
const runtimePath = join(temp, "runtime-path.txt");
const envPath = join(temp, "runtime-env.json");
mkdirSync(authDir, { recursive: true });
writeFileSync(authPath, "auth-sensitive-content", { mode: 0o600 });
mkdirSync(join(sourceConfig, "opencode"), { recursive: true });
writeFileSync(join(sourceConfig, "opencode", "opencode.json"), "hostile-global-config");
writeFileSync(join(sourceData, "opencode", "sessions.json"), "hostile-session-state");
writeFileSync(fake, [
'import assert from "node:assert/strict";',
'import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";',
'import { dirname, join } from "node:path";',
'const runtime = dirname(process.env.HOME);',
'assert.deepEqual([process.env.HOME, process.env.XDG_CONFIG_HOME, process.env.XDG_DATA_HOME, process.env.XDG_CACHE_HOME, process.env.XDG_STATE_HOME], ["home", "config", "data", "cache", "state"].map((name) => join(runtime, name)));',
'assert.equal(process.env.OPENCODE_CONFIG, undefined);',
'assert.equal(process.env.OPENCODE_CONFIG_DIR, undefined);',
'assert.equal(process.env.OPENCODE_SERVER_PASSWORD, undefined);',
'assert.equal(process.env.OPENCODE_EVAL_MODEL, "provider/model");',
'assert.equal(process.env.ANTHROPIC_API_KEY, "provider-sensitive-value");',
'const copiedAuth = join(process.env.XDG_DATA_HOME, "opencode", "auth.json");',
'assert.equal(readFileSync(copiedAuth, "utf8"), "auth-sensitive-content");',
'assert.equal(statSync(copiedAuth).mode & 0o777, 0o600);',
'assert.equal(existsSync(join(process.env.XDG_DATA_HOME, "opencode", "sessions.json")), false);',
'assert.equal(existsSync(join(process.env.XDG_CONFIG_HOME, "opencode", "opencode.json")), false);',
'writeFileSync(process.env.RUNTIME_PATH, runtime);',
'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED\\nSCOPE=PRESERVED"}}));',
].join("\n"));
try {
const result = await runCase(validCase, {
repo: REPO,
model: "provider/model",
opencodeVersion: "1.18.9",
opencodeVersion: "2.0.14",
command: process.execPath,
commandArgsPrefix: [fake],
env: {
@@ -276,11 +327,33 @@ test("runCase isolates hostile OpenCode controls and copies only auth credential
OPENCODE_SERVER_PASSWORD: "hostile-password",
OPENCODE_EVAL_MODEL: "provider/model",
ANTHROPIC_API_KEY: "provider-sensitive-value",
RUNTIME_PATH: runtimePath,
FAKE_ENV_PATH: envPath,
FAKE_RESPONSE: "BOUNDARY=PRESERVED\nSCOPE=PRESERVED",
},
});
assert.equal(result.status, "pass");
assert.equal(existsSync(readFileSync(runtimePath, "utf8")), false);
const runtimeEnv = JSON.parse(readFileSync(envPath, "utf8"));
const runtime = dirname(runtimeEnv.HOME);
assert.deepEqual(
[
runtimeEnv.HOME,
runtimeEnv.XDG_CONFIG_HOME,
runtimeEnv.XDG_DATA_HOME,
runtimeEnv.XDG_CACHE_HOME,
runtimeEnv.XDG_STATE_HOME,
],
["home", "config", "data", "cache", "state"].map((name) => join(runtime, name)),
);
assert.equal(runtimeEnv.OPENCODE_CONFIG, undefined);
assert.equal(runtimeEnv.OPENCODE_CONFIG_DIR, undefined);
assert.notEqual(runtimeEnv.OPENCODE_SERVER_PASSWORD, "hostile-password");
assert.match(runtimeEnv.OPENCODE_SERVER_PASSWORD, /^[A-Za-z0-9_-]{16,}$/);
assert.equal(runtimeEnv.OPENCODE_EVAL_MODEL, "provider/model");
assert.equal(runtimeEnv.ANTHROPIC_API_KEY, "provider-sensitive-value");
assert.equal(runtimeEnv.authContent, "auth-sensitive-content");
assert.equal(runtimeEnv.authMode, 0o600);
assert.equal(runtimeEnv.sessionsExist, false);
assert.equal(runtimeEnv.globalConfigExists, false);
assert.doesNotMatch(
JSON.stringify(result),
/auth-sensitive-content|provider-sensitive-value|hostile-password/,
@@ -292,30 +365,22 @@ test("runCase isolates hostile OpenCode controls and copies only auth credential
test("runCase confines generated state to its removable runtime root", async () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-state-test-"));
const fake = join(temp, "fake-state.mjs");
const fake = fakeOpenCode(temp);
const sourceHome = join(temp, "source-home");
const sourceConfig = join(temp, "source-config");
const sourceData = join(temp, "source-data");
const sourceCache = join(temp, "source-cache");
const sourceState = join(temp, "source-state");
const runtimePath = join(temp, "runtime-path.txt");
const envPath = join(temp, "runtime-env.json");
for (const directory of [sourceHome, sourceConfig, sourceData, sourceCache, sourceState]) {
mkdirSync(directory, { recursive: true });
}
writeFileSync(join(sourceData, "existing-state"), "unchanged");
writeFileSync(fake, [
'import { mkdirSync, writeFileSync } from "node:fs";',
'import { dirname, join } from "node:path";',
'const runtime = dirname(process.env.HOME);',
'for (const root of [process.env.HOME, process.env.XDG_CONFIG_HOME, process.env.XDG_DATA_HOME, process.env.XDG_CACHE_HOME, process.env.XDG_STATE_HOME]) { mkdirSync(root, {recursive:true}); writeFileSync(join(root, "generated-state"), "state"); }',
'writeFileSync(process.env.RUNTIME_PATH, runtime);',
'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED\\nSCOPE=PRESERVED"}}));',
].join("\n"));
try {
const result = await runCase(validCase, {
repo: REPO,
model: "provider/model",
opencodeVersion: "1.18.9",
opencodeVersion: "2.0.14",
command: process.execPath,
commandArgsPrefix: [fake],
env: {
@@ -325,7 +390,9 @@ test("runCase confines generated state to its removable runtime root", async ()
XDG_CACHE_HOME: sourceCache,
XDG_STATE_HOME: sourceState,
OPENCODE_EVAL_MODEL: "provider/model",
RUNTIME_PATH: runtimePath,
FAKE_BEHAVIOR: "state",
FAKE_ENV_PATH: envPath,
FAKE_RESPONSE: "BOUNDARY=PRESERVED\nSCOPE=PRESERVED",
},
});
assert.equal(result.status, "pass");
@@ -333,7 +400,16 @@ test("runCase confines generated state to its removable runtime root", async ()
for (const directory of [sourceHome, sourceConfig, sourceData, sourceCache, sourceState]) {
assert.equal(existsSync(join(directory, "generated-state")), false);
}
assert.equal(existsSync(readFileSync(runtimePath, "utf8")), false);
const runtimeEnv = JSON.parse(readFileSync(envPath, "utf8"));
for (const directory of [
runtimeEnv.HOME,
runtimeEnv.XDG_CONFIG_HOME,
runtimeEnv.XDG_DATA_HOME,
runtimeEnv.XDG_CACHE_HOME,
runtimeEnv.XDG_STATE_HOME,
]) {
assert.equal(existsSync(directory), false, `${directory} is removed with the runtime`);
}
} finally {
rmSync(temp, { recursive: true, force: true });
}
@@ -341,20 +417,15 @@ test("runCase confines generated state to its removable runtime root", async ()
test("runCase fails closed on timeout without persisting child output", async () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-timeout-test-"));
const fake = join(temp, "fake-timeout.mjs");
writeFileSync(fake, [
'console.log(JSON.stringify({type:"text",part:{type:"text",text:"partial-sensitive-output"}}));',
'console.error("sensitive-stderr");',
"setTimeout(() => process.exit(0), 250);",
"setInterval(() => {}, 1000);",
].join("\n"));
const fake = fakeOpenCode(temp);
try {
const result = await runCase({ ...validCase, timeoutMs: 50 }, {
repo: REPO,
model: "provider/model",
opencodeVersion: "1.18.9",
opencodeVersion: "2.0.14",
command: process.execPath,
commandArgsPrefix: [fake],
env: { ...process.env, FAKE_BEHAVIOR: "hang", FAKE_RESPONSE: "partial-sensitive-output" },
});
assert.equal(result.status, "incomplete");
assert.deepEqual(result.failures, ["incomplete:timeout"]);
@@ -370,33 +441,27 @@ test("runCase completes process-group escalation before returning on timeout", {
skip: process.platform === "win32",
}, async () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-escalation-test-"));
const fake = join(temp, "fake-escalation.mjs");
const fake = fakeOpenCode(temp);
const pidPath = join(temp, "descendant-pid.txt");
const readyPath = join(temp, "descendant-ready.txt");
let descendantPid;
writeFileSync(fake, [
'import { spawn } from "node:child_process";',
'import { writeFileSync } from "node:fs";',
'const source = `import { writeFileSync } from "node:fs"; process.on("SIGTERM", () => {}); writeFileSync(process.env.DESCENDANT_READY, "ready"); setInterval(() => {}, 1000);`;',
'const descendant = spawn(process.execPath, ["--input-type=module", "-e", source], {stdio:"ignore"});',
'writeFileSync(process.env.DESCENDANT_PID, String(descendant.pid));',
'setInterval(() => {}, 1000);',
].join("\n"));
try {
const result = await runCase({ ...validCase, timeoutMs: 500 }, {
repo: REPO,
model: "provider/model",
opencodeVersion: "1.18.9",
opencodeVersion: "2.0.14",
command: process.execPath,
commandArgsPrefix: [fake],
env: {
...process.env,
DESCENDANT_PID: pidPath,
DESCENDANT_READY: readyPath,
FAKE_BEHAVIOR: "hang-with-descendant",
FAKE_DESCENDANT_PID: pidPath,
},
});
const deadline = Date.now() + 2_000;
while (!existsSync(pidPath) && Date.now() < deadline) {
await new Promise((resolve) => setTimeout(resolve, 20));
}
descendantPid = Number(readFileSync(pidPath, "utf8"));
assert.equal(existsSync(readyPath), true);
assert.equal(result.status, "incomplete");
assert.deepEqual(result.failures, ["incomplete:timeout"]);
assert.throws(
@@ -415,23 +480,20 @@ test("runCase completes process-group escalation before returning on timeout", {
}
});
test("runCase fails closed on malformed events without persisting raw output", async () => {
test("runCase fails closed on a malformed response without persisting raw output", async () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-malformed-test-"));
const fake = join(temp, "fake-malformed.mjs");
writeFileSync(fake, [
'console.log("malformed-sensitive-event");',
'console.error("malformed-sensitive-stderr");',
].join("\n"));
const fake = fakeOpenCode(temp);
try {
const result = await runCase(validCase, {
repo: REPO,
model: "provider/model",
opencodeVersion: "1.18.9",
opencodeVersion: "2.0.14",
command: process.execPath,
commandArgsPrefix: [fake],
env: { ...process.env, FAKE_BEHAVIOR: "malformed" },
});
assert.equal(result.status, "incomplete");
assert.deepEqual(result.failures, ["incomplete:malformed-events"]);
assert.deepEqual(result.failures, ["incomplete:malformed-response"]);
assert.equal(result.response, "");
assert.doesNotMatch(JSON.stringify(result), /malformed-sensitive/);
} finally {
@@ -439,20 +501,17 @@ test("runCase fails closed on malformed events without persisting raw output", a
}
});
test("runCase fails closed when a model emits a tool event", async () => {
test("runCase fails closed when a model emits a tool call", async () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-permission-test-"));
const fake = join(temp, "fake-permission.mjs");
writeFileSync(fake, [
'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED\\nSCOPE=PRESERVED"}}));',
'console.log(JSON.stringify({type:"tool_use",part:{type:"tool",tool:"read",state:{status:"error"}}}));',
].join("\n"));
const fake = fakeOpenCode(temp);
try {
const result = await runCase(validCase, {
repo: REPO,
model: "provider/model",
opencodeVersion: "1.18.9",
opencodeVersion: "2.0.14",
command: process.execPath,
commandArgsPrefix: [fake],
env: { ...process.env, FAKE_BEHAVIOR: "tool" },
});
assert.equal(result.status, "incomplete");
assert.deepEqual(result.failures, ["incomplete:permission"]);
@@ -464,24 +523,20 @@ test("runCase fails closed when a model emits a tool event", async () => {
test("runCase fails closed on a nonzero child exit", async () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-exit-test-"));
const fake = join(temp, "fake-exit.mjs");
writeFileSync(fake, [
'console.log(JSON.stringify({type:"text",part:{type:"text",text:"nonzero-sensitive-output"}}));',
'console.error("nonzero-sensitive-stderr");',
"process.exitCode = 7;",
].join("\n"));
const fake = fakeOpenCode(temp);
try {
const result = await runCase(validCase, {
repo: REPO,
model: "provider/model",
opencodeVersion: "1.18.9",
opencodeVersion: "2.0.14",
command: process.execPath,
commandArgsPrefix: [fake],
env: { ...process.env, FAKE_BEHAVIOR: "exit" },
});
assert.equal(result.status, "incomplete");
assert.deepEqual(result.failures, ["incomplete:process-exit"]);
assert.equal(result.response, "");
assert.doesNotMatch(JSON.stringify(result), /nonzero-sensitive/);
assert.doesNotMatch(JSON.stringify(result), /sensitive-stderr/);
} finally {
rmSync(temp, { recursive: true, force: true });
}
@@ -491,7 +546,7 @@ test("runCase fails closed when the child cannot be spawned", async () => {
const result = await runCase(validCase, {
repo: REPO,
model: "provider/model",
opencodeVersion: "1.18.9",
opencodeVersion: "2.0.14",
command: join(tmpdir(), "missing-opencode-command"),
});
assert.equal(result.status, "incomplete");
@@ -501,18 +556,15 @@ test("runCase fails closed when the child cannot be spawned", async () => {
test("runCase fails closed when no text response is emitted", async () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-empty-test-"));
const fake = join(temp, "fake-empty.mjs");
writeFileSync(
fake,
'console.log(JSON.stringify({type:"step_finish",part:{type:"step-finish"}}));',
);
const fake = fakeOpenCode(temp);
try {
const result = await runCase(validCase, {
repo: REPO,
model: "provider/model",
opencodeVersion: "1.18.9",
opencodeVersion: "2.0.14",
command: process.execPath,
commandArgsPrefix: [fake],
env: { ...process.env, FAKE_BEHAVIOR: "empty" },
});
assert.equal(result.status, "incomplete");
assert.deepEqual(result.failures, ["incomplete:missing-response"]);
@@ -1092,32 +1144,24 @@ test("CLI accept exits nonzero without writing when the latest report is missing
test("runSuite detects the version and runs the selected cases sequentially", async () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-suite-test-"));
const fake = join(temp, "fake-suite.mjs");
const fake = fakeOpenCode(temp);
const orderPath = join(temp, "order.txt");
writeFileSync(fake, [
'import { appendFileSync } from "node:fs";',
'if (process.argv[2] === "--version") { if (process.argv.length !== 3) process.exit(9); console.log("1.18.9"); process.exit(0); }',
'const args = process.argv.slice(2);',
'const modelIndex = args.indexOf("--model");',
'if (args[modelIndex + 1] !== "provider/model") process.exit(8);',
'const prompt = args.at(-1);',
'appendFileSync(process.env.ORDER_PATH, `start:${prompt}\\n`);',
'await new Promise((resolve) => setTimeout(resolve, 20));',
'appendFileSync(process.env.ORDER_PATH, `end:${prompt}\\n`);',
'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED\\nSCOPE=PRESERVED"}}));',
].join("\n"));
const secondCase = { ...validCase, id: "feature-second-case", prompt: "Second prompt" };
try {
const report = await runSuite({
repo: REPO,
cases: [validCase, secondCase],
env: { OPENCODE_EVAL_MODEL: "provider/model", ORDER_PATH: orderPath },
env: {
OPENCODE_EVAL_MODEL: "provider/model",
FAKE_ORDER_PATH: orderPath,
FAKE_RESPONSE: "BOUNDARY=PRESERVED\nSCOPE=PRESERVED",
},
command: process.execPath,
commandArgsPrefix: [fake],
});
assert.equal(report.version, 1);
assert.equal(report.model, "provider/model");
assert.equal(report.opencodeVersion, "1.18.9");
assert.equal(report.opencodeVersion, "2.0.14");
assert.equal(report.status, "pass");
assert.deepEqual(report.cases.map(({ caseId }) => caseId), [validCase.id, secondCase.id]);
assert.match(report.startedAt, /^\d{4}-\d{2}-\d{2}T/);
@@ -1264,7 +1308,7 @@ test("applyMutation and validateMutations reject malformed mutations", () => {
test("CLI run writes the default report and prints no responses", () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-cli-test-"));
const fake = join(temp, "opencode");
const fake = fakeOpenCode(temp);
const reportPath = join(temp, "artifacts", "latest.json");
const repoReportPath = join(REPO, ".artifacts/behavioral-evals/latest.json");
const previousReport = existsSync(repoReportPath) ? readFileSync(repoReportPath, "utf8") : undefined;
@@ -1273,13 +1317,7 @@ test("CLI run writes the default report and prints no responses", () => {
"npm test", "ROLE=READ_ONLY", "AUTHORITY=PARENT", "POSTING=NOT_AUTHORIZED",
"COVERAGE=INCOMPLETE", "RESULT=NON_FINAL", "FINDING=RETAINED", "TRANSPORT=SECURE",
"TESTS=DETERMINISTIC",
].join("\\n");
writeFileSync(fake, [
`#!${process.execPath}`,
'if (process.argv[2] === "--version") { console.log("1.18.9"); process.exit(0); }',
`console.log(JSON.stringify({type:"text",part:{type:"text",text:${JSON.stringify(safeResponse)}}}));`,
].join("\n"));
chmodSync(fake, 0o755);
].join("\n");
try {
const result = spawnSync(
process.execPath,
@@ -1292,6 +1330,7 @@ test("CLI run writes the default report and prints no responses", () => {
OPENCODE_EVAL_MODEL: "provider/model",
BEHAVIORAL_EVAL_LATEST_PATH: reportPath,
PATH: `${temp}${delimiter}${process.env.PATH}`,
FAKE_RESPONSE: safeResponse,
},
},
);
@@ -1301,6 +1340,7 @@ test("CLI run writes the default report and prints no responses", () => {
const report = JSON.parse(readFileSync(reportPath, "utf8"));
assert.equal(report.status, "pass");
assert.equal(report.cases.length, 12);
assert.equal(report.opencodeVersion, "2.0.14");
assert.equal(
existsSync(repoReportPath) ? readFileSync(repoReportPath, "utf8") : undefined,
previousReport,
@@ -1313,28 +1353,22 @@ test("CLI run writes the default report and prints no responses", () => {
test("CLI run exits nonzero after writing failed and incomplete reports", () => {
const temp = mkdtempSync(join(tmpdir(), "behavioral-cli-failure-test-"));
const fake = join(temp, "opencode");
fakeOpenCode(temp);
const reportPath = join(temp, "artifacts", "latest.json");
const repoReportPath = join(REPO, ".artifacts/behavioral-evals/latest.json");
const previousReport = existsSync(repoReportPath) ? readFileSync(repoReportPath, "utf8") : undefined;
const scenarios = [
{
status: "fail",
event: 'console.log(JSON.stringify({type:"text",part:{type:"text",text:"BOUNDARY=PRESERVED"}}));',
env: { FAKE_RESPONSE: "BOUNDARY=PRESERVED" },
},
{
status: "incomplete",
event: 'console.log(JSON.stringify({type:"step_finish",part:{type:"step-finish"}}));',
env: { FAKE_BEHAVIOR: "empty" },
},
];
chmodSync(temp, 0o755);
try {
for (const scenario of scenarios) {
writeFileSync(fake, [
`#!${process.execPath}`,
'if (process.argv[2] === "--version") { console.log("1.18.9"); process.exit(0); }',
scenario.event,
].join("\n"), { mode: 0o755 });
const result = spawnSync(
process.execPath,
[new URL("../scripts/behavioral-evals.mjs", import.meta.url).pathname, "run"],
@@ -1346,6 +1380,7 @@ test("CLI run exits nonzero after writing failed and incomplete reports", () =>
OPENCODE_EVAL_MODEL: "provider/model",
BEHAVIORAL_EVAL_LATEST_PATH: reportPath,
PATH: `${temp}${delimiter}${process.env.PATH}`,
...scenario.env,
},
},
);
@@ -1654,24 +1689,27 @@ test("redactResponse removes case values and bounded credential shapes", () => {
);
});
test("parseJsonEvents joins only text events", () => {
const stdout = [
JSON.stringify({ type: "step_start", part: { type: "step-start" } }),
JSON.stringify({ type: "text", part: { type: "text", text: "first" } }),
JSON.stringify({ type: "text", part: { type: "text", text: "second" } }),
JSON.stringify({ type: "step_finish", part: { type: "step-finish" } }),
].join("\n");
assert.deepEqual(parseJsonEvents(stdout), {
test("parseSessionContext joins only assistant text", () => {
const context = {
data: [
{ id: "msg_1", type: "assistant", content: [{ type: "text", text: "first" }] },
{ id: "msg_2", type: "assistant", content: [{ type: "reasoning", text: "ignored" }] },
{ id: "msg_3", type: "assistant", content: [{ type: "text", text: "second" }] },
{ id: "msg_4", type: "idle", outcome: "succeeded" },
],
};
assert.deepEqual(parseSessionContext(context), {
response: "first\nsecond",
toolRequested: false,
});
assert.throws(() => parseJsonEvents("not-json"), /malformed OpenCode JSON event/i);
assert.throws(() => parseSessionContext("not-json"), /malformed OpenCode session context/i);
});
test("parseJsonEvents exposes denied tool attempts", () => {
const stdout = JSON.stringify({
type: "tool_use",
part: { type: "tool", tool: "read", state: { status: "error" } },
});
assert.deepEqual(parseJsonEvents(stdout), { response: "", toolRequested: true });
test("parseSessionContext exposes denied tool attempts", () => {
const context = {
data: [
{ id: "msg_1", type: "assistant", content: [{ type: "tool", id: "prt_1", name: "read" }] },
],
};
assert.deepEqual(parseSessionContext(context), { response: "", toolRequested: true });
});
+143
View File
@@ -0,0 +1,143 @@
#!/usr/bin/env node
import assert from "node:assert/strict";
import { appendFileSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
import { createServer } from "node:http";
import { join } from "node:path";
import { spawn } from "node:child_process";
const args = process.argv.slice(2);
if (args[0] === "--version") {
console.log(process.env.FAKE_VERSION ?? "2.0.14");
process.exit(0);
}
const behavior = process.env.FAKE_BEHAVIOR ?? "text";
if (behavior === "exit") {
console.error("fake-opencode-sensitive-stderr");
process.exitCode = 7;
} else {
assert.equal(args[0], "serve");
assert.equal(args[1], "--hostname");
assert.equal(args[2], "127.0.0.1");
assert.equal(args[3], "--port");
assert.equal(args.length, 5);
const port = Number(args[4]);
assert.ok(Number.isInteger(port) && port > 0);
const config = JSON.parse(process.env.OPENCODE_CONFIG_CONTENT);
assert.deepEqual(config.permissions, [{ action: "*", resource: "*", effect: "deny" }]);
assert.equal(config.plugins.length, 1);
assert.match(readFileSync(join(config.plugins[0], "index.js"), "utf8"), /opencode-power-pack\.js/);
assert.equal(process.env.OPENCODE_EVAL_MODEL, "provider/model");
assert.ok(process.env.OPENCODE_SERVER_PASSWORD);
if (process.env.FAKE_CWD_PATH) writeFileSync(process.env.FAKE_CWD_PATH, process.cwd());
if (process.env.FAKE_ENV_PATH) {
const authPath = join(process.env.XDG_DATA_HOME, "opencode", "auth.json");
writeFileSync(process.env.FAKE_ENV_PATH, JSON.stringify({
HOME: process.env.HOME,
XDG_CONFIG_HOME: process.env.XDG_CONFIG_HOME,
XDG_DATA_HOME: process.env.XDG_DATA_HOME,
XDG_CACHE_HOME: process.env.XDG_CACHE_HOME,
XDG_STATE_HOME: process.env.XDG_STATE_HOME,
OPENCODE_CONFIG: process.env.OPENCODE_CONFIG,
OPENCODE_CONFIG_DIR: process.env.OPENCODE_CONFIG_DIR,
OPENCODE_SERVER_PASSWORD: process.env.OPENCODE_SERVER_PASSWORD,
OPENCODE_EVAL_MODEL: process.env.OPENCODE_EVAL_MODEL,
ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
authContent: existsSync(authPath) ? readFileSync(authPath, "utf8") : null,
authMode: existsSync(authPath) ? statSync(authPath).mode & 0o777 : null,
sessionsExist: existsSync(join(process.env.XDG_DATA_HOME, "opencode", "sessions.json")),
globalConfigExists: existsSync(join(process.env.XDG_CONFIG_HOME, "opencode", "opencode.json")),
}));
}
const authorization = `Basic ${Buffer.from(`opencode:${process.env.OPENCODE_SERVER_PASSWORD}`).toString("base64")}`;
let promptText = "";
if (behavior === "state") {
for (const root of [
process.env.HOME,
process.env.XDG_CONFIG_HOME,
process.env.XDG_DATA_HOME,
process.env.XDG_CACHE_HOME,
process.env.XDG_STATE_HOME,
]) {
mkdirSync(root, { recursive: true });
writeFileSync(join(root, "generated-state"), "state");
}
}
const respondJson = (response, status, value) => {
const body = JSON.stringify(value);
response.writeHead(status, { "content-type": "application/json", "content-length": Buffer.byteLength(body) });
response.end(body);
};
const server = createServer((request, response) => {
if (request.headers.authorization !== authorization) {
response.writeHead(401);
response.end();
return;
}
let body = "";
request.on("data", (chunk) => (body += chunk));
request.on("end", () => {
if (request.method === "GET" && request.url === "/api/info") {
respondJson(response, 200, {});
return;
}
if (behavior === "hang" || behavior === "hang-with-descendant") {
return;
}
if (request.method === "POST" && request.url === "/api/session") {
if (process.env.FAKE_SESSION_PATH) writeFileSync(process.env.FAKE_SESSION_PATH, body);
respondJson(response, 200, { data: { id: "ses_fake" } });
return;
}
if (request.method === "POST" && request.url === "/api/session/ses_fake/prompt") {
const parsed = JSON.parse(body);
promptText = parsed.text;
if (process.env.FAKE_PROMPT_PATH) writeFileSync(process.env.FAKE_PROMPT_PATH, body);
if (process.env.FAKE_ORDER_PATH) appendFileSync(process.env.FAKE_ORDER_PATH, `start:${promptText}\n`);
respondJson(response, 200, { data: { id: "msg_fake" } });
return;
}
if (request.method === "POST" && request.url === "/api/experimental/session/ses_fake/wait") {
if (process.env.FAKE_ORDER_PATH) appendFileSync(process.env.FAKE_ORDER_PATH, `end:${promptText}\n`);
response.writeHead(204);
response.end();
return;
}
if (request.method === "GET" && request.url === "/api/session/ses_fake/context") {
if (behavior === "malformed") {
response.writeHead(200, { "content-type": "application/json" });
response.end("malformed-sensitive-response");
return;
}
const content = [];
if (behavior !== "empty") {
content.push({ type: "text", text: process.env.FAKE_RESPONSE ?? "MOCK=RESPONSE" });
}
if (behavior === "tool") {
content.push({ type: "tool", id: "prt_fake", name: "read", state: { type: "completed" }, time: { created: 1 } });
}
respondJson(response, 200, { data: [
{ id: "msg_fake", type: "assistant", content,
...(behavior === "assistant-error" ? { error: { type: "api", message: "private provider failure" } } : {}) },
{ id: "msg_idle", type: "idle", outcome: ["failed", "interrupted"].includes(behavior) ? behavior : "succeeded" },
] });
return;
}
response.writeHead(404);
response.end();
});
});
server.listen(port, "127.0.0.1");
if (behavior === "hang-with-descendant") {
const source = "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);";
const descendant = spawn(process.execPath, ["--input-type=module", "-e", source], { stdio: "ignore" });
writeFileSync(process.env.FAKE_DESCENDANT_PID, String(descendant.pid));
}
}
+40 -1
View File
@@ -3,7 +3,46 @@ import { createServer } from "node:http";
import { spawn } from "node:child_process";
import { test } from "node:test";
import assert from "node:assert/strict";
import { fetchJson, stopServer } from "../scripts/opencode-smoke.mjs";
import { fetchJson, stopServer, waitForEntries } from "../scripts/opencode-smoke.mjs";
test("catalog readiness waits for final plugin entries after an empty startup response", async () => {
let requests = 0;
const server = createServer((request, response) => {
requests += 1;
response.setHeader("content-type", "application/json");
response.end(JSON.stringify({ data: requests === 1 ? [] : [{ id: "skill-a" }] }));
});
server.listen(0, "127.0.0.1");
await once(server, "listening");
try {
const result = await waitForEntries(`http://127.0.0.1:${server.address().port}`, ["skill-a"], {}, 1_000);
assert.deepEqual(result, [{ id: "skill-a" }]);
assert.equal(requests, 2);
} finally {
server.closeAllConnections();
server.close();
}
});
test("catalog readiness retries a transient startup socket closure", async () => {
let requests = 0;
const server = createServer((request, response) => {
requests += 1;
if (requests === 1) return request.socket.destroy();
response.setHeader("content-type", "application/json");
response.end(JSON.stringify({ data: [{ id: "skill-a" }] }));
});
server.listen(0, "127.0.0.1");
await once(server, "listening");
try {
const result = await waitForEntries(`http://127.0.0.1:${server.address().port}`, ["skill-a"], {}, 2_000);
assert.deepEqual(result, [{ id: "skill-a" }]);
assert.equal(requests, 2);
} finally {
server.closeAllConnections();
server.close();
}
});
test("fetchJson aborts a stalled OpenCode response", async () => {
const server = createServer(() => {});
+22 -1
View File
@@ -21,7 +21,9 @@ test("published package ships every skill and the sandbox runtime entrypoint", (
cwd: REPO,
encoding: "utf8",
});
const [{ files }] = JSON.parse(output);
const parsed = JSON.parse(output);
// npm 12 keys the dry-run report by package name; older npm returns an array.
const { files } = Array.isArray(parsed) ? parsed[0] : Object.values(parsed)[0];
const packaged = new Set(files.map((file) => file.path));
const skillNames = readdirSync(join(REPO, "skills"))
.filter((name) => existsSync(join(REPO, "skills", name, "SKILL.md")));
@@ -53,6 +55,15 @@ test("published package ships every skill and the sandbox runtime entrypoint", (
"focused review metadata is published",
);
assert.ok(packaged.has("bin/opencode-power-pack.mjs"), "selective installer is published");
assert.ok(packaged.has("index.js"), "OpenCode 2 directory entrypoint is published");
assert.ok(
packaged.has(".opencode/plugins/opencode-power-pack.js"),
"OpenCode plugin entrypoint is published",
);
assert.ok(
packaged.has(".opencode/lib/agent-config.js"),
"OpenCode plugin definitions module is published",
);
assert.ok(packaged.has("bin/sandbox/policy.mjs"), "sandbox policy resolver is published");
assert.ok(packaged.has("bin/sandbox/runtime.mjs"), "sandbox runtime adapter is published");
assert.ok(packaged.has("sandbox/contract.json"), "sandbox contract is published");
@@ -71,11 +82,21 @@ test("published package ships every skill and the sandbox runtime entrypoint", (
}
});
test("package root exposes the same dual-host plugin as its npm entrypoint", async () => {
const { default: root } = await import("../index.js");
const { default: direct } = await import("../.opencode/plugins/opencode-power-pack.js");
assert.equal(root, direct);
assert.equal(root.id, "opencode-power-pack");
assert.equal(typeof root.setup, "function");
assert.equal(typeof root.server, "function");
});
test("plugin loads as an ES module without runtime warnings", () => {
const packedRoot = mkdtempSync(join(tmpdir(), "opp-package-surface-"));
try {
mkdirSync(join(packedRoot, ".opencode"), { recursive: true });
cpSync(join(REPO, ".opencode", "plugins"), join(packedRoot, ".opencode", "plugins"), { recursive: true });
cpSync(join(REPO, ".opencode", "lib"), join(packedRoot, ".opencode", "lib"), { recursive: true });
cpSync(join(REPO, "skills"), join(packedRoot, "skills"), { recursive: true });
writeFileSync(join(packedRoot, "package.json"), '{"type":"module"}\n', "utf8");
+251 -13
View File
@@ -1,20 +1,215 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { OpencodePowerPack } from "../.opencode/plugins/opencode-power-pack.js";
import { readdirSync } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import plugin from "../.opencode/plugins/opencode-power-pack.js";
import { agents as agentPolicies, loadSkills } from "../.opencode/lib/agent-config.js";
const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const SKILLS_DIR = path.join(REPO, "skills");
const SKILL_NAMES = readdirSync(SKILLS_DIR, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
.sort();
function fakeEditor(entries = new Map()) {
return {
entries,
list: () => [...entries.values()],
get: (id) => entries.get(id),
add: (entry) => entries.set(entry.id, entry),
update: (id, update) => {
const draft = entries.get(id) ?? { id, permissions: [{ action: "*", resource: "*", effect: "allow" }] };
update(draft);
entries.set(id, draft);
},
remove: (id) => entries.delete(id),
};
}
test("V2 permission enforcement survives global allows and preserves later custom agents", async () => {
const skills = fakeEditor();
const agents = fakeEditor();
let evaluate;
await plugin.setup({
skill: { transform: async (callback) => callback(skills) },
agent: {
transform: async (callback) => callback(agents),
get: async ({ agentID }) => ({ data: agents.get(agentID) }),
},
session: { get: async () => ({ data: { agent: "code-explorer" } }) },
permission: { hook: async (name, callback) => {
assert.equal(name, "evaluate");
evaluate = callback;
} },
});
assert.equal(typeof evaluate, "function");
for (const name of ["code-explorer", "code-architect", "code-reviewer"]) {
// The real host applies global/configured agents after plugin transforms.
const agent = agents.get(name);
assert.equal(agent.permissions.some(({ effect }) => effect === "deny"), false,
"bundled defaults do not constrain a later user-defined agent");
agent.permissions.push({ action: "*", resource: "*", effect: "allow" });
for (const [action, resources] of [["edit", ["file.js"]], ["read", [".env"]],
["shell", ["rm -rf build"]], ["subagent", ["build"]]]) {
const event = { agent: name, action, resources, effect: "allow", sessionID: "ses_test" };
await evaluate(event);
assert.equal(event.effect, "deny", `${name} denies ${action} after global allow`);
}
}
const implicit = { action: "edit", resources: ["file.js"], effect: "allow", sessionID: "ses_test" };
await evaluate(implicit);
assert.equal(implicit.effect, "deny", "session agent is resolved when omitted");
for (const resource of ["git status; touch unexpected-file", "git status && touch unexpected-file",
"git log | sh", "git show $(touch unexpected-file)", "git status\ntouch unexpected-file"]) {
const event = { agent: "code-reviewer", action: "shell", resources: [resource], effect: "allow" };
await evaluate(event);
assert.equal(event.effect, "deny", "Git prefixes cannot authorize compound shell commands");
}
const unrelated = { agent: "toString", action: "edit", resources: ["file.js"], effect: "allow" };
await evaluate(unrelated);
assert.equal(unrelated.effect, "allow", "unrelated agent IDs cannot select inherited object properties");
for (const effect of ["ask", "deny"]) {
const event = { agent: "code-explorer", action: "read", resources: ["README.md"], effect };
await evaluate(event);
assert.equal(event.effect, effect, "the hook never grants additional access");
}
Object.assign(agents.get("code-reviewer"), { mode: "primary", system: "My custom assistant" });
const custom = { agent: "code-reviewer", action: "edit", resources: ["file.js"], effect: "allow" };
await evaluate(custom);
assert.equal(custom.effect, "allow", "a configured replacement keeps host permissions");
});
async function register() {
const skills = fakeEditor();
const agents = fakeEditor();
const context = contextFor(skills, agents);
await plugin.setup(context);
return { skills, agents };
}
function contextFor(skills, agents) {
return {
skill: { transform: async (callback) => callback(skills) },
agent: {
transform: async (callback) => callback(agents),
get: async ({ agentID }) => ({ data: agents.get(agentID) }),
},
permission: { hook: async () => {} },
};
}
test("plugin registers every bundled skill with the V2 Skill.Info shape", async () => {
const { skills } = await register();
assert.equal(plugin.id, "opencode-power-pack");
assert.equal(skills.list().length, SKILL_NAMES.length);
for (const name of SKILL_NAMES) {
const skill = skills.get(name);
assert.ok(skill, `${name} is registered`);
assert.equal(skill.id, name);
assert.match(skill.name, /\S/);
assert.match(skill.description, /\S/);
assert.equal(skill.path, path.join(SKILLS_DIR, name, "SKILL.md"));
assert.match(skill.content, /\S/);
assert.doesNotMatch(skill.content, /^---/, `${name} registers its body without frontmatter`);
}
assert.match(
skills.get("feature-dev").content,
/# Feature Development/,
"skill content keeps the workflow body",
);
});
test("plugin registers the feature workflow roles as read-only subagents", async () => {
const plugin = await OpencodePowerPack();
const config = {};
await plugin.config(config);
const { agents } = await register();
for (const name of ["code-explorer", "code-architect", "code-reviewer"]) {
const agent = config.agent?.[name];
const agent = agents.get(name);
assert.ok(agent, `${name} is registered`);
assert.equal(agent.mode, "subagent");
assert.equal(agent.hidden, false);
assert.match(agent.description, /\S/);
assert.match(agent.system, /\S/);
assert.doesNotMatch(agent.system, /^---/);
const rule = (action, resource) => agentPolicies(loadSkills(SKILLS_DIR))[name].permissions
.filter((entry) => entry.action === action && entry.resource === resource)
.at(-1)?.effect;
assert.equal(rule("*", "*"), "deny", `${name} denies unlisted actions`);
assert.equal(rule("read", "*"), "allow");
assert.equal(rule("read", "*.env"), "deny");
assert.equal(rule("read", "*.env.*"), "deny");
assert.equal(rule("read", "*.env.example"), "allow");
assert.equal(rule("glob", "*"), "allow");
assert.equal(rule("grep", "*"), "allow");
assert.equal(rule("edit", "*"), "deny");
assert.equal(rule("subagent", "*"), "deny");
assert.equal(rule("webfetch", "*"), "deny");
assert.equal(rule("websearch", "*"), "deny");
assert.equal(rule("external_directory", "*"), "deny");
assert.equal(rule("shell", "*"), "deny");
}
assert.match(
agents.get("code-reviewer").system,
/Dispatched handoff/i,
"registered reviewer inherits the handoff contract",
);
const reviewer = agentPolicies(loadSkills(SKILLS_DIR))["code-reviewer"].permissions;
assert.deepEqual(
reviewer.filter(({ action }) => action === "shell"),
[
{ action: "shell", resource: "*", effect: "deny" },
{ action: "shell", resource: "git status*", effect: "allow" },
{ action: "shell", resource: "git diff*", effect: "allow" },
{ action: "shell", resource: "git show*", effect: "allow" },
{ action: "shell", resource: "git log*", effect: "allow" },
{ action: "shell", resource: "git blame*", effect: "allow" },
{ action: "shell", resource: "git rev-parse*", effect: "allow" },
{ action: "shell", resource: "git merge-base*", effect: "allow" },
{ action: "shell", resource: "git ls-files*", effect: "allow" },
{ action: "shell", resource: "git *--output*", effect: "deny" },
{ action: "shell", resource: "git *--ext-diff*", effect: "deny" },
{ action: "shell", resource: "git *>*", effect: "deny" },
],
"code-reviewer keeps the read-only Git allowlist and write-through denials",
);
});
test("plugin preserves agents registered before it", async () => {
const customReviewer = { id: "code-reviewer", name: "Custom", mode: "subagent" };
const skills = fakeEditor();
const agents = fakeEditor(new Map([["code-reviewer", customReviewer]]));
await plugin.setup(contextFor(skills, agents));
assert.equal(agents.get("code-reviewer"), customReviewer);
assert.ok(agents.get("code-explorer"), "other roles are still registered");
assert.ok(agents.get("code-architect"), "other roles are still registered");
});
test("plugin exposes the OpenCode 1 config hook for skills and agents", async () => {
const { config } = await plugin.server();
const target = {};
await config(target);
assert.deepEqual(target.skills.paths, [SKILLS_DIR]);
await config(target);
assert.deepEqual(target.skills.paths, [SKILLS_DIR], "skill path is registered once");
for (const name of ["code-explorer", "code-architect", "code-reviewer"]) {
const agent = target.agent[name];
assert.ok(agent, `${name} is registered`);
assert.equal(agent.mode, "subagent");
assert.equal(agent.permission.edit, "deny");
assert.equal(agent.permission.task, "deny");
assert.equal(agent.permission.webfetch, "deny");
if (name !== "code-reviewer") assert.equal(agent.permission.bash, "deny");
assert.deepEqual(agent.permission.read, {
"*": "allow",
"*.env": "deny",
@@ -27,12 +222,12 @@ test("plugin registers the feature workflow roles as read-only subagents", async
}
assert.match(
config.agent["code-reviewer"].prompt,
target.agent["code-reviewer"].prompt,
/Dispatched handoff/i,
"registered reviewer inherits the handoff contract",
);
assert.deepEqual(config.agent["code-reviewer"].permission.bash, {
assert.deepEqual(target.agent["code-reviewer"].permission.bash, {
"*": "deny",
"git status*": "allow",
"git diff*": "allow",
@@ -48,12 +243,55 @@ test("plugin registers the feature workflow roles as read-only subagents", async
});
});
test("plugin preserves user-defined agents with the same names", async () => {
const plugin = await OpencodePowerPack();
test("OpenCode 1 config hook preserves user-defined agents", async () => {
const customReviewer = { description: "My reviewer", mode: "subagent" };
const config = { agent: { "code-reviewer": customReviewer } };
const target = { agent: { "code-reviewer": customReviewer } };
const { config } = await plugin.server();
await plugin.config(config);
await config(target);
assert.equal(config.agent["code-reviewer"], customReviewer);
assert.equal(target.agent["code-reviewer"], customReviewer);
assert.ok(target.agent["code-explorer"], "other roles are still registered");
assert.ok(target.agent["code-architect"], "other roles are still registered");
});
test("both hosts derive the same read-only rules from one permission source", async () => {
const { agents } = await register();
const target = {};
await (await plugin.server()).config(target);
assert.deepEqual(agentPolicies(loadSkills(SKILLS_DIR))["code-explorer"].permissions, [
{ action: "*", resource: "*", effect: "deny" },
{ action: "read", resource: "*", effect: "allow" },
{ action: "read", resource: "*.env", effect: "deny" },
{ action: "read", resource: "*.env.*", effect: "deny" },
{ action: "read", resource: "*.env.example", effect: "allow" },
{ action: "glob", resource: "*", effect: "allow" },
{ action: "grep", resource: "*", effect: "allow" },
{ action: "edit", resource: "*", effect: "deny" },
{ action: "subagent", resource: "*", effect: "deny" },
{ action: "webfetch", resource: "*", effect: "deny" },
{ action: "websearch", resource: "*", effect: "deny" },
{ action: "external_directory", resource: "*", effect: "deny" },
{ action: "shell", resource: "*", effect: "deny" },
]);
assert.equal(target.agent["code-explorer"].permission.list, "allow", "OpenCode 1 keeps its list tool");
});
test("host edits to one registered agent do not leak into others or later registrations", async () => {
// OpenCode 2 mutates host permission arrays after our registration.
const first = await register();
first.agents.get("code-explorer").permissions.push({ action: "edit", resource: "*", effect: "allow" });
assert.equal(first.agents.get("code-architect").permissions.at(-1).action, "*");
const second = await register();
assert.equal(second.agents.get("code-explorer").permissions.at(-1).action, "*");
const { config } = await plugin.server();
const legacy = {};
await config(legacy);
legacy.agent["code-explorer"].permission.read["*.env"] = "allow";
assert.equal(legacy.agent["code-architect"].permission.read["*.env"], "deny");
const fresh = {};
await config(fresh);
assert.equal(fresh.agent["code-explorer"].permission.read["*.env"], "deny");
});
+2
View File
@@ -543,6 +543,8 @@ test("runner rejects a workspace inside an upstream shared-write compatibility p
const workspace = path.join(fixture, "claude", "project");
await mkdir(workspace, { recursive: true });
try {
// A repository-local TMPDIR must not resolve to the enclosing checkout.
await execFileAsync("git", ["init", "--quiet", workspace]);
await assert.rejects(runSandboxedCommand({
profile: profile("develop", { workspace: "write" }),
allowedDomains: [],