mirror of
https://github.com/rohitg00/skillkit.git
synced 2026-11-18 18:04:26 +00:00
* test: add comprehensive E2E test suite for CLI commands - Add 183 E2E tests covering all major CLI commands - Core commands: init, list, read, validate, enable, disable, sync - Translation: all 17 agents, context detection, recommendations - Team collaboration: init, share, sync, bundle operations - Methodology & Plan: 5 methodology packs, plan CRUD operations - Hooks & Orchestration: event triggers, multi-agent coordination - Plugin system: install, uninstall, enable, disable, marketplace - Commands & Settings: slash commands, memory, audit, CI/CD CI/CD updates: - Add dedicated E2E job running on Node 22 after unit tests - Upload artifacts on E2E test failures for debugging - Add ci-success job for branch protection New npm scripts: - pnpm test:e2e - Run E2E tests - pnpm test:e2e:watch - Run E2E tests in watch mode - pnpm test:all - Run unit + E2E tests * fix: improve E2E test reliability and align with actual CLI interface - Fix ESM compatibility: use import.meta.url for __dirname equivalent - Fix security: use execFileSync instead of execSync to prevent shell injection - Fix skillCount: use nullish coalescing (??) to respect explicit 0 - Fix weak assertions: replace always-true checks (exitCode !== undefined) with meaningful assertions (exitCode <= 1, hasRelevantOutput) - Fix CLI alignment: update tests to use actual command actions - methodology: load -> install, apply -> sync, validate -> info - plan: list -> templates, status -> generate - team init: add required --registry parameter All 180 E2E tests now pass. * fix: add .agent/skills directory check and timeout guards - Add .agent/skills to directory checks in init tests (fixes CI failure) - Add explicit timeout guards (exitCode !== -1) before exitCode <= 1 checks to fail fast on runCli timeouts instead of false positives
286 lines
7.2 KiB
TypeScript
286 lines
7.2 KiB
TypeScript
/**
|
|
* E2E Test CLI Runner
|
|
* Executes skillkit CLI commands and captures output for testing
|
|
*/
|
|
|
|
import { spawn, execFileSync } from 'node:child_process';
|
|
import { join, resolve, dirname } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { mkdirSync, rmSync, existsSync, writeFileSync, readFileSync, mkdtempSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { randomUUID } from 'node:crypto';
|
|
|
|
// ESM-compatible __dirname equivalent
|
|
const __filename = fileURLToPath(import.meta.url);
|
|
const __dirname = dirname(__filename);
|
|
|
|
export interface CliResult {
|
|
stdout: string;
|
|
stderr: string;
|
|
exitCode: number;
|
|
success: boolean;
|
|
duration: number;
|
|
}
|
|
|
|
export interface RunOptions {
|
|
cwd?: string;
|
|
env?: NodeJS.ProcessEnv;
|
|
timeout?: number;
|
|
input?: string;
|
|
}
|
|
|
|
const ROOT_DIR = resolve(__dirname, '../..');
|
|
const CLI_PATH = join(ROOT_DIR, 'apps/skillkit/dist/cli.js');
|
|
|
|
/**
|
|
* Run a skillkit CLI command and capture the result
|
|
*/
|
|
export async function runCli(args: string[], options: RunOptions = {}): Promise<CliResult> {
|
|
const startTime = Date.now();
|
|
const cwd = options.cwd || ROOT_DIR;
|
|
const timeout = options.timeout || 30000;
|
|
|
|
return new Promise((resolve) => {
|
|
const child = spawn('node', [CLI_PATH, ...args], {
|
|
cwd,
|
|
env: { ...process.env, ...options.env, NO_COLOR: '1', FORCE_COLOR: '0' },
|
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
});
|
|
|
|
let stdout = '';
|
|
let stderr = '';
|
|
let killed = false;
|
|
|
|
const timeoutId = setTimeout(() => {
|
|
killed = true;
|
|
child.kill('SIGTERM');
|
|
}, timeout);
|
|
|
|
child.stdout?.on('data', (data) => {
|
|
stdout += data.toString();
|
|
});
|
|
|
|
child.stderr?.on('data', (data) => {
|
|
stderr += data.toString();
|
|
});
|
|
|
|
if (options.input) {
|
|
child.stdin?.write(options.input);
|
|
child.stdin?.end();
|
|
}
|
|
|
|
child.on('close', (code) => {
|
|
clearTimeout(timeoutId);
|
|
const duration = Date.now() - startTime;
|
|
const exitCode = killed ? -1 : (code ?? 1);
|
|
resolve({
|
|
stdout,
|
|
stderr,
|
|
exitCode,
|
|
success: exitCode === 0,
|
|
duration,
|
|
});
|
|
});
|
|
|
|
child.on('error', (err) => {
|
|
clearTimeout(timeoutId);
|
|
const duration = Date.now() - startTime;
|
|
resolve({
|
|
stdout,
|
|
stderr: stderr + err.message,
|
|
exitCode: 1,
|
|
success: false,
|
|
duration,
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Run skillkit CLI synchronously (for simpler tests)
|
|
* Uses execFileSync to avoid shell injection vulnerabilities
|
|
*/
|
|
export function runCliSync(args: string[], options: RunOptions = {}): CliResult {
|
|
const startTime = Date.now();
|
|
const cwd = options.cwd || ROOT_DIR;
|
|
|
|
try {
|
|
const result = execFileSync('node', [CLI_PATH, ...args], {
|
|
cwd,
|
|
env: { ...process.env, ...options.env, NO_COLOR: '1', FORCE_COLOR: '0' },
|
|
encoding: 'utf-8',
|
|
timeout: options.timeout || 30000,
|
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
});
|
|
|
|
return {
|
|
stdout: result,
|
|
stderr: '',
|
|
exitCode: 0,
|
|
success: true,
|
|
duration: Date.now() - startTime,
|
|
};
|
|
} catch (error: any) {
|
|
return {
|
|
stdout: error.stdout || '',
|
|
stderr: error.stderr || error.message,
|
|
exitCode: error.status || 1,
|
|
success: false,
|
|
duration: Date.now() - startTime,
|
|
};
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Create a temporary test directory
|
|
*/
|
|
export function createTestDir(prefix = 'skillkit-e2e-'): string {
|
|
return mkdtempSync(join(tmpdir(), prefix));
|
|
}
|
|
|
|
/**
|
|
* Clean up a test directory
|
|
*/
|
|
export function cleanupTestDir(dir: string): void {
|
|
if (existsSync(dir)) {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Create a test skill in a directory
|
|
*/
|
|
export function createTestSkill(
|
|
dir: string,
|
|
name: string,
|
|
options: {
|
|
description?: string;
|
|
content?: string;
|
|
additionalFrontmatter?: Record<string, unknown>;
|
|
} = {}
|
|
): string {
|
|
const skillDir = join(dir, name);
|
|
mkdirSync(skillDir, { recursive: true });
|
|
|
|
const frontmatter: Record<string, unknown> = {
|
|
name,
|
|
description: options.description || `Test skill: ${name} for E2E testing`,
|
|
...options.additionalFrontmatter,
|
|
};
|
|
|
|
const frontmatterStr = Object.entries(frontmatter)
|
|
.map(([key, value]) => {
|
|
if (typeof value === 'object') {
|
|
return `${key}:\n${Object.entries(value as Record<string, unknown>)
|
|
.map(([k, v]) => ` ${k}: ${JSON.stringify(v)}`)
|
|
.join('\n')}`;
|
|
}
|
|
return `${key}: ${value}`;
|
|
})
|
|
.join('\n');
|
|
|
|
const content = `---
|
|
${frontmatterStr}
|
|
---
|
|
|
|
# ${name}
|
|
|
|
${options.content || `This is a test skill named ${name}.`}
|
|
`;
|
|
|
|
writeFileSync(join(skillDir, 'SKILL.md'), content);
|
|
return skillDir;
|
|
}
|
|
|
|
/**
|
|
* Create a test project with common setup
|
|
*/
|
|
export function createTestProject(options: {
|
|
withSkills?: boolean;
|
|
skillCount?: number;
|
|
withGit?: boolean;
|
|
projectType?: 'node' | 'python' | 'rust' | 'go';
|
|
} = {}): { dir: string; cleanup: () => void } {
|
|
const dir = createTestDir();
|
|
|
|
if (options.withGit) {
|
|
mkdirSync(join(dir, '.git'), { recursive: true });
|
|
writeFileSync(join(dir, '.git', 'config'), '[core]\n\trepositoryformatversion = 0');
|
|
}
|
|
|
|
if (options.projectType === 'node') {
|
|
writeFileSync(
|
|
join(dir, 'package.json'),
|
|
JSON.stringify({ name: 'test-project', version: '1.0.0' }, null, 2)
|
|
);
|
|
} else if (options.projectType === 'python') {
|
|
writeFileSync(join(dir, 'requirements.txt'), 'pytest>=7.0.0\n');
|
|
} else if (options.projectType === 'rust') {
|
|
writeFileSync(join(dir, 'Cargo.toml'), '[package]\nname = "test"\nversion = "0.1.0"');
|
|
} else if (options.projectType === 'go') {
|
|
writeFileSync(join(dir, 'go.mod'), 'module test\n\ngo 1.21');
|
|
}
|
|
|
|
if (options.withSkills) {
|
|
const skillsDir = join(dir, 'skills');
|
|
mkdirSync(skillsDir, { recursive: true });
|
|
// Use nullish coalescing to respect explicit 0
|
|
const count = options.skillCount ?? 3;
|
|
for (let i = 1; i <= count; i++) {
|
|
createTestSkill(skillsDir, `test-skill-${i}`);
|
|
}
|
|
}
|
|
|
|
return {
|
|
dir,
|
|
cleanup: () => cleanupTestDir(dir),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Read file content from test directory
|
|
*/
|
|
export function readTestFile(dir: string, ...paths: string[]): string {
|
|
return readFileSync(join(dir, ...paths), 'utf-8');
|
|
}
|
|
|
|
/**
|
|
* Check if file exists in test directory
|
|
*/
|
|
export function testFileExists(dir: string, ...paths: string[]): boolean {
|
|
return existsSync(join(dir, ...paths));
|
|
}
|
|
|
|
/**
|
|
* Write file in test directory
|
|
*/
|
|
export function writeTestFile(dir: string, path: string, content: string): void {
|
|
const fullPath = join(dir, path);
|
|
mkdirSync(join(fullPath, '..'), { recursive: true });
|
|
writeFileSync(fullPath, content);
|
|
}
|
|
|
|
/**
|
|
* Assert CLI output contains expected text
|
|
*/
|
|
export function assertContains(result: CliResult, expected: string, message?: string): void {
|
|
const combined = result.stdout + result.stderr;
|
|
if (!combined.includes(expected)) {
|
|
throw new Error(
|
|
message || `Expected output to contain "${expected}"\nActual output:\n${combined}`
|
|
);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Assert CLI output matches regex
|
|
*/
|
|
export function assertMatches(result: CliResult, pattern: RegExp, message?: string): void {
|
|
const combined = result.stdout + result.stderr;
|
|
if (!pattern.test(combined)) {
|
|
throw new Error(
|
|
message || `Expected output to match ${pattern}\nActual output:\n${combined}`
|
|
);
|
|
}
|
|
}
|