Files
skillkit/packages/cli/src/commands/eval.ts
T
Rohit Ghumare 0c82c58905 feat(cli): add --json output and loading spinners to remaining commands
Add --json flag with structured JSON output to: create, enable, disable,
eval, sync, tap (add/remove/list), translate, save, read.

Add loading spinners to: audit, fix, test, plugin, team, methodology,
hook, remove. Commands that already had spinners get --json-aware no-op
spinners when --json is active to suppress visual output.
2026-04-12 12:02:38 +01:00

171 lines
5.5 KiB
TypeScript

import { Command, Option } from 'clipanion';
import { resolve } from 'node:path';
import { existsSync } from 'node:fs';
import {
createEvalEngine,
formatEvalResult,
LLMQualityEvaluator,
ContradictionEvaluator,
BehavioralSecurityEvaluator,
SandboxEvaluator,
DynamicBenchmarkEvaluator,
CommunitySignalsEvaluator,
} from '@skillkit/core';
import type { EvalTier, EvalOptions } from '@skillkit/core';
import { spinner } from '../onboarding/index.js';
export class EvalCommand extends Command {
static override paths = [['eval']];
static override usage = Command.Usage({
description: 'Evaluate a skill with multi-tier analysis (LLM quality, contradictions, security, benchmarks)',
details: `
Runs a comprehensive evaluation engine across up to 6 tiers:
Tier 1: LLM-based quality scoring (G-Eval pattern)
Tier 2: Contradiction detection (formal + semantic)
Tier 3: Behavioral security analysis (AST + taint + LLM)
Tier 4: Sandbox execution testing (Docker)
Tier 5: Dynamic marketplace benchmarks
Tier 6: Community signals (GitHub, installs, freshness)
Works without API keys (heuristic fallback for Tier 1, Tiers 5-6 always available).
Configure a provider for full LLM-powered evaluation.
`,
examples: [
['Evaluate a skill', '$0 eval ./my-skill'],
['Run specific tiers', '$0 eval ./my-skill --tier 1,2,3'],
['Use Anthropic provider', '$0 eval ./my-skill --provider anthropic'],
['JSON output', '$0 eval ./my-skill --format json'],
['Set minimum score', '$0 eval ./my-skill --min-score 70'],
['Verbose output', '$0 eval ./my-skill --verbose'],
],
});
skillPath = Option.String({ required: true, name: 'path' });
tier = Option.String('--tier,-t', {
description: 'Comma-separated tier numbers to run (1-6). Default: 1,2,3,5,6',
});
provider = Option.String('--provider,-p', {
description: 'LLM provider: anthropic, openai, google, ollama, openrouter',
});
model = Option.String('--model,-m', {
description: 'Model name to use with the provider',
});
format = Option.String('--format,-f', 'summary', {
description: 'Output format: summary, json, table',
});
verbose = Option.Boolean('--verbose,-v', false, {
description: 'Show detailed output for each tier',
});
minScore = Option.String('--min-score', {
description: 'Exit with code 1 if overall score is below this threshold',
});
sandboxImage = Option.String('--sandbox-image', {
description: 'Docker image for sandbox testing (Tier 4)',
});
timeout = Option.String('--timeout', {
description: 'Timeout in seconds for each tier',
});
json = Option.Boolean('--json', false, {
description: 'Output as JSON',
});
async execute(): Promise<number> {
const targetPath = resolve(this.skillPath);
if (!existsSync(targetPath)) {
this.context.stderr.write(`Path not found: ${targetPath}\n`);
return 1;
}
const validFormats = ['summary', 'json', 'table'];
if (!validFormats.includes(this.format)) {
this.context.stderr.write(`Invalid format: "${this.format}". Must be one of: ${validFormats.join(', ')}\n`);
return 1;
}
let tiers: EvalTier[] | undefined;
if (this.tier) {
tiers = this.tier.split(',').map((s) => {
const n = parseInt(s.trim(), 10);
if (isNaN(n) || n < 1 || n > 6) {
return null;
}
return n as EvalTier;
}).filter((n): n is EvalTier => n !== null);
if (tiers.length === 0) {
this.context.stderr.write(`Invalid --tier value: "${this.tier}". Must be comma-separated numbers 1-6.\n`);
return 1;
}
}
if (this.minScore) {
const threshold = parseInt(this.minScore, 10);
if (isNaN(threshold)) {
this.context.stderr.write(`Invalid --min-score value: "${this.minScore}". Must be a number.\n`);
return 1;
}
}
const parsedTimeout = this.timeout ? parseInt(this.timeout, 10) : NaN;
const options: EvalOptions = {
tiers,
provider: this.provider,
model: this.model,
format: this.format as 'summary' | 'json' | 'table',
verbose: this.verbose,
sandboxImage: this.sandboxImage,
timeout: !isNaN(parsedTimeout) ? parsedTimeout : undefined,
};
const engine = createEvalEngine();
engine.registerEvaluator(new LLMQualityEvaluator());
engine.registerEvaluator(new ContradictionEvaluator());
engine.registerEvaluator(new BehavioralSecurityEvaluator());
engine.registerEvaluator(new SandboxEvaluator());
engine.registerEvaluator(new DynamicBenchmarkEvaluator());
engine.registerEvaluator(new CommunitySignalsEvaluator());
const isJson = this.json || this.format === 'json';
const s = isJson ? { start: () => {}, stop: () => {}, message: () => {} } : spinner();
s.start('Evaluating skill...');
let result;
try {
result = await engine.evaluate(targetPath, options);
s.stop(`Evaluation complete (score: ${result.overallScore})`);
} catch (err) {
s.stop('Evaluation failed');
throw err;
}
if (this.json) {
console.log(JSON.stringify(result));
} else {
this.context.stdout.write(formatEvalResult(result, this.format) + '\n');
}
if (this.minScore) {
const threshold = parseInt(this.minScore, 10);
if (result.overallScore < threshold) {
if (!this.json) {
this.context.stderr.write(`Score ${result.overallScore} is below minimum ${threshold}\n`);
}
return 1;
}
}
return 0;
}
}