mirror of
https://github.com/rohitg00/skillkit.git
synced 2026-11-20 12:23:29 +00:00
* feat(quality): add skill quality evaluation system - Add quality scoring module with structure, clarity, and specificity metrics - Evaluate skills for metadata, triggers, boundaries, code examples - Detect vague terms and penalize overly long skills - Update validate command to include quality reports - Add --verbose, --quality, --issues, --json flags - Quality scores: A (90+), B (80+), C (70+), D (60+), F (<60) * fix: address CodeRabbit and Devin review suggestions - Fix JSON output to return exit code 1 on validation failures - Add --min-score NaN validation with warning message - Add directory guard for --all flag with .mdc file support - Fix CRLF handling in frontmatter extraction - Fix code blocks double-counting in specificity evaluation - Fix warning message inconsistency for code examples - Add try/catch error handling in evaluateSkillFile - Fix isHighQuality threshold to match UI display (80) * feat(quality): extend quality evaluation system with badges, deeper analysis, and fix command - Phase 2: Add quality badges to CLI commands (list, recommend, install) - formatQualityBadge() helper for color-coded grade badges - --quality flag for list command - Low-quality skill warnings during install - Phase 3: Add TUI quality badges - Extended SkillItem with quality, grade, warnings fields - Quality evaluation in loadSkills() - Grade badges in Installed and Recommend screens - Phase 4: Deeper quality analysis - Deprecated pattern detection (CommonJS, class components, var) - Security pattern detection (hardcoded credentials, eval, innerHTML) - Conflicting instruction detection - Completeness assessment (TODOs, empty sections, example coverage) - New weight distribution: Structure 35%, Clarity 25%, Specificity 25%, Advanced 15% - Phase 1: Pre-compute marketplace quality scores - Updated schema.json with quality field (0-100) - Created scripts/compute-quality.ts for batch computation - Phase 5: Benchmark against category leaders - New benchmark.ts module with category detection - Percentile calculation and comparison notes - --benchmark flag for validate command - Phase 6: Fix command for automated quality fixes - skillkit fix command with --dry-run and --interactive modes - Auto-generates frontmatter, "When to Use", "Boundaries" sections - --fix flag alias on validate command * fix: address CodeRabbit and Devin review suggestions - Fix .mdc file name extraction in fix.ts (handles both .md and .mdc) - Add missing advanced field to test fixtures in quality.test.ts - Fix duplicate header detection in assessCompleteness using regex.exec - Fix ESM compatibility in compute-quality.ts using import.meta.url - Add trailing newline to skills.json output * fix: address remaining CodeRabbit review suggestions - Escape regex metacharacters in vague term patterns (fix.ts) - Improve diff algorithm using set-based comparison (fix.ts) - Fix validateSkill receiving file path instead of directory (validate.ts) - Check final section in completeness assessment (index.ts) - Add error handling and empty skills guard (compute-quality.ts) * fix: address additional review feedback - validate.ts: Add error when --fix is passed without targets - index.ts: Skip headers inside fenced code blocks during completeness check - compute-quality.ts: Use nullish coalescing (??) to preserve zero quality scores
442 lines
16 KiB
TypeScript
442 lines
16 KiB
TypeScript
import { existsSync, readdirSync, statSync } from 'node:fs';
|
|
import { join, basename, resolve, dirname } from 'node:path';
|
|
import { Command, Option } from 'clipanion';
|
|
import {
|
|
validateSkill,
|
|
evaluateSkillFile,
|
|
evaluateSkillDirectory,
|
|
getQualityGrade,
|
|
isHighQuality,
|
|
findAllSkills,
|
|
benchmarkSkill,
|
|
type QualityScore,
|
|
type BenchmarkResult,
|
|
} from '@skillkit/core';
|
|
import { getSearchDirs } from '../helpers.js';
|
|
import {
|
|
colors,
|
|
symbols,
|
|
spinner,
|
|
success,
|
|
warn,
|
|
error,
|
|
header,
|
|
} from '../onboarding/index.js';
|
|
|
|
function formatScore(score: number): string {
|
|
if (score >= 80) return colors.success(`${score}`);
|
|
if (score >= 60) return colors.warning(`${score}`);
|
|
return colors.error(`${score}`);
|
|
}
|
|
|
|
function formatGrade(grade: string): string {
|
|
switch (grade) {
|
|
case 'A':
|
|
case 'B':
|
|
return colors.success(grade);
|
|
case 'C':
|
|
case 'D':
|
|
return colors.warning(grade);
|
|
default:
|
|
return colors.error(grade);
|
|
}
|
|
}
|
|
|
|
function printQualityReport(name: string, quality: QualityScore, verbose: boolean): void {
|
|
const grade = getQualityGrade(quality.overall);
|
|
const gradeDisplay = formatGrade(grade);
|
|
const scoreDisplay = formatScore(quality.overall);
|
|
|
|
console.log(`\n${colors.primary(name)}`);
|
|
console.log(` Grade: ${gradeDisplay} Score: ${scoreDisplay}/100`);
|
|
|
|
if (verbose) {
|
|
console.log('');
|
|
console.log(` ${colors.muted('Structure:')} ${formatScore(quality.structure.score)}/100`);
|
|
console.log(` ${quality.structure.hasMetadata ? colors.success(symbols.success) : colors.error(symbols.error)} Metadata`);
|
|
console.log(` ${quality.structure.hasDescription ? colors.success(symbols.success) : colors.error(symbols.error)} Description`);
|
|
console.log(` ${quality.structure.hasTriggers ? colors.success(symbols.success) : colors.error(symbols.error)} Trigger conditions`);
|
|
console.log(` ${quality.structure.hasExamples ? colors.success(symbols.success) : colors.error(symbols.error)} Code examples`);
|
|
console.log(` ${quality.structure.hasBoundaries ? colors.success(symbols.success) : colors.error(symbols.error)} Boundaries defined`);
|
|
|
|
console.log('');
|
|
console.log(` ${colors.muted('Clarity:')} ${formatScore(quality.clarity.score)}/100`);
|
|
console.log(` Lines: ${quality.clarity.lineCount}${quality.clarity.lineCount > 300 ? colors.warning(' (consider splitting)') : ''}`);
|
|
console.log(` Tokens: ${quality.clarity.tokenCount}${quality.clarity.tokenCount > 2000 ? colors.warning(' (high)') : ''}`);
|
|
console.log(` ${quality.clarity.hasHeaders ? colors.success(symbols.success) : colors.error(symbols.error)} Uses headers`);
|
|
|
|
console.log('');
|
|
console.log(` ${colors.muted('Specificity:')} ${formatScore(quality.specificity.score)}/100`);
|
|
console.log(` ${quality.specificity.hasConcreteCommands ? colors.success(symbols.success) : colors.error(symbols.error)} Concrete commands`);
|
|
console.log(` ${quality.specificity.hasFilePatterns ? colors.success(symbols.success) : colors.error(symbols.error)} File patterns`);
|
|
console.log(` ${quality.specificity.hasCodeExamples ? colors.success(symbols.success) : colors.error(symbols.error)} Multiple code examples`);
|
|
if (quality.specificity.vagueTermCount > 0) {
|
|
console.log(` ${colors.warning(symbols.warning)} ${quality.specificity.vagueTermCount} vague term(s)`);
|
|
}
|
|
|
|
console.log('');
|
|
console.log(` ${colors.muted('Advanced:')} ${formatScore(quality.advanced.score)}/100`);
|
|
if (quality.advanced.deprecatedPatterns.length > 0) {
|
|
console.log(` ${colors.warning(symbols.warning)} ${quality.advanced.deprecatedPatterns.length} deprecated pattern(s)`);
|
|
for (const pattern of quality.advanced.deprecatedPatterns) {
|
|
console.log(` ${colors.muted(symbols.bullet)} ${pattern}`);
|
|
}
|
|
}
|
|
if (quality.advanced.securityIssues.length > 0) {
|
|
console.log(` ${colors.error(symbols.error)} ${quality.advanced.securityIssues.length} security issue(s)`);
|
|
for (const issue of quality.advanced.securityIssues) {
|
|
console.log(` ${colors.muted(symbols.bullet)} ${issue}`);
|
|
}
|
|
}
|
|
if (quality.advanced.conflictingInstructions.length > 0) {
|
|
console.log(` ${colors.warning(symbols.warning)} ${quality.advanced.conflictingInstructions.length} conflicting instruction(s)`);
|
|
}
|
|
console.log(` Completeness: ${quality.advanced.completeness.score}/100`);
|
|
console.log(` Example coverage: ${quality.advanced.completeness.exampleCoverage}%`);
|
|
if (quality.advanced.completeness.hasTodos) {
|
|
console.log(` ${colors.warning(symbols.warning)} ${quality.advanced.completeness.todoCount} TODO/FIXME comment(s)`);
|
|
}
|
|
}
|
|
|
|
if (quality.warnings.length > 0) {
|
|
console.log('');
|
|
console.log(` ${colors.warning('Warnings:')}`);
|
|
for (const w of quality.warnings) {
|
|
console.log(` ${colors.warning(symbols.warning)} ${w}`);
|
|
}
|
|
}
|
|
|
|
if (quality.suggestions.length > 0 && verbose) {
|
|
console.log('');
|
|
console.log(` ${colors.muted('Suggestions:')}`);
|
|
for (const s of quality.suggestions) {
|
|
console.log(` ${colors.muted(symbols.bullet)} ${s}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
function printBenchmarkReport(benchmark: BenchmarkResult): void {
|
|
console.log('');
|
|
console.log(colors.bold('Benchmark Comparison'));
|
|
console.log('');
|
|
|
|
console.log(` ${colors.muted('Your Score:')} ${formatScore(benchmark.score)}/100 ${formatGrade(benchmark.grade)}`);
|
|
console.log(` ${colors.muted('Category Average:')} ${benchmark.categoryAvg}/100`);
|
|
console.log(` ${colors.muted('Top Skill Score:')} ${benchmark.topSkillScore}/100`);
|
|
console.log(` ${colors.muted('Percentile:')} ${benchmark.percentile}${getOrdinal(benchmark.percentile)}`);
|
|
|
|
if (benchmark.comparisonNotes.length > 0) {
|
|
console.log('');
|
|
console.log(` ${colors.muted('Comparison:')}`);
|
|
for (const note of benchmark.comparisonNotes) {
|
|
console.log(` ${colors.muted(symbols.bullet)} ${note}`);
|
|
}
|
|
}
|
|
|
|
if (benchmark.recommendations.length > 0) {
|
|
console.log('');
|
|
console.log(` ${colors.muted('To Improve:')}`);
|
|
for (const rec of benchmark.recommendations) {
|
|
console.log(` ${colors.info(symbols.info)} ${rec}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
function getOrdinal(n: number): string {
|
|
const s = ['th', 'st', 'nd', 'rd'];
|
|
const v = n % 100;
|
|
return s[(v - 20) % 10] || s[v] || s[0];
|
|
}
|
|
|
|
export class ValidateCommand extends Command {
|
|
static override paths = [['validate']];
|
|
|
|
static override usage = Command.Usage({
|
|
description: 'Validate skill format and quality',
|
|
details: `
|
|
Validates skills against the specification and evaluates quality.
|
|
Quality scores are based on structure, clarity, and specificity.
|
|
`,
|
|
examples: [
|
|
['Validate all installed skills', '$0 validate'],
|
|
['Validate specific skill', '$0 validate my-skill'],
|
|
['Validate skill file', '$0 validate ./path/to/SKILL.md'],
|
|
['Detailed quality report', '$0 validate --verbose'],
|
|
['Only show issues', '$0 validate --issues'],
|
|
['Quality check only', '$0 validate --quality'],
|
|
['JSON output', '$0 validate --json'],
|
|
],
|
|
});
|
|
|
|
targets = Option.Rest();
|
|
|
|
verbose = Option.Boolean('--verbose,-v', false, {
|
|
description: 'Show detailed quality breakdown',
|
|
});
|
|
|
|
quality = Option.Boolean('--quality,-q', false, {
|
|
description: 'Only check quality (skip format validation)',
|
|
});
|
|
|
|
issues = Option.Boolean('--issues,-i', false, {
|
|
description: 'Only show skills with issues',
|
|
});
|
|
|
|
minScore = Option.String('--min-score,-m', {
|
|
description: 'Minimum acceptable score (default: 60)',
|
|
});
|
|
|
|
json = Option.Boolean('--json', false, {
|
|
description: 'Output as JSON',
|
|
});
|
|
|
|
all = Option.Boolean('--all,-a', false, {
|
|
description: 'Validate all skills in directory',
|
|
});
|
|
|
|
benchmark = Option.Boolean('--benchmark,-b', false, {
|
|
description: 'Compare skill against category leaders',
|
|
});
|
|
|
|
fix = Option.Boolean('--fix', false, {
|
|
description: 'Automatically fix common quality issues (runs fix command)',
|
|
});
|
|
|
|
async execute(): Promise<number> {
|
|
if (this.fix) {
|
|
if (this.targets.length === 0) {
|
|
error('--fix requires at least one target');
|
|
console.log(colors.muted('Usage: skillkit validate --fix <skill-name>'));
|
|
return 1;
|
|
}
|
|
const { FixCommand } = await import('./fix.js');
|
|
const fixCmd = new FixCommand();
|
|
fixCmd.targets = this.targets;
|
|
fixCmd.yes = false;
|
|
return fixCmd.execute();
|
|
}
|
|
|
|
let minScore = 60;
|
|
if (this.minScore) {
|
|
const parsed = parseInt(this.minScore, 10);
|
|
if (Number.isNaN(parsed) || parsed < 0 || parsed > 100) {
|
|
if (!this.json) {
|
|
warn(`Invalid --min-score value "${this.minScore}", using default: 60`);
|
|
}
|
|
} else {
|
|
minScore = parsed;
|
|
}
|
|
}
|
|
const results: Array<{ name: string; quality: QualityScore | null; formatValid: boolean; formatErrors: string[]; path?: string }> = [];
|
|
|
|
if (this.targets.length > 0) {
|
|
for (const target of this.targets) {
|
|
const resolved = resolve(target);
|
|
|
|
if (existsSync(resolved)) {
|
|
const isDirectory = statSync(resolved).isDirectory();
|
|
if (this.all) {
|
|
if (!isDirectory) {
|
|
if (!this.json) {
|
|
warn(`--all requires a directory, but "${target}" is a file`);
|
|
}
|
|
continue;
|
|
}
|
|
const entries = readdirSync(resolved, { withFileTypes: true });
|
|
for (const entry of entries) {
|
|
if (entry.isDirectory()) {
|
|
const skillPath = join(resolved, entry.name);
|
|
const hasSkillMd = existsSync(join(skillPath, 'SKILL.md'));
|
|
const hasIndexMdc = existsSync(join(skillPath, 'index.mdc'));
|
|
const hasNamedMdc = existsSync(join(skillPath, `${entry.name}.mdc`));
|
|
if (hasSkillMd || hasIndexMdc || hasNamedMdc) {
|
|
const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(skillPath);
|
|
const quality = evaluateSkillDirectory(skillPath);
|
|
results.push({
|
|
name: entry.name,
|
|
quality,
|
|
formatValid: formatResult.valid,
|
|
formatErrors: formatResult.errors || [],
|
|
path: skillPath,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
const isFile = resolved.endsWith('.md') || resolved.endsWith('.mdc');
|
|
const skillDir = isFile ? dirname(resolved) : resolved;
|
|
const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(skillDir);
|
|
const quality = isFile ? evaluateSkillFile(resolved) : evaluateSkillDirectory(resolved);
|
|
results.push({
|
|
name: basename(target),
|
|
quality,
|
|
formatValid: formatResult.valid,
|
|
formatErrors: formatResult.errors || [],
|
|
path: resolved,
|
|
});
|
|
}
|
|
} else {
|
|
const searchDirs = getSearchDirs();
|
|
const skills = findAllSkills(searchDirs);
|
|
const found = skills.find(s => s.name === target);
|
|
|
|
if (found) {
|
|
const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(found.path);
|
|
const quality = evaluateSkillDirectory(found.path);
|
|
results.push({
|
|
name: found.name,
|
|
quality,
|
|
formatValid: formatResult.valid,
|
|
formatErrors: formatResult.errors || [],
|
|
path: found.path,
|
|
});
|
|
} else {
|
|
if (!this.json) {
|
|
warn(`Skill not found: ${target}`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
const searchDirs = getSearchDirs();
|
|
const skills = findAllSkills(searchDirs);
|
|
|
|
if (skills.length === 0) {
|
|
if (!this.json) {
|
|
warn('No installed skills found');
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
const s = spinner();
|
|
if (!this.json) {
|
|
s.start(`Evaluating ${skills.length} skill(s)...`);
|
|
}
|
|
|
|
for (const skill of skills) {
|
|
const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(skill.path);
|
|
const quality = evaluateSkillDirectory(skill.path);
|
|
results.push({
|
|
name: skill.name,
|
|
quality,
|
|
formatValid: formatResult.valid,
|
|
formatErrors: formatResult.errors || [],
|
|
path: skill.path,
|
|
});
|
|
}
|
|
|
|
if (!this.json) {
|
|
s.stop(`Evaluated ${results.length} skill(s)`);
|
|
}
|
|
}
|
|
|
|
if (results.length === 0) {
|
|
if (!this.json) {
|
|
error('No skills to validate');
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
if (this.json) {
|
|
const output = results.map(r => {
|
|
const result: Record<string, unknown> = {
|
|
name: r.name,
|
|
path: r.path,
|
|
formatValid: r.formatValid,
|
|
formatErrors: r.formatErrors,
|
|
score: r.quality?.overall ?? null,
|
|
grade: r.quality ? getQualityGrade(r.quality.overall) : null,
|
|
highQuality: r.quality ? isHighQuality(r.quality) : false,
|
|
structure: r.quality?.structure ?? null,
|
|
clarity: r.quality?.clarity ?? null,
|
|
specificity: r.quality?.specificity ?? null,
|
|
advanced: r.quality?.advanced ?? null,
|
|
warnings: r.quality?.warnings ?? [],
|
|
suggestions: r.quality?.suggestions ?? [],
|
|
};
|
|
if (this.benchmark && r.quality) {
|
|
result.benchmark = benchmarkSkill(r.name, r.quality);
|
|
}
|
|
return result;
|
|
});
|
|
console.log(JSON.stringify(output, null, 2));
|
|
const hasFormatFailures = results.some(r => !r.formatValid);
|
|
const hasBelowThreshold = results.some(r => r.quality && r.quality.overall < minScore);
|
|
return hasFormatFailures || hasBelowThreshold ? 1 : 0;
|
|
}
|
|
|
|
header('Skill Validation Report');
|
|
|
|
let filteredResults = results;
|
|
if (this.issues) {
|
|
filteredResults = results.filter(r =>
|
|
!r.formatValid ||
|
|
(r.quality && r.quality.overall < minScore) ||
|
|
(r.quality && r.quality.warnings.length > 0)
|
|
);
|
|
}
|
|
|
|
filteredResults.sort((a, b) => (b.quality?.overall ?? 0) - (a.quality?.overall ?? 0));
|
|
|
|
for (const { name, quality, formatValid, formatErrors } of filteredResults) {
|
|
if (!formatValid) {
|
|
console.log(`\n${colors.error(symbols.error)} ${colors.primary(name)}`);
|
|
console.log(` ${colors.error('Format validation failed:')}`);
|
|
for (const err of formatErrors) {
|
|
console.log(` ${colors.error(symbols.bullet)} ${err}`);
|
|
}
|
|
}
|
|
|
|
if (quality) {
|
|
printQualityReport(name, quality, this.verbose);
|
|
|
|
if (this.benchmark) {
|
|
const benchmark = benchmarkSkill(name, quality);
|
|
printBenchmarkReport(benchmark);
|
|
}
|
|
}
|
|
}
|
|
|
|
console.log('');
|
|
|
|
const formatFailed = results.filter(r => !r.formatValid).length;
|
|
const withQuality = results.filter(r => r.quality !== null);
|
|
const highQuality = withQuality.filter(r => r.quality!.overall >= 80).length;
|
|
const passing = withQuality.filter(r => r.quality!.overall >= minScore).length;
|
|
const failing = withQuality.filter(r => r.quality!.overall < minScore).length;
|
|
|
|
console.log(colors.muted('Summary:'));
|
|
|
|
if (!this.quality) {
|
|
console.log(` ${formatFailed === 0 ? colors.success(symbols.success) : colors.error(symbols.error)} Format valid: ${results.length - formatFailed}/${results.length}`);
|
|
}
|
|
|
|
console.log(` ${colors.success(symbols.success)} High quality (80+): ${highQuality}`);
|
|
console.log(` ${colors.primary(symbols.bullet)} Passing (${minScore}+): ${passing}`);
|
|
|
|
if (failing > 0) {
|
|
console.log(` ${colors.error(symbols.error)} Below threshold: ${failing}`);
|
|
}
|
|
|
|
if (withQuality.length > 0) {
|
|
const avgScore = Math.round(withQuality.reduce((sum, r) => sum + r.quality!.overall, 0) / withQuality.length);
|
|
console.log(` ${colors.muted('Average score:')} ${formatScore(avgScore)}/100`);
|
|
}
|
|
|
|
if (formatFailed > 0 || failing > 0) {
|
|
console.log('');
|
|
if (formatFailed > 0) {
|
|
warn(`${formatFailed} skill(s) failed format validation`);
|
|
}
|
|
if (failing > 0) {
|
|
warn(`${failing} skill(s) below quality threshold`);
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
console.log('');
|
|
success('All skills validated successfully');
|
|
return 0;
|
|
}
|
|
}
|