Files
skillkit/packages/cli/src/commands/validate.ts
T
Rohit Ghumare b3f561dbc9 feat(quality): Quality Evaluation System Enhancements (#21)
* feat(quality): add skill quality evaluation system

- Add quality scoring module with structure, clarity, and specificity metrics
- Evaluate skills for metadata, triggers, boundaries, code examples
- Detect vague terms and penalize overly long skills
- Update validate command to include quality reports
- Add --verbose, --quality, --issues, --json flags
- Quality scores: A (90+), B (80+), C (70+), D (60+), F (<60)

* fix: address CodeRabbit and Devin review suggestions

- Fix JSON output to return exit code 1 on validation failures
- Add --min-score NaN validation with warning message
- Add directory guard for --all flag with .mdc file support
- Fix CRLF handling in frontmatter extraction
- Fix code blocks double-counting in specificity evaluation
- Fix warning message inconsistency for code examples
- Add try/catch error handling in evaluateSkillFile
- Fix isHighQuality threshold to match UI display (80)

* feat(quality): extend quality evaluation system with badges, deeper analysis, and fix command

- Phase 2: Add quality badges to CLI commands (list, recommend, install)
  - formatQualityBadge() helper for color-coded grade badges
  - --quality flag for list command
  - Low-quality skill warnings during install

- Phase 3: Add TUI quality badges
  - Extended SkillItem with quality, grade, warnings fields
  - Quality evaluation in loadSkills()
  - Grade badges in Installed and Recommend screens

- Phase 4: Deeper quality analysis
  - Deprecated pattern detection (CommonJS, class components, var)
  - Security pattern detection (hardcoded credentials, eval, innerHTML)
  - Conflicting instruction detection
  - Completeness assessment (TODOs, empty sections, example coverage)
  - New weight distribution: Structure 35%, Clarity 25%, Specificity 25%, Advanced 15%

- Phase 1: Pre-compute marketplace quality scores
  - Updated schema.json with quality field (0-100)
  - Created scripts/compute-quality.ts for batch computation

- Phase 5: Benchmark against category leaders
  - New benchmark.ts module with category detection
  - Percentile calculation and comparison notes
  - --benchmark flag for validate command

- Phase 6: Fix command for automated quality fixes
  - skillkit fix command with --dry-run and --interactive modes
  - Auto-generates frontmatter, "When to Use", "Boundaries" sections
  - --fix flag alias on validate command

* fix: address CodeRabbit and Devin review suggestions

- Fix .mdc file name extraction in fix.ts (handles both .md and .mdc)
- Add missing advanced field to test fixtures in quality.test.ts
- Fix duplicate header detection in assessCompleteness using regex.exec
- Fix ESM compatibility in compute-quality.ts using import.meta.url
- Add trailing newline to skills.json output

* fix: address remaining CodeRabbit review suggestions

- Escape regex metacharacters in vague term patterns (fix.ts)
- Improve diff algorithm using set-based comparison (fix.ts)
- Fix validateSkill receiving file path instead of directory (validate.ts)
- Check final section in completeness assessment (index.ts)
- Add error handling and empty skills guard (compute-quality.ts)

* fix: address additional review feedback

- validate.ts: Add error when --fix is passed without targets
- index.ts: Skip headers inside fenced code blocks during completeness check
- compute-quality.ts: Use nullish coalescing (??) to preserve zero quality scores
2026-01-28 14:18:42 +00:00

442 lines
16 KiB
TypeScript

import { existsSync, readdirSync, statSync } from 'node:fs';
import { join, basename, resolve, dirname } from 'node:path';
import { Command, Option } from 'clipanion';
import {
validateSkill,
evaluateSkillFile,
evaluateSkillDirectory,
getQualityGrade,
isHighQuality,
findAllSkills,
benchmarkSkill,
type QualityScore,
type BenchmarkResult,
} from '@skillkit/core';
import { getSearchDirs } from '../helpers.js';
import {
colors,
symbols,
spinner,
success,
warn,
error,
header,
} from '../onboarding/index.js';
function formatScore(score: number): string {
if (score >= 80) return colors.success(`${score}`);
if (score >= 60) return colors.warning(`${score}`);
return colors.error(`${score}`);
}
function formatGrade(grade: string): string {
switch (grade) {
case 'A':
case 'B':
return colors.success(grade);
case 'C':
case 'D':
return colors.warning(grade);
default:
return colors.error(grade);
}
}
function printQualityReport(name: string, quality: QualityScore, verbose: boolean): void {
const grade = getQualityGrade(quality.overall);
const gradeDisplay = formatGrade(grade);
const scoreDisplay = formatScore(quality.overall);
console.log(`\n${colors.primary(name)}`);
console.log(` Grade: ${gradeDisplay} Score: ${scoreDisplay}/100`);
if (verbose) {
console.log('');
console.log(` ${colors.muted('Structure:')} ${formatScore(quality.structure.score)}/100`);
console.log(` ${quality.structure.hasMetadata ? colors.success(symbols.success) : colors.error(symbols.error)} Metadata`);
console.log(` ${quality.structure.hasDescription ? colors.success(symbols.success) : colors.error(symbols.error)} Description`);
console.log(` ${quality.structure.hasTriggers ? colors.success(symbols.success) : colors.error(symbols.error)} Trigger conditions`);
console.log(` ${quality.structure.hasExamples ? colors.success(symbols.success) : colors.error(symbols.error)} Code examples`);
console.log(` ${quality.structure.hasBoundaries ? colors.success(symbols.success) : colors.error(symbols.error)} Boundaries defined`);
console.log('');
console.log(` ${colors.muted('Clarity:')} ${formatScore(quality.clarity.score)}/100`);
console.log(` Lines: ${quality.clarity.lineCount}${quality.clarity.lineCount > 300 ? colors.warning(' (consider splitting)') : ''}`);
console.log(` Tokens: ${quality.clarity.tokenCount}${quality.clarity.tokenCount > 2000 ? colors.warning(' (high)') : ''}`);
console.log(` ${quality.clarity.hasHeaders ? colors.success(symbols.success) : colors.error(symbols.error)} Uses headers`);
console.log('');
console.log(` ${colors.muted('Specificity:')} ${formatScore(quality.specificity.score)}/100`);
console.log(` ${quality.specificity.hasConcreteCommands ? colors.success(symbols.success) : colors.error(symbols.error)} Concrete commands`);
console.log(` ${quality.specificity.hasFilePatterns ? colors.success(symbols.success) : colors.error(symbols.error)} File patterns`);
console.log(` ${quality.specificity.hasCodeExamples ? colors.success(symbols.success) : colors.error(symbols.error)} Multiple code examples`);
if (quality.specificity.vagueTermCount > 0) {
console.log(` ${colors.warning(symbols.warning)} ${quality.specificity.vagueTermCount} vague term(s)`);
}
console.log('');
console.log(` ${colors.muted('Advanced:')} ${formatScore(quality.advanced.score)}/100`);
if (quality.advanced.deprecatedPatterns.length > 0) {
console.log(` ${colors.warning(symbols.warning)} ${quality.advanced.deprecatedPatterns.length} deprecated pattern(s)`);
for (const pattern of quality.advanced.deprecatedPatterns) {
console.log(` ${colors.muted(symbols.bullet)} ${pattern}`);
}
}
if (quality.advanced.securityIssues.length > 0) {
console.log(` ${colors.error(symbols.error)} ${quality.advanced.securityIssues.length} security issue(s)`);
for (const issue of quality.advanced.securityIssues) {
console.log(` ${colors.muted(symbols.bullet)} ${issue}`);
}
}
if (quality.advanced.conflictingInstructions.length > 0) {
console.log(` ${colors.warning(symbols.warning)} ${quality.advanced.conflictingInstructions.length} conflicting instruction(s)`);
}
console.log(` Completeness: ${quality.advanced.completeness.score}/100`);
console.log(` Example coverage: ${quality.advanced.completeness.exampleCoverage}%`);
if (quality.advanced.completeness.hasTodos) {
console.log(` ${colors.warning(symbols.warning)} ${quality.advanced.completeness.todoCount} TODO/FIXME comment(s)`);
}
}
if (quality.warnings.length > 0) {
console.log('');
console.log(` ${colors.warning('Warnings:')}`);
for (const w of quality.warnings) {
console.log(` ${colors.warning(symbols.warning)} ${w}`);
}
}
if (quality.suggestions.length > 0 && verbose) {
console.log('');
console.log(` ${colors.muted('Suggestions:')}`);
for (const s of quality.suggestions) {
console.log(` ${colors.muted(symbols.bullet)} ${s}`);
}
}
}
function printBenchmarkReport(benchmark: BenchmarkResult): void {
console.log('');
console.log(colors.bold('Benchmark Comparison'));
console.log('');
console.log(` ${colors.muted('Your Score:')} ${formatScore(benchmark.score)}/100 ${formatGrade(benchmark.grade)}`);
console.log(` ${colors.muted('Category Average:')} ${benchmark.categoryAvg}/100`);
console.log(` ${colors.muted('Top Skill Score:')} ${benchmark.topSkillScore}/100`);
console.log(` ${colors.muted('Percentile:')} ${benchmark.percentile}${getOrdinal(benchmark.percentile)}`);
if (benchmark.comparisonNotes.length > 0) {
console.log('');
console.log(` ${colors.muted('Comparison:')}`);
for (const note of benchmark.comparisonNotes) {
console.log(` ${colors.muted(symbols.bullet)} ${note}`);
}
}
if (benchmark.recommendations.length > 0) {
console.log('');
console.log(` ${colors.muted('To Improve:')}`);
for (const rec of benchmark.recommendations) {
console.log(` ${colors.info(symbols.info)} ${rec}`);
}
}
}
function getOrdinal(n: number): string {
const s = ['th', 'st', 'nd', 'rd'];
const v = n % 100;
return s[(v - 20) % 10] || s[v] || s[0];
}
export class ValidateCommand extends Command {
static override paths = [['validate']];
static override usage = Command.Usage({
description: 'Validate skill format and quality',
details: `
Validates skills against the specification and evaluates quality.
Quality scores are based on structure, clarity, and specificity.
`,
examples: [
['Validate all installed skills', '$0 validate'],
['Validate specific skill', '$0 validate my-skill'],
['Validate skill file', '$0 validate ./path/to/SKILL.md'],
['Detailed quality report', '$0 validate --verbose'],
['Only show issues', '$0 validate --issues'],
['Quality check only', '$0 validate --quality'],
['JSON output', '$0 validate --json'],
],
});
targets = Option.Rest();
verbose = Option.Boolean('--verbose,-v', false, {
description: 'Show detailed quality breakdown',
});
quality = Option.Boolean('--quality,-q', false, {
description: 'Only check quality (skip format validation)',
});
issues = Option.Boolean('--issues,-i', false, {
description: 'Only show skills with issues',
});
minScore = Option.String('--min-score,-m', {
description: 'Minimum acceptable score (default: 60)',
});
json = Option.Boolean('--json', false, {
description: 'Output as JSON',
});
all = Option.Boolean('--all,-a', false, {
description: 'Validate all skills in directory',
});
benchmark = Option.Boolean('--benchmark,-b', false, {
description: 'Compare skill against category leaders',
});
fix = Option.Boolean('--fix', false, {
description: 'Automatically fix common quality issues (runs fix command)',
});
async execute(): Promise<number> {
if (this.fix) {
if (this.targets.length === 0) {
error('--fix requires at least one target');
console.log(colors.muted('Usage: skillkit validate --fix <skill-name>'));
return 1;
}
const { FixCommand } = await import('./fix.js');
const fixCmd = new FixCommand();
fixCmd.targets = this.targets;
fixCmd.yes = false;
return fixCmd.execute();
}
let minScore = 60;
if (this.minScore) {
const parsed = parseInt(this.minScore, 10);
if (Number.isNaN(parsed) || parsed < 0 || parsed > 100) {
if (!this.json) {
warn(`Invalid --min-score value "${this.minScore}", using default: 60`);
}
} else {
minScore = parsed;
}
}
const results: Array<{ name: string; quality: QualityScore | null; formatValid: boolean; formatErrors: string[]; path?: string }> = [];
if (this.targets.length > 0) {
for (const target of this.targets) {
const resolved = resolve(target);
if (existsSync(resolved)) {
const isDirectory = statSync(resolved).isDirectory();
if (this.all) {
if (!isDirectory) {
if (!this.json) {
warn(`--all requires a directory, but "${target}" is a file`);
}
continue;
}
const entries = readdirSync(resolved, { withFileTypes: true });
for (const entry of entries) {
if (entry.isDirectory()) {
const skillPath = join(resolved, entry.name);
const hasSkillMd = existsSync(join(skillPath, 'SKILL.md'));
const hasIndexMdc = existsSync(join(skillPath, 'index.mdc'));
const hasNamedMdc = existsSync(join(skillPath, `${entry.name}.mdc`));
if (hasSkillMd || hasIndexMdc || hasNamedMdc) {
const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(skillPath);
const quality = evaluateSkillDirectory(skillPath);
results.push({
name: entry.name,
quality,
formatValid: formatResult.valid,
formatErrors: formatResult.errors || [],
path: skillPath,
});
}
}
}
} else {
const isFile = resolved.endsWith('.md') || resolved.endsWith('.mdc');
const skillDir = isFile ? dirname(resolved) : resolved;
const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(skillDir);
const quality = isFile ? evaluateSkillFile(resolved) : evaluateSkillDirectory(resolved);
results.push({
name: basename(target),
quality,
formatValid: formatResult.valid,
formatErrors: formatResult.errors || [],
path: resolved,
});
}
} else {
const searchDirs = getSearchDirs();
const skills = findAllSkills(searchDirs);
const found = skills.find(s => s.name === target);
if (found) {
const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(found.path);
const quality = evaluateSkillDirectory(found.path);
results.push({
name: found.name,
quality,
formatValid: formatResult.valid,
formatErrors: formatResult.errors || [],
path: found.path,
});
} else {
if (!this.json) {
warn(`Skill not found: ${target}`);
}
}
}
}
} else {
const searchDirs = getSearchDirs();
const skills = findAllSkills(searchDirs);
if (skills.length === 0) {
if (!this.json) {
warn('No installed skills found');
}
return 0;
}
const s = spinner();
if (!this.json) {
s.start(`Evaluating ${skills.length} skill(s)...`);
}
for (const skill of skills) {
const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(skill.path);
const quality = evaluateSkillDirectory(skill.path);
results.push({
name: skill.name,
quality,
formatValid: formatResult.valid,
formatErrors: formatResult.errors || [],
path: skill.path,
});
}
if (!this.json) {
s.stop(`Evaluated ${results.length} skill(s)`);
}
}
if (results.length === 0) {
if (!this.json) {
error('No skills to validate');
}
return 1;
}
if (this.json) {
const output = results.map(r => {
const result: Record<string, unknown> = {
name: r.name,
path: r.path,
formatValid: r.formatValid,
formatErrors: r.formatErrors,
score: r.quality?.overall ?? null,
grade: r.quality ? getQualityGrade(r.quality.overall) : null,
highQuality: r.quality ? isHighQuality(r.quality) : false,
structure: r.quality?.structure ?? null,
clarity: r.quality?.clarity ?? null,
specificity: r.quality?.specificity ?? null,
advanced: r.quality?.advanced ?? null,
warnings: r.quality?.warnings ?? [],
suggestions: r.quality?.suggestions ?? [],
};
if (this.benchmark && r.quality) {
result.benchmark = benchmarkSkill(r.name, r.quality);
}
return result;
});
console.log(JSON.stringify(output, null, 2));
const hasFormatFailures = results.some(r => !r.formatValid);
const hasBelowThreshold = results.some(r => r.quality && r.quality.overall < minScore);
return hasFormatFailures || hasBelowThreshold ? 1 : 0;
}
header('Skill Validation Report');
let filteredResults = results;
if (this.issues) {
filteredResults = results.filter(r =>
!r.formatValid ||
(r.quality && r.quality.overall < minScore) ||
(r.quality && r.quality.warnings.length > 0)
);
}
filteredResults.sort((a, b) => (b.quality?.overall ?? 0) - (a.quality?.overall ?? 0));
for (const { name, quality, formatValid, formatErrors } of filteredResults) {
if (!formatValid) {
console.log(`\n${colors.error(symbols.error)} ${colors.primary(name)}`);
console.log(` ${colors.error('Format validation failed:')}`);
for (const err of formatErrors) {
console.log(` ${colors.error(symbols.bullet)} ${err}`);
}
}
if (quality) {
printQualityReport(name, quality, this.verbose);
if (this.benchmark) {
const benchmark = benchmarkSkill(name, quality);
printBenchmarkReport(benchmark);
}
}
}
console.log('');
const formatFailed = results.filter(r => !r.formatValid).length;
const withQuality = results.filter(r => r.quality !== null);
const highQuality = withQuality.filter(r => r.quality!.overall >= 80).length;
const passing = withQuality.filter(r => r.quality!.overall >= minScore).length;
const failing = withQuality.filter(r => r.quality!.overall < minScore).length;
console.log(colors.muted('Summary:'));
if (!this.quality) {
console.log(` ${formatFailed === 0 ? colors.success(symbols.success) : colors.error(symbols.error)} Format valid: ${results.length - formatFailed}/${results.length}`);
}
console.log(` ${colors.success(symbols.success)} High quality (80+): ${highQuality}`);
console.log(` ${colors.primary(symbols.bullet)} Passing (${minScore}+): ${passing}`);
if (failing > 0) {
console.log(` ${colors.error(symbols.error)} Below threshold: ${failing}`);
}
if (withQuality.length > 0) {
const avgScore = Math.round(withQuality.reduce((sum, r) => sum + r.quality!.overall, 0) / withQuality.length);
console.log(` ${colors.muted('Average score:')} ${formatScore(avgScore)}/100`);
}
if (formatFailed > 0 || failing > 0) {
console.log('');
if (formatFailed > 0) {
warn(`${formatFailed} skill(s) failed format validation`);
}
if (failing > 0) {
warn(`${failing} skill(s) below quality threshold`);
}
return 1;
}
console.log('');
success('All skills validated successfully');
return 0;
}
}