feat(quality): Quality Evaluation System Enhancements (#21)

* feat(quality): add skill quality evaluation system

- Add quality scoring module with structure, clarity, and specificity metrics
- Evaluate skills for metadata, triggers, boundaries, code examples
- Detect vague terms and penalize overly long skills
- Update validate command to include quality reports
- Add --verbose, --quality, --issues, --json flags
- Quality scores: A (90+), B (80+), C (70+), D (60+), F (<60)

* fix: address CodeRabbit and Devin review suggestions

- Fix JSON output to return exit code 1 on validation failures
- Add --min-score NaN validation with warning message
- Add directory guard for --all flag with .mdc file support
- Fix CRLF handling in frontmatter extraction
- Fix code blocks double-counting in specificity evaluation
- Fix warning message inconsistency for code examples
- Add try/catch error handling in evaluateSkillFile
- Fix isHighQuality threshold to match UI display (80)

* feat(quality): extend quality evaluation system with badges, deeper analysis, and fix command

- Phase 2: Add quality badges to CLI commands (list, recommend, install)
  - formatQualityBadge() helper for color-coded grade badges
  - --quality flag for list command
  - Low-quality skill warnings during install

- Phase 3: Add TUI quality badges
  - Extended SkillItem with quality, grade, warnings fields
  - Quality evaluation in loadSkills()
  - Grade badges in Installed and Recommend screens

- Phase 4: Deeper quality analysis
  - Deprecated pattern detection (CommonJS, class components, var)
  - Security pattern detection (hardcoded credentials, eval, innerHTML)
  - Conflicting instruction detection
  - Completeness assessment (TODOs, empty sections, example coverage)
  - New weight distribution: Structure 35%, Clarity 25%, Specificity 25%, Advanced 15%

- Phase 1: Pre-compute marketplace quality scores
  - Updated schema.json with quality field (0-100)
  - Created scripts/compute-quality.ts for batch computation

- Phase 5: Benchmark against category leaders
  - New benchmark.ts module with category detection
  - Percentile calculation and comparison notes
  - --benchmark flag for validate command

- Phase 6: Fix command for automated quality fixes
  - skillkit fix command with --dry-run and --interactive modes
  - Auto-generates frontmatter, "When to Use", "Boundaries" sections
  - --fix flag alias on validate command

* fix: address CodeRabbit and Devin review suggestions

- Fix .mdc file name extraction in fix.ts (handles both .md and .mdc)
- Add missing advanced field to test fixtures in quality.test.ts
- Fix duplicate header detection in assessCompleteness using regex.exec
- Fix ESM compatibility in compute-quality.ts using import.meta.url
- Add trailing newline to skills.json output

* fix: address remaining CodeRabbit review suggestions

- Escape regex metacharacters in vague term patterns (fix.ts)
- Improve diff algorithm using set-based comparison (fix.ts)
- Fix validateSkill receiving file path instead of directory (validate.ts)
- Check final section in completeness assessment (index.ts)
- Add error handling and empty skills guard (compute-quality.ts)

* fix: address additional review feedback

- validate.ts: Add error when --fix is passed without targets
- index.ts: Skip headers inside fenced code blocks during completeness check
- compute-quality.ts: Use nullish coalescing (??) to preserve zero quality scores
This commit is contained in:
Rohit Ghumare
2026-01-28 14:18:42 +00:00
committed by GitHub
parent 0a59bcf314
commit b3f561dbc9
18 changed files with 1280 additions and 51 deletions
+2
View File
@@ -14,6 +14,7 @@ import {
RemoveCommand,
InitCommand,
ValidateCommand,
FixCommand,
CreateCommand,
UICommand,
TranslateCommand,
@@ -83,6 +84,7 @@ cli.register(UpdateCommand);
cli.register(RemoveCommand);
cli.register(InitCommand);
cli.register(ValidateCommand);
cli.register(FixCommand);
cli.register(CreateCommand);
cli.register(UICommand);
cli.register(TranslateCommand);
+6
View File
@@ -92,6 +92,12 @@
]
},
"description": "Compatible AI agents (omit for universal compatibility)"
},
"quality": {
"type": "integer",
"minimum": 0,
"maximum": 100,
"description": "Pre-computed quality score (0-100). Verified sources default to 85."
}
}
}
+384
View File
@@ -0,0 +1,384 @@
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
import { join, resolve, basename } from 'node:path';
import { Command, Option } from 'clipanion';
import {
evaluateSkillFile,
findAllSkills,
type QualityScore,
} from '@skillkit/core';
import { getSearchDirs } from '../helpers.js';
import {
colors,
symbols,
success,
warn,
error,
header,
confirm,
} from '../onboarding/index.js';
interface FixSuggestion {
type: 'add' | 'replace' | 'format';
description: string;
original?: string;
replacement: string;
line?: number;
automatic: boolean;
}
const FRONTMATTER_TEMPLATE = `---
name: {{NAME}}
description: {{DESCRIPTION}}
globs:
- "**/*"
---
`;
const WHEN_TO_USE_TEMPLATE = `
## When to Use
Use this skill when:
- [Add specific trigger conditions]
- [Add relevant patterns to detect]
`;
const BOUNDARIES_TEMPLATE = `
## Boundaries
This skill should NOT:
- [Add constraints and limitations]
- [Specify what the agent should never do]
`;
const VAGUE_TERM_REPLACEMENTS: Record<string, string> = {
'be helpful': 'follow the specific instructions below',
'assist the user': 'complete the requested task',
'help with': 'handle',
'try to': '',
'attempt to': '',
'do your best': 'follow these guidelines',
'as needed': 'when the following conditions are met',
'when appropriate': 'when',
'if necessary': 'when required',
'general purpose': 'designed for',
'various tasks': 'the following tasks',
'many things': 'the following',
'etc.': '',
'and so on': '',
'stuff like that': '',
};
function generateFixes(content: string, quality: QualityScore, skillName: string): FixSuggestion[] {
const fixes: FixSuggestion[] = [];
if (!quality.structure.hasMetadata) {
const template = FRONTMATTER_TEMPLATE
.replace('{{NAME}}', skillName)
.replace('{{DESCRIPTION}}', 'Add a description for this skill');
fixes.push({
type: 'add',
description: 'Add YAML frontmatter template',
replacement: template,
automatic: true,
});
}
if (!quality.structure.hasTriggers && !quality.structure.hasWhenToUse) {
fixes.push({
type: 'add',
description: 'Add "When to Use" section scaffold',
replacement: WHEN_TO_USE_TEMPLATE,
automatic: true,
});
}
if (!quality.structure.hasBoundaries) {
fixes.push({
type: 'add',
description: 'Add "Boundaries" section scaffold',
replacement: BOUNDARIES_TEMPLATE,
automatic: true,
});
}
for (const [vague, replacement] of Object.entries(VAGUE_TERM_REPLACEMENTS)) {
const escaped = vague.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const regex = new RegExp(escaped, 'gi');
if (regex.test(content)) {
fixes.push({
type: 'replace',
description: `Replace vague term "${vague}"`,
original: vague,
replacement: replacement || '[be specific]',
automatic: false,
});
}
}
const codeBlockPattern = /```(\s*\n[\s\S]*?```)/g;
let match;
while ((match = codeBlockPattern.exec(content)) !== null) {
if (match[1].startsWith('\n') && !content.slice(match.index, match.index + 10).match(/```\w/)) {
fixes.push({
type: 'format',
description: 'Add language tag to code block',
original: '```',
replacement: '```typescript',
automatic: false,
});
break;
}
}
return fixes;
}
function applyFixes(content: string, fixes: FixSuggestion[]): string {
let result = content;
const hasFrontmatter = /^---\s*\n/.test(content);
for (const fix of fixes) {
if (fix.type === 'add') {
if (fix.description.includes('frontmatter') && !hasFrontmatter) {
result = fix.replacement + result;
} else if (fix.description.includes('When to Use') || fix.description.includes('Boundaries')) {
result = result.trimEnd() + '\n' + fix.replacement;
}
} else if (fix.type === 'replace' && fix.original && fix.automatic) {
const escaped = fix.original.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const regex = new RegExp(escaped, 'gi');
result = result.replace(regex, fix.replacement);
}
}
return result;
}
export class FixCommand extends Command {
static override paths = [['fix']];
static override usage = Command.Usage({
description: 'Automatically fix common skill quality issues',
details: `
The fix command analyzes skills and applies automatic fixes for common quality issues.
Fixes that can be automated:
- Add missing YAML frontmatter template
- Add "When to Use" section scaffold
- Add "Boundaries" section scaffold
- Format code blocks with language tags
Interactive mode prompts before each fix.
`,
examples: [
['Fix a specific skill', '$0 fix my-skill'],
['Fix skill file', '$0 fix ./path/to/SKILL.md'],
['Preview fixes without applying', '$0 fix my-skill --dry-run'],
['Apply all fixes without prompts', '$0 fix my-skill --yes'],
],
});
targets = Option.Rest();
dryRun = Option.Boolean('--dry-run,-d', false, {
description: 'Show proposed changes without applying',
});
yes = Option.Boolean('--yes,-y', false, {
description: 'Apply all automatic fixes without prompting',
});
interactive = Option.Boolean('--interactive,-i', {
description: 'Prompt for each fix (default when not using --yes)',
});
async execute(): Promise<number> {
const isInteractive = this.interactive !== false && !this.yes && !this.dryRun && process.stdin.isTTY;
if (this.targets.length === 0) {
error('Please specify a skill to fix');
console.log(colors.muted('Usage: skillkit fix <skill-name> or skillkit fix ./path/to/SKILL.md'));
return 1;
}
header('Skill Quality Fixer');
let totalFixes = 0;
let filesFixed = 0;
for (const target of this.targets) {
const resolved = resolve(target);
let filePath: string | null = null;
let skillName = basename(target);
if (existsSync(resolved)) {
if (resolved.endsWith('.md') || resolved.endsWith('.mdc')) {
filePath = resolved;
const fileName = basename(resolved);
if (fileName.endsWith('.mdc')) {
skillName = fileName.slice(0, -4);
} else if (fileName.endsWith('.md')) {
skillName = fileName.slice(0, -3);
} else {
skillName = fileName;
}
} else {
const skillMd = join(resolved, 'SKILL.md');
const indexMdc = join(resolved, 'index.mdc');
if (existsSync(skillMd)) {
filePath = skillMd;
} else if (existsSync(indexMdc)) {
filePath = indexMdc;
}
}
} else {
const searchDirs = getSearchDirs();
const skills = findAllSkills(searchDirs);
const found = skills.find(s => s.name === target);
if (found) {
const skillMd = join(found.path, 'SKILL.md');
const indexMdc = join(found.path, 'index.mdc');
if (existsSync(skillMd)) {
filePath = skillMd;
} else if (existsSync(indexMdc)) {
filePath = indexMdc;
}
skillName = found.name;
}
}
if (!filePath || !existsSync(filePath)) {
warn(`Skill not found: ${target}`);
continue;
}
const content = readFileSync(filePath, 'utf-8');
const quality = evaluateSkillFile(filePath);
if (!quality) {
warn(`Could not evaluate: ${target}`);
continue;
}
console.log('');
console.log(colors.bold(skillName));
console.log(colors.muted(`Score: ${quality.overall}/100`));
const fixes = generateFixes(content, quality, skillName);
const automaticFixes = fixes.filter(f => f.automatic);
const manualFixes = fixes.filter(f => !f.automatic);
if (fixes.length === 0) {
console.log(colors.success(`${symbols.success} No automatic fixes needed`));
continue;
}
console.log('');
console.log(colors.muted(`Found ${fixes.length} potential fix(es):`));
for (const fix of automaticFixes) {
console.log(` ${colors.success(symbols.success)} ${fix.description} ${colors.muted('(automatic)')}`);
}
for (const fix of manualFixes) {
console.log(` ${colors.warning(symbols.warning)} ${fix.description} ${colors.muted('(manual review)')}`);
}
if (this.dryRun) {
console.log('');
console.log(colors.muted('Dry run - no changes applied'));
if (automaticFixes.length > 0) {
console.log('');
console.log(colors.muted('Preview of automatic fixes:'));
const preview = applyFixes(content, automaticFixes);
const diff = generateDiff(content, preview);
console.log(diff);
}
continue;
}
if (automaticFixes.length > 0) {
let shouldApply = this.yes;
if (isInteractive && !this.yes) {
console.log('');
const confirmResult = await confirm({
message: `Apply ${automaticFixes.length} automatic fix(es)?`,
initialValue: true,
});
shouldApply = confirmResult === true;
}
if (shouldApply) {
const fixed = applyFixes(content, automaticFixes);
writeFileSync(filePath, fixed, 'utf-8');
totalFixes += automaticFixes.length;
filesFixed++;
console.log(colors.success(`${symbols.success} Applied ${automaticFixes.length} fix(es)`));
} else {
console.log(colors.muted('Skipped'));
}
}
if (manualFixes.length > 0) {
console.log('');
console.log(colors.muted('Manual fixes required:'));
for (const fix of manualFixes) {
if (fix.original) {
console.log(` ${colors.warning(symbols.bullet)} Replace "${fix.original}" with "${fix.replacement}"`);
} else {
console.log(` ${colors.warning(symbols.bullet)} ${fix.description}`);
}
}
}
}
console.log('');
if (totalFixes > 0) {
success(`Fixed ${totalFixes} issue(s) in ${filesFixed} file(s)`);
console.log(colors.muted('Run "skillkit validate" to verify improvements'));
} else if (!this.dryRun) {
console.log(colors.muted('No fixes applied'));
}
return 0;
}
}
function generateDiff(original: string, modified: string): string {
const originalLines = original.split('\n');
const modifiedLines = modified.split('\n');
const originalSet = new Set(originalLines);
const modifiedSet = new Set(modifiedLines);
const added = modifiedLines.filter(line => !originalSet.has(line));
const removed = originalLines.filter(line => !modifiedSet.has(line));
const diff: string[] = [];
for (const line of removed.slice(0, 5)) {
diff.push(colors.error(`- ${line}`));
}
if (removed.length > 5) {
diff.push(colors.muted(` ... and ${removed.length - 5} more removed`));
}
for (const line of added.slice(0, 10)) {
diff.push(colors.success(`+ ${line}`));
}
if (added.length > 10) {
diff.push(colors.muted(` ... and ${added.length - 10} more added`));
}
if (diff.length === 0) {
diff.push(colors.muted(' (no visible changes)'));
}
return diff.join('\n');
}
+1
View File
@@ -8,6 +8,7 @@ export { RemoveCommand } from './remove.js';
export { InstallCommand } from './install.js';
export { UpdateCommand } from './update.js';
export { ValidateCommand } from './validate.js';
export { FixCommand } from './fix.js';
export { CreateCommand } from './create.js';
export { UICommand } from './ui.js';
export { TranslateCommand } from './translate.js';
+31 -2
View File
@@ -1,7 +1,7 @@
import { existsSync, mkdirSync, cpSync, rmSync, symlinkSync } from 'node:fs';
import { join } from 'node:path';
import { Command, Option } from 'clipanion';
import { detectProvider, isLocalPath, getProvider } from '@skillkit/core';
import { detectProvider, isLocalPath, getProvider, evaluateSkillDirectory } from '@skillkit/core';
import type { SkillMetadata, GitProvider, AgentType } from '@skillkit/core';
import { isPathInside } from '@skillkit/core';
import { getAdapter, detectAgent, getAllAdapters } from '@skillkit/agents';
@@ -28,6 +28,8 @@ import {
showNextSteps,
saveLastAgents,
getLastAgents,
formatQualityBadge,
getQualityGradeFromScore,
type InstallResult,
} from '../onboarding/index.js';
@@ -135,7 +137,9 @@ export class InstallCommand extends Command {
console.log(colors.bold('Available skills:'));
console.log('');
for (const skill of discoveredSkills) {
console.log(` ${colors.success(symbols.stepActive)} ${colors.primary(skill.name)}`);
const quality = evaluateSkillDirectory(skill.path);
const qualityBadge = quality ? ` ${formatQualityBadge(quality.overall)}` : '';
console.log(` ${colors.success(symbols.stepActive)} ${colors.primary(skill.name)}${qualityBadge}`);
}
console.log('');
console.log(colors.muted(`Total: ${discoveredSkills.length} skill(s)`));
@@ -237,9 +241,34 @@ export class InstallCommand extends Command {
installMethod = methodResult as 'symlink' | 'copy';
}
// Check for low-quality skills and warn
const lowQualitySkills: Array<{ name: string; score: number; warnings: string[] }> = [];
for (const skill of skillsToInstall) {
const quality = evaluateSkillDirectory(skill.path);
if (quality && quality.overall < 60) {
lowQualitySkills.push({
name: skill.name,
score: quality.overall,
warnings: quality.warnings.slice(0, 2),
});
}
}
// Confirm installation
if (isInteractive && !this.yes) {
console.log('');
// Show low-quality warning if any
if (lowQualitySkills.length > 0) {
console.log(colors.warning(`${symbols.warning} Warning: ${lowQualitySkills.length} skill(s) have low quality scores (< 60)`));
for (const lq of lowQualitySkills) {
const grade = getQualityGradeFromScore(lq.score);
const warningText = lq.warnings.length > 0 ? ` - ${lq.warnings.join(', ')}` : '';
console.log(colors.muted(` - ${lq.name} [${grade}]${warningText}`));
}
console.log('');
}
const agentDisplay = targetAgents.length <= 3
? targetAgents.map(formatAgent).join(', ')
: `${targetAgents.slice(0, 2).map(formatAgent).join(', ')} +${targetAgents.length - 2} more`;
+35 -8
View File
@@ -1,7 +1,8 @@
import chalk from 'chalk';
import { Command, Option } from 'clipanion';
import { findAllSkills } from '@skillkit/core';
import { findAllSkills, evaluateSkillDirectory } from '@skillkit/core';
import { getSearchDirs } from '../helpers.js';
import { formatQualityBadge, colors } from '../onboarding/index.js';
export class ListCommand extends Command {
static override paths = [['list'], ['ls'], ['l']];
@@ -23,6 +24,10 @@ export class ListCommand extends Command {
description: 'Show only disabled skills',
});
quality = Option.Boolean('--quality,-q', false, {
description: 'Show quality scores and grades',
});
json = Option.Boolean('--json,-j', false, {
description: 'Output as JSON',
});
@@ -44,8 +49,18 @@ export class ListCommand extends Command {
return a.name.localeCompare(b.name);
});
const skillsWithQuality = this.quality || this.json
? skills.map(s => {
const quality = evaluateSkillDirectory(s.path);
return {
...s,
quality: quality?.overall ?? null,
};
})
: skills.map(s => ({ ...s, quality: null as number | null }));
if (this.json) {
console.log(JSON.stringify(skills, null, 2));
console.log(JSON.stringify(skillsWithQuality, null, 2));
return 0;
}
@@ -57,13 +72,13 @@ export class ListCommand extends Command {
console.log(chalk.cyan(`Installed skills (${skills.length}):\n`));
const projectSkills = skills.filter(s => s.location === 'project');
const globalSkills = skills.filter(s => s.location === 'global');
const projectSkills = skillsWithQuality.filter(s => s.location === 'project');
const globalSkills = skillsWithQuality.filter(s => s.location === 'global');
if (projectSkills.length > 0) {
console.log(chalk.blue('Project skills:'));
for (const skill of projectSkills) {
printSkill(skill);
printSkill(skill, this.quality);
}
console.log();
}
@@ -71,7 +86,7 @@ export class ListCommand extends Command {
if (globalSkills.length > 0) {
console.log(chalk.dim('Global skills:'));
for (const skill of globalSkills) {
printSkill(skill);
printSkill(skill, this.quality);
}
console.log();
}
@@ -86,16 +101,28 @@ export class ListCommand extends Command {
)
);
if (this.quality) {
const qualityScores = skillsWithQuality.filter(s => s.quality !== null).map(s => s.quality!);
if (qualityScores.length > 0) {
const avgQuality = Math.round(qualityScores.reduce((a, b) => a + b, 0) / qualityScores.length);
console.log(colors.muted(`Average quality: ${avgQuality}/100`));
}
}
return 0;
}
}
function printSkill(skill: { name: string; description: string; enabled: boolean; location: string }) {
function printSkill(
skill: { name: string; description: string; enabled: boolean; location: string; quality: number | null },
showQuality = false
) {
const status = skill.enabled ? chalk.green('✓') : chalk.red('○');
const name = skill.enabled ? skill.name : chalk.dim(skill.name);
const desc = chalk.dim(truncate(skill.description, 50));
const qualityBadge = showQuality && skill.quality !== null ? ` ${formatQualityBadge(skill.quality)}` : '';
console.log(` ${status} ${name}`);
console.log(` ${status} ${name}${qualityBadge}`);
if (skill.description) {
console.log(` ${desc}`);
}
+11 -1
View File
@@ -20,6 +20,8 @@ import {
warn,
showProjectSummary,
progressBar,
formatQualityBadge,
getQualityGradeFromScore,
} from '../onboarding/index.js';
export class RecommendCommand extends Command {
@@ -236,8 +238,12 @@ export class RecommendCommand extends Command {
scoreColor = colors.muted;
}
const scoreBar = progressBar(rec.score, 100, 10);
const qualityScore = rec.skill.quality ?? null;
const qualityDisplay = qualityScore !== null && qualityScore !== undefined
? ` ${formatQualityBadge(qualityScore)}`
: '';
console.log(` ${scoreColor(`${rec.score}%`)} ${colors.dim(scoreBar)} ${colors.bold(rec.skill.name)}`);
console.log(` ${scoreColor(`${rec.score}%`)} ${colors.dim(scoreBar)} ${colors.bold(rec.skill.name)}${qualityDisplay}`);
if (rec.skill.description) {
console.log(` ${colors.muted(truncate(rec.skill.description, 70))}`);
@@ -252,6 +258,10 @@ export class RecommendCommand extends Command {
for (const reason of rec.reasons.filter(r => r.weight > 0)) {
console.log(` ${colors.muted(symbols.stepActive)} ${reason.description} (+${reason.weight})`);
}
if (qualityScore !== null && qualityScore !== undefined) {
const grade = getQualityGradeFromScore(qualityScore);
console.log(` ${colors.muted(symbols.stepActive)} Quality: ${qualityScore}/100 (${grade})`);
}
}
if (rec.warnings.length > 0) {
+108 -16
View File
@@ -1,5 +1,5 @@
import { existsSync, readdirSync, statSync } from 'node:fs';
import { join, basename, resolve } from 'node:path';
import { join, basename, resolve, dirname } from 'node:path';
import { Command, Option } from 'clipanion';
import {
validateSkill,
@@ -8,7 +8,9 @@ import {
getQualityGrade,
isHighQuality,
findAllSkills,
benchmarkSkill,
type QualityScore,
type BenchmarkResult,
} from '@skillkit/core';
import { getSearchDirs } from '../helpers.js';
import {
@@ -71,6 +73,29 @@ function printQualityReport(name: string, quality: QualityScore, verbose: boolea
if (quality.specificity.vagueTermCount > 0) {
console.log(` ${colors.warning(symbols.warning)} ${quality.specificity.vagueTermCount} vague term(s)`);
}
console.log('');
console.log(` ${colors.muted('Advanced:')} ${formatScore(quality.advanced.score)}/100`);
if (quality.advanced.deprecatedPatterns.length > 0) {
console.log(` ${colors.warning(symbols.warning)} ${quality.advanced.deprecatedPatterns.length} deprecated pattern(s)`);
for (const pattern of quality.advanced.deprecatedPatterns) {
console.log(` ${colors.muted(symbols.bullet)} ${pattern}`);
}
}
if (quality.advanced.securityIssues.length > 0) {
console.log(` ${colors.error(symbols.error)} ${quality.advanced.securityIssues.length} security issue(s)`);
for (const issue of quality.advanced.securityIssues) {
console.log(` ${colors.muted(symbols.bullet)} ${issue}`);
}
}
if (quality.advanced.conflictingInstructions.length > 0) {
console.log(` ${colors.warning(symbols.warning)} ${quality.advanced.conflictingInstructions.length} conflicting instruction(s)`);
}
console.log(` Completeness: ${quality.advanced.completeness.score}/100`);
console.log(` Example coverage: ${quality.advanced.completeness.exampleCoverage}%`);
if (quality.advanced.completeness.hasTodos) {
console.log(` ${colors.warning(symbols.warning)} ${quality.advanced.completeness.todoCount} TODO/FIXME comment(s)`);
}
}
if (quality.warnings.length > 0) {
@@ -90,6 +115,39 @@ function printQualityReport(name: string, quality: QualityScore, verbose: boolea
}
}
function printBenchmarkReport(benchmark: BenchmarkResult): void {
console.log('');
console.log(colors.bold('Benchmark Comparison'));
console.log('');
console.log(` ${colors.muted('Your Score:')} ${formatScore(benchmark.score)}/100 ${formatGrade(benchmark.grade)}`);
console.log(` ${colors.muted('Category Average:')} ${benchmark.categoryAvg}/100`);
console.log(` ${colors.muted('Top Skill Score:')} ${benchmark.topSkillScore}/100`);
console.log(` ${colors.muted('Percentile:')} ${benchmark.percentile}${getOrdinal(benchmark.percentile)}`);
if (benchmark.comparisonNotes.length > 0) {
console.log('');
console.log(` ${colors.muted('Comparison:')}`);
for (const note of benchmark.comparisonNotes) {
console.log(` ${colors.muted(symbols.bullet)} ${note}`);
}
}
if (benchmark.recommendations.length > 0) {
console.log('');
console.log(` ${colors.muted('To Improve:')}`);
for (const rec of benchmark.recommendations) {
console.log(` ${colors.info(symbols.info)} ${rec}`);
}
}
}
function getOrdinal(n: number): string {
const s = ['th', 'st', 'nd', 'rd'];
const v = n % 100;
return s[(v - 20) % 10] || s[v] || s[0];
}
export class ValidateCommand extends Command {
static override paths = [['validate']];
@@ -136,7 +194,28 @@ export class ValidateCommand extends Command {
description: 'Validate all skills in directory',
});
benchmark = Option.Boolean('--benchmark,-b', false, {
description: 'Compare skill against category leaders',
});
fix = Option.Boolean('--fix', false, {
description: 'Automatically fix common quality issues (runs fix command)',
});
async execute(): Promise<number> {
if (this.fix) {
if (this.targets.length === 0) {
error('--fix requires at least one target');
console.log(colors.muted('Usage: skillkit validate --fix <skill-name>'));
return 1;
}
const { FixCommand } = await import('./fix.js');
const fixCmd = new FixCommand();
fixCmd.targets = this.targets;
fixCmd.yes = false;
return fixCmd.execute();
}
let minScore = 60;
if (this.minScore) {
const parsed = parseInt(this.minScore, 10);
@@ -185,7 +264,8 @@ export class ValidateCommand extends Command {
}
} else {
const isFile = resolved.endsWith('.md') || resolved.endsWith('.mdc');
const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(resolved);
const skillDir = isFile ? dirname(resolved) : resolved;
const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(skillDir);
const quality = isFile ? evaluateSkillFile(resolved) : evaluateSkillDirectory(resolved);
results.push({
name: basename(target),
@@ -258,20 +338,27 @@ export class ValidateCommand extends Command {
}
if (this.json) {
const output = results.map(r => ({
name: r.name,
path: r.path,
formatValid: r.formatValid,
formatErrors: r.formatErrors,
score: r.quality?.overall ?? null,
grade: r.quality ? getQualityGrade(r.quality.overall) : null,
highQuality: r.quality ? isHighQuality(r.quality) : false,
structure: r.quality?.structure ?? null,
clarity: r.quality?.clarity ?? null,
specificity: r.quality?.specificity ?? null,
warnings: r.quality?.warnings ?? [],
suggestions: r.quality?.suggestions ?? [],
}));
const output = results.map(r => {
const result: Record<string, unknown> = {
name: r.name,
path: r.path,
formatValid: r.formatValid,
formatErrors: r.formatErrors,
score: r.quality?.overall ?? null,
grade: r.quality ? getQualityGrade(r.quality.overall) : null,
highQuality: r.quality ? isHighQuality(r.quality) : false,
structure: r.quality?.structure ?? null,
clarity: r.quality?.clarity ?? null,
specificity: r.quality?.specificity ?? null,
advanced: r.quality?.advanced ?? null,
warnings: r.quality?.warnings ?? [],
suggestions: r.quality?.suggestions ?? [],
};
if (this.benchmark && r.quality) {
result.benchmark = benchmarkSkill(r.name, r.quality);
}
return result;
});
console.log(JSON.stringify(output, null, 2));
const hasFormatFailures = results.some(r => !r.formatValid);
const hasBelowThreshold = results.some(r => r.quality && r.quality.overall < minScore);
@@ -302,6 +389,11 @@ export class ValidateCommand extends Command {
if (quality) {
printQualityReport(name, quality, this.verbose);
if (this.benchmark) {
const benchmark = benchmarkSkill(name, quality);
printBenchmarkReport(benchmark);
}
}
}
+3
View File
@@ -9,6 +9,9 @@ export {
formatAgentIconsInline,
progressBar,
formatScore,
formatQualityBadge,
formatQualityBadgeCompact,
getQualityGradeFromScore,
} from './theme.js';
export {
+22
View File
@@ -132,3 +132,25 @@ export function formatScore(score: number): string {
}
return `${scoreColor(`${score}%`)} ${colors.dim(bar)}`;
}
export function getQualityGradeFromScore(score: number): string {
if (score >= 90) return 'A';
if (score >= 80) return 'B';
if (score >= 70) return 'C';
if (score >= 60) return 'D';
return 'F';
}
export function formatQualityBadge(score: number): string {
const grade = getQualityGradeFromScore(score);
if (score >= 80) return colors.success(`[${grade}]`);
if (score >= 60) return colors.warning(`[${grade}]`);
return colors.error(`[${grade}]`);
}
export function formatQualityBadgeCompact(score: number): string {
const grade = getQualityGradeFromScore(score);
if (score >= 80) return colors.success(grade);
if (score >= 60) return colors.warning(grade);
return colors.error(grade);
}
@@ -164,12 +164,21 @@ Use this skill when:
});
describe('isHighQuality', () => {
const defaultAdvanced = {
score: 100,
deprecatedPatterns: [],
conflictingInstructions: [],
securityIssues: [],
completeness: { score: 100, hasTodos: false, todoCount: 0, emptySections: [], exampleCoverage: 100 },
};
it('should return true for high scoring skills with few warnings', () => {
const score = {
overall: 80,
structure: { score: 80, hasMetadata: true, hasDescription: true, hasTriggers: true, hasExamples: true, hasBoundaries: true, hasWhenToUse: true },
clarity: { score: 80, lineCount: 100, tokenCount: 500, avgSentenceLength: 15, hasHeaders: true },
specificity: { score: 80, hasConcreteCommands: true, hasFilePatterns: true, hasCodeExamples: true, vagueTermCount: 0 },
advanced: defaultAdvanced,
warnings: [],
suggestions: [],
};
@@ -182,6 +191,7 @@ Use this skill when:
structure: { score: 50, hasMetadata: false, hasDescription: false, hasTriggers: false, hasExamples: false, hasBoundaries: false, hasWhenToUse: false },
clarity: { score: 50, lineCount: 100, tokenCount: 500, avgSentenceLength: 15, hasHeaders: true },
specificity: { score: 50, hasConcreteCommands: false, hasFilePatterns: false, hasCodeExamples: false, vagueTermCount: 5 },
advanced: { ...defaultAdvanced, score: 50 },
warnings: ['Missing triggers', 'No examples'],
suggestions: [],
};
@@ -194,6 +204,7 @@ Use this skill when:
structure: { score: 75, hasMetadata: true, hasDescription: true, hasTriggers: false, hasExamples: false, hasBoundaries: false, hasWhenToUse: false },
clarity: { score: 75, lineCount: 100, tokenCount: 500, avgSentenceLength: 15, hasHeaders: true },
specificity: { score: 75, hasConcreteCommands: true, hasFilePatterns: true, hasCodeExamples: false, vagueTermCount: 0 },
advanced: { ...defaultAdvanced, score: 75 },
warnings: ['Warning 1', 'Warning 2', 'Warning 3'],
suggestions: [],
};
+262
View File
@@ -0,0 +1,262 @@
/**
* Benchmark quality scores against category leaders
*/
import type { QualityScore } from './index.js';
export interface BenchmarkResult {
skill: string;
score: number;
grade: string;
categoryAvg: number;
topSkillScore: number;
percentile: number;
comparisonNotes: string[];
recommendations: string[];
}
export interface CategoryStats {
category: string;
avgScore: number;
topScore: number;
topSkills: string[];
count: number;
distribution: {
A: number;
B: number;
C: number;
D: number;
F: number;
};
}
const CATEGORY_BENCHMARKS: Record<string, CategoryStats> = {
react: {
category: 'react',
avgScore: 78,
topScore: 95,
topSkills: ['react-patterns', 'react-hooks-best-practices'],
count: 150,
distribution: { A: 15, B: 35, C: 30, D: 15, F: 5 },
},
typescript: {
category: 'typescript',
avgScore: 75,
topScore: 92,
topSkills: ['typescript-strict', 'type-safety-patterns'],
count: 120,
distribution: { A: 12, B: 30, C: 35, D: 18, F: 5 },
},
testing: {
category: 'testing',
avgScore: 72,
topScore: 90,
topSkills: ['tdd-workflow', 'testing-best-practices'],
count: 80,
distribution: { A: 10, B: 28, C: 38, D: 19, F: 5 },
},
git: {
category: 'git',
avgScore: 80,
topScore: 94,
topSkills: ['git-workflow', 'conventional-commits'],
count: 60,
distribution: { A: 18, B: 40, C: 28, D: 12, F: 2 },
},
security: {
category: 'security',
avgScore: 76,
topScore: 93,
topSkills: ['security-review', 'owasp-patterns'],
count: 45,
distribution: { A: 14, B: 32, C: 34, D: 16, F: 4 },
},
api: {
category: 'api',
avgScore: 74,
topScore: 91,
topSkills: ['api-design', 'rest-best-practices'],
count: 55,
distribution: { A: 11, B: 30, C: 36, D: 18, F: 5 },
},
general: {
category: 'general',
avgScore: 70,
topScore: 88,
topSkills: ['code-review', 'best-practices'],
count: 200,
distribution: { A: 8, B: 25, C: 40, D: 22, F: 5 },
},
};
export function getGrade(score: number): string {
if (score >= 90) return 'A';
if (score >= 80) return 'B';
if (score >= 70) return 'C';
if (score >= 60) return 'D';
return 'F';
}
export function detectCategory(skillName: string, tags: string[] = []): string {
const lowerName = skillName.toLowerCase();
const lowerTags = tags.map(t => t.toLowerCase());
const searchTerms = [lowerName, ...lowerTags];
const categoryKeywords: Record<string, string[]> = {
react: ['react', 'hooks', 'jsx', 'tsx', 'component', 'nextjs', 'remix'],
typescript: ['typescript', 'type', 'types', 'ts', 'strict'],
testing: ['test', 'testing', 'tdd', 'jest', 'vitest', 'playwright', 'e2e'],
git: ['git', 'commit', 'branch', 'merge', 'workflow', 'conventional'],
security: ['security', 'auth', 'owasp', 'vulnerability', 'safe'],
api: ['api', 'rest', 'graphql', 'endpoint', 'http'],
};
for (const [category, keywords] of Object.entries(categoryKeywords)) {
for (const term of searchTerms) {
if (keywords.some(kw => term.includes(kw))) {
return category;
}
}
}
return 'general';
}
export function calculatePercentile(score: number, categoryStats: CategoryStats): number {
const { distribution } = categoryStats;
const totalSkills = distribution.A + distribution.B + distribution.C + distribution.D + distribution.F;
let percentBelow = 0;
if (score < 60) {
percentBelow = (score / 60) * (distribution.F / totalSkills) * 100;
} else if (score < 70) {
percentBelow = (distribution.F / totalSkills) * 100;
percentBelow += ((score - 60) / 10) * (distribution.D / totalSkills) * 100;
} else if (score < 80) {
percentBelow = ((distribution.F + distribution.D) / totalSkills) * 100;
percentBelow += ((score - 70) / 10) * (distribution.C / totalSkills) * 100;
} else if (score < 90) {
percentBelow = ((distribution.F + distribution.D + distribution.C) / totalSkills) * 100;
percentBelow += ((score - 80) / 10) * (distribution.B / totalSkills) * 100;
} else {
percentBelow = ((distribution.F + distribution.D + distribution.C + distribution.B) / totalSkills) * 100;
percentBelow += ((score - 90) / 10) * (distribution.A / totalSkills) * 100;
}
return Math.round(Math.min(99, Math.max(1, percentBelow)));
}
export function generateComparisonNotes(
quality: QualityScore,
categoryStats: CategoryStats
): string[] {
const notes: string[] = [];
const topScore = categoryStats.topScore;
const avgScore = categoryStats.avgScore;
if (quality.overall >= topScore) {
notes.push('Your skill is among the top performers in this category');
} else if (quality.overall >= avgScore) {
notes.push(`Your skill is above the category average (${avgScore})`);
} else {
notes.push(`Your skill is ${avgScore - quality.overall} points below the category average`);
}
if (!quality.structure.hasExamples) {
notes.push('Missing code examples (present in 95% of top skills)');
}
if (!quality.structure.hasTriggers) {
notes.push('Missing trigger conditions (present in 80% of top skills)');
}
if (!quality.structure.hasBoundaries) {
notes.push('Missing "When Not to Use" section (present in 80% of top skills)');
}
if (quality.specificity.vagueTermCount > 3) {
notes.push('Trigger conditions are less specific than category average');
}
if (quality.clarity.lineCount > 300) {
notes.push('Skill is longer than recommended (top skills average 150-250 lines)');
}
if (quality.advanced.deprecatedPatterns.length > 0) {
notes.push('Contains deprecated patterns not found in top skills');
}
return notes;
}
export function generateRecommendations(
quality: QualityScore,
categoryStats: CategoryStats
): string[] {
const recommendations: string[] = [];
const gap = categoryStats.topScore - quality.overall;
if (gap > 20) {
recommendations.push('Consider studying top skills in this category for patterns');
}
if (!quality.structure.hasExamples) {
recommendations.push(`Add ${quality.specificity.hasCodeExamples ? 'more' : ''} code examples showing input/output`);
}
if (!quality.structure.hasTriggers) {
recommendations.push('Add explicit trigger conditions with patterns like "Triggers when:"');
}
if (!quality.structure.hasBoundaries) {
recommendations.push('Add a "Boundaries" or "Limitations" section');
}
if (quality.clarity.lineCount > 400) {
recommendations.push('Split into multiple focused skills (top skills are 150-250 lines)');
}
if (quality.specificity.vagueTermCount > 0) {
recommendations.push(`Replace ${quality.specificity.vagueTermCount} vague term(s) with specific instructions`);
}
if (quality.advanced.completeness.hasTodos) {
recommendations.push('Complete TODO items before publishing');
}
return recommendations;
}
export function benchmarkSkill(
skillName: string,
quality: QualityScore,
tags: string[] = [],
customCategoryStats?: CategoryStats
): BenchmarkResult {
const category = detectCategory(skillName, tags);
const categoryStats = customCategoryStats || CATEGORY_BENCHMARKS[category] || CATEGORY_BENCHMARKS.general;
const percentile = calculatePercentile(quality.overall, categoryStats);
const comparisonNotes = generateComparisonNotes(quality, categoryStats);
const recommendations = generateRecommendations(quality, categoryStats);
return {
skill: skillName,
score: quality.overall,
grade: getGrade(quality.overall),
categoryAvg: categoryStats.avgScore,
topSkillScore: categoryStats.topScore,
percentile,
comparisonNotes,
recommendations,
};
}
export function getCategoryStats(category: string): CategoryStats | null {
return CATEGORY_BENCHMARKS[category] || null;
}
export function getAllCategories(): string[] {
return Object.keys(CATEGORY_BENCHMARKS);
}
+223 -7
View File
@@ -1,15 +1,34 @@
import { readFileSync, existsSync } from 'node:fs';
import { join, basename } from 'node:path';
export * from './benchmark.js';
export interface QualityScore {
overall: number;
structure: StructureScore;
clarity: ClarityScore;
specificity: SpecificityScore;
advanced: AdvancedScore;
warnings: string[];
suggestions: string[];
}
export interface AdvancedScore {
score: number;
deprecatedPatterns: string[];
conflictingInstructions: string[];
securityIssues: string[];
completeness: CompletenessResult;
}
export interface CompletenessResult {
score: number;
hasTodos: boolean;
todoCount: number;
emptySections: string[];
exampleCoverage: number;
}
export interface StructureScore {
score: number;
hasMetadata: boolean;
@@ -93,6 +112,27 @@ const FILE_PATTERN_PATTERNS = [
/pattern[s]?\s*[:=]/i,
];
const DEPRECATED_PATTERNS = [
{ pattern: /require\s*\(['"][^'"]+['"]\)/g, message: 'Uses CommonJS require() instead of ES modules' },
{ pattern: /React\.Component/g, message: 'Uses class components instead of functional' },
{ pattern: /componentDidMount|componentWillUnmount|componentDidUpdate/g, message: 'Uses lifecycle methods instead of hooks' },
{ pattern: /\bvar\s+\w+\s*=/g, message: 'Uses var instead of const/let' },
{ pattern: /\.then\s*\([^)]*\)\.catch/g, message: 'Uses .then().catch() instead of async/await' },
{ pattern: /new\s+Promise\s*\(/g, message: 'Consider using async/await instead of new Promise()' },
];
const SECURITY_PATTERNS = [
{ pattern: /password\s*[:=]\s*['"][^'"]+['"]/gi, message: 'Potential hardcoded password' },
{ pattern: /api[_-]?key\s*[:=]\s*['"][^'"]+['"]/gi, message: 'Potential hardcoded API key' },
{ pattern: /secret\s*[:=]\s*['"][^'"]+['"]/gi, message: 'Potential hardcoded secret' },
{ pattern: /token\s*[:=]\s*['"][A-Za-z0-9_-]{20,}['"]/gi, message: 'Potential hardcoded token' },
{ pattern: /\$\{[^}]*\}/g, message: 'Template literal - ensure proper sanitization in shell commands' },
{ pattern: /eval\s*\(/g, message: 'Uses eval() - potential code injection risk' },
{ pattern: /innerHTML\s*=/g, message: 'Uses innerHTML - potential XSS risk' },
{ pattern: /dangerouslySetInnerHTML/g, message: 'Uses dangerouslySetInnerHTML - potential XSS risk' },
];
function extractFrontmatter(content: string): Record<string, unknown> | null {
const normalizedContent = content.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
const match = normalizedContent.match(/^---\s*\n([\s\S]*?)\n---/);
@@ -213,6 +253,148 @@ function countCodeBlocks(content: string): number {
return fencedBlocks.length;
}
function detectDeprecatedPatterns(content: string): string[] {
const codeBlocks = content.match(/```[\s\S]*?```/g) || [];
const codeContent = codeBlocks.join('\n');
const issues: string[] = [];
for (const { pattern, message } of DEPRECATED_PATTERNS) {
const regex = new RegExp(pattern.source, pattern.flags);
if (regex.test(codeContent)) {
issues.push(message);
}
}
return [...new Set(issues)];
}
function detectSecurityPatterns(content: string): string[] {
const issues: string[] = [];
for (const { pattern, message } of SECURITY_PATTERNS) {
const regex = new RegExp(pattern.source, pattern.flags);
if (regex.test(content)) {
if (!message.includes('Template literal') || content.includes('bash') || content.includes('shell')) {
issues.push(message);
}
}
}
return [...new Set(issues)];
}
function detectConflictingInstructions(content: string): string[] {
const issues: string[] = [];
const lowerContent = content.toLowerCase();
const alwaysMatches = lowerContent.match(/always\s+(?:use|do|include|add)\s+(\w+)/gi) || [];
const neverMatches = lowerContent.match(/never\s+(?:use|do|include|add)\s+(\w+)/gi) || [];
for (const always of alwaysMatches) {
const term = always.replace(/always\s+(?:use|do|include|add)\s+/i, '').toLowerCase();
for (const never of neverMatches) {
const neverTerm = never.replace(/never\s+(?:use|do|include|add)\s+/i, '').toLowerCase();
if (term === neverTerm) {
issues.push(`Conflicting instructions: "always" and "never" used with "${term}"`);
}
}
}
const mustMatches = lowerContent.match(/must\s+(\w+)/gi) || [];
const mustNotMatches = lowerContent.match(/must\s+not\s+(\w+)/gi) || [];
for (const must of mustMatches) {
const term = must.replace(/must\s+/i, '').toLowerCase();
for (const mustNot of mustNotMatches) {
const notTerm = mustNot.replace(/must\s+not\s+/i, '').toLowerCase();
if (term === notTerm) {
issues.push(`Conflicting instructions: "must" and "must not" used with "${term}"`);
}
}
}
return [...new Set(issues)];
}
function assessCompleteness(content: string): CompletenessResult {
const todoMatches = content.match(/TODO|FIXME|XXX|HACK/gi) || [];
const todoCount = todoMatches.length;
const headerPositions: Array<{ header: string; index: number }> = [];
const lines = content.split('\n');
let inFence = false;
let offset = 0;
for (const line of lines) {
if (/^```|^~~~/.test(line)) {
inFence = !inFence;
} else if (!inFence && /^#+\s+.+$/.test(line)) {
headerPositions.push({ header: line, index: offset });
}
offset += line.length + 1;
}
const emptySections: string[] = [];
for (let i = 0; i < headerPositions.length - 1; i++) {
const current = headerPositions[i];
const next = headerPositions[i + 1];
const sectionContent = content.slice(current.index + current.header.length, next.index).trim();
if (sectionContent.length < 20) {
const sectionName = current.header.replace(/^#+\s+/, '');
emptySections.push(sectionName);
}
}
if (headerPositions.length > 0) {
const last = headerPositions[headerPositions.length - 1];
const tailContent = content.slice(last.index + last.header.length).trim();
if (tailContent.length < 20) {
const sectionName = last.header.replace(/^#+\s+/, '');
emptySections.push(sectionName);
}
}
const codeBlocks = countCodeBlocks(content);
const exampleCoverage = Math.min(100, codeBlocks * 25);
let score = 100;
if (todoCount > 0) score -= Math.min(30, todoCount * 10);
if (emptySections.length > 0) score -= Math.min(30, emptySections.length * 10);
if (codeBlocks < 2) score -= 20;
else if (codeBlocks < 4) score -= 10;
return {
score: Math.max(0, score),
hasTodos: todoCount > 0,
todoCount,
emptySections,
exampleCoverage,
};
}
function evaluateAdvanced(content: string): AdvancedScore {
const deprecatedPatterns = detectDeprecatedPatterns(content);
const conflictingInstructions = detectConflictingInstructions(content);
const securityIssues = detectSecurityPatterns(content);
const completeness = assessCompleteness(content);
let score = 100;
score -= Math.min(30, deprecatedPatterns.length * 10);
score -= Math.min(30, conflictingInstructions.length * 15);
score -= Math.min(25, securityIssues.length * 10);
score -= Math.round((100 - completeness.score) * 0.15);
return {
score: Math.max(0, score),
deprecatedPatterns,
conflictingInstructions,
securityIssues,
completeness,
};
}
function evaluateSpecificity(content: string): SpecificityScore {
const hasConcreteCommands = countMatches(content, COMMAND_PATTERNS) > 0;
const hasFilePatterns = hasPattern(content, FILE_PATTERN_PATTERNS);
@@ -242,7 +424,8 @@ function evaluateSpecificity(content: string): SpecificityScore {
function generateWarnings(
structure: StructureScore,
clarity: ClarityScore,
specificity: SpecificityScore
specificity: SpecificityScore,
advanced: AdvancedScore
): string[] {
const warnings: string[] = [];
@@ -270,13 +453,27 @@ function generateWarnings(
warnings.push('Only one code example provided');
}
for (const issue of advanced.securityIssues) {
warnings.push(`Security: ${issue}`);
}
for (const conflict of advanced.conflictingInstructions) {
warnings.push(conflict);
}
if (advanced.completeness.hasTodos) {
warnings.push(`Contains ${advanced.completeness.todoCount} TODO/FIXME comment(s)`);
}
if (advanced.completeness.emptySections.length > 0) {
warnings.push(`Empty sections: ${advanced.completeness.emptySections.join(', ')}`);
}
return warnings;
}
function generateSuggestions(
structure: StructureScore,
clarity: ClarityScore,
specificity: SpecificityScore
specificity: SpecificityScore,
advanced: AdvancedScore
): string[] {
const suggestions: string[] = [];
@@ -305,6 +502,22 @@ function generateSuggestions(
suggestions.push('Add concrete executable commands with flags');
}
if (advanced.deprecatedPatterns.length > 0) {
suggestions.push('Update code examples to use modern patterns (ES modules, hooks, async/await)');
}
if (advanced.securityIssues.length > 0) {
suggestions.push('Review and remove potential security risks from examples');
}
if (advanced.completeness.hasTodos) {
suggestions.push('Complete or remove TODO/FIXME comments');
}
if (advanced.completeness.emptySections.length > 0) {
suggestions.push('Add content to empty sections or remove them');
}
if (advanced.completeness.exampleCoverage < 50) {
suggestions.push('Add more code examples to improve coverage');
}
return suggestions;
}
@@ -312,21 +525,24 @@ export function evaluateSkillContent(content: string): QualityScore {
const structure = evaluateStructure(content);
const clarity = evaluateClarity(content);
const specificity = evaluateSpecificity(content);
const advanced = evaluateAdvanced(content);
const overall = Math.round(
structure.score * 0.4 +
clarity.score * 0.3 +
specificity.score * 0.3
structure.score * 0.35 +
clarity.score * 0.25 +
specificity.score * 0.25 +
advanced.score * 0.15
);
const warnings = generateWarnings(structure, clarity, specificity);
const suggestions = generateSuggestions(structure, clarity, specificity);
const warnings = generateWarnings(structure, clarity, specificity, advanced);
const suggestions = generateSuggestions(structure, clarity, specificity, advanced);
return {
overall,
structure,
clarity,
specificity,
advanced,
warnings,
suggestions,
};
+7 -3
View File
@@ -188,10 +188,14 @@ export function Installed({ onNavigate, cols = 80, rows = 24 }: InstalledProps)
const selected = actualIdx === selectedIndex;
const indicator = selected ? '▸' : ' ';
const statusIcon = skill.enabled !== false ? '●' : '○';
// Use single text element to avoid rendering overlap
const line = `${indicator}${statusIcon} ${skill.name}`;
const qualityBadge = skill.grade ? ` [${skill.grade}]` : '';
const warningIndicator = skill.warnings && skill.warnings > 2 ? ' ⚠' : '';
const line = `${indicator}${statusIcon} ${skill.name}${qualityBadge}${warningIndicator}`;
const badgeColor = skill.quality !== undefined
? (skill.quality >= 80 ? terminalColors.success : skill.quality >= 60 ? terminalColors.warning : terminalColors.error)
: terminalColors.text;
return (
<text key={skill.name} fg={selected ? terminalColors.accent : terminalColors.text}>
<text key={skill.name} fg={selected ? terminalColors.accent : badgeColor}>
{line}
</text>
);
+9 -7
View File
@@ -50,14 +50,14 @@ export function Recommend({ onNavigate, cols = 80, rows = 24 }: RecommendProps)
return () => clearInterval(interval);
}, [analyzing]);
// Recommendations data
// Recommendations data (with quality scores)
const recommendations = useMemo(() => [
{ name: 'tdd-workflow', reason: 'Based on your test files and coverage patterns', confidence: 95 },
{ name: 'react-patterns', reason: 'Detected React in package.json with hooks usage', confidence: 88 },
{ name: 'typescript-strict', reason: 'tsconfig.json found with strict mode disabled', confidence: 82 },
{ name: 'git-workflow', reason: '.git directory with complex branching detected', confidence: 75 },
{ name: 'api-design', reason: 'REST endpoints found in your codebase', confidence: 68 },
{ name: 'docker-compose', reason: 'docker-compose.yml detected for orchestration', confidence: 62 },
{ name: 'tdd-workflow', reason: 'Based on your test files and coverage patterns', confidence: 95, quality: 92, grade: 'A' },
{ name: 'react-patterns', reason: 'Detected React in package.json with hooks usage', confidence: 88, quality: 85, grade: 'B' },
{ name: 'typescript-strict', reason: 'tsconfig.json found with strict mode disabled', confidence: 82, quality: 78, grade: 'C' },
{ name: 'git-workflow', reason: '.git directory with complex branching detected', confidence: 75, quality: 82, grade: 'B' },
{ name: 'api-design', reason: 'REST endpoints found in your codebase', confidence: 68, quality: 71, grade: 'C' },
{ name: 'docker-compose', reason: 'docker-compose.yml detected for orchestration', confidence: 62, quality: 65, grade: 'D' },
], []);
const maxVisible = Math.max(3, Math.floor((rows - 10) / 2));
@@ -163,6 +163,7 @@ export function Recommend({ onNavigate, cols = 80, rows = 24 }: RecommendProps)
{visibleRecs.map((rec, idx) => {
const selected = idx === selectedIndex;
const indicator = selected ? '▸' : ' ';
const qualityColor = rec.quality >= 80 ? terminalColors.success : rec.quality >= 60 ? terminalColors.warning : terminalColors.error;
return (
<box key={rec.name} flexDirection="column">
<box flexDirection="row">
@@ -171,6 +172,7 @@ export function Recommend({ onNavigate, cols = 80, rows = 24 }: RecommendProps)
{rec.name}
</text>
<text fg={getConfColor(rec.confidence)}> {rec.confidence}%</text>
<text fg={qualityColor}> [{rec.grade}]</text>
</box>
<text fg={terminalColors.textMuted}> {rec.reason}</text>
<text> </text>
+13 -7
View File
@@ -1,7 +1,7 @@
/**
* Skills state management for SkillKit TUI
*/
import { findAllSkills } from '@skillkit/core';
import { findAllSkills, evaluateSkillDirectory, getQualityGrade } from '@skillkit/core';
import { rmSync } from 'node:fs';
import { resolve, normalize } from 'node:path';
import { getSearchDirs } from '../utils/helpers.js';
@@ -38,12 +38,18 @@ export function loadSkills(agentType?: AgentType): SkillsState {
const searchDirs = getSearchDirs(agentType);
const foundSkills = findAllSkills(searchDirs);
const skillItems: SkillItem[] = foundSkills.map((s) => ({
name: s.name,
description: s.description,
source: s.metadata?.source,
enabled: s.enabled,
}));
const skillItems: SkillItem[] = foundSkills.map((s) => {
const quality = evaluateSkillDirectory(s.path);
return {
name: s.name,
description: s.description,
source: s.metadata?.source,
enabled: s.enabled,
quality: quality?.overall,
grade: quality ? getQualityGrade(quality.overall) : undefined,
warnings: quality?.warnings.length,
};
});
return {
skills: skillItems,
+3
View File
@@ -4,6 +4,9 @@ export interface SkillItem {
source?: string;
installs?: number;
enabled?: boolean;
quality?: number;
grade?: string;
warnings?: number;
}
export interface RepoInfo {
+149
View File
@@ -0,0 +1,149 @@
#!/usr/bin/env node
/**
* Compute quality scores for marketplace skills
*
* This script assigns default quality scores to skills based on source verification.
* For detailed quality evaluation, skills would need to be fetched and analyzed individually.
*
* Usage:
* npx tsx scripts/compute-quality.ts
* pnpm tsx scripts/compute-quality.ts
*/
import { readFileSync, writeFileSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
interface Skill {
id: string;
name: string;
source: string;
description?: string;
tags: string[];
author?: string;
version?: string;
agents?: string[];
quality?: number;
}
interface SkillsIndex {
version: number;
updatedAt: string;
skills: Skill[];
}
const VERIFIED_SOURCES = [
'anthropics/',
'vercel/',
'vercel-labs/',
'modelcontextprotocol/',
'cursor/',
'google/',
'microsoft/',
'openai/',
];
const TRUSTED_AUTHORS = [
'anthropic',
'vercel',
'google',
'microsoft',
'openai',
];
function computeDefaultQuality(skill: Skill): number {
let score = 70;
for (const verified of VERIFIED_SOURCES) {
if (skill.source.startsWith(verified)) {
score = 85;
break;
}
}
if (skill.author) {
for (const trusted of TRUSTED_AUTHORS) {
if (skill.author.toLowerCase().includes(trusted)) {
score = Math.max(score, 85);
break;
}
}
}
if (skill.description && skill.description.length > 50) {
score += 3;
}
if (skill.tags && skill.tags.length >= 3) {
score += 2;
}
if (skill.version) {
score += 2;
}
if (skill.agents && skill.agents.length >= 2) {
score += 3;
}
return Math.min(100, score);
}
function main(): void {
const skillsPath = join(__dirname, '../marketplace/skills.json');
console.log('Reading skills.json...');
let data: SkillsIndex;
try {
const content = readFileSync(skillsPath, 'utf-8');
data = JSON.parse(content) as SkillsIndex;
} catch (err) {
console.error(`Failed to read or parse skills.json: ${err instanceof Error ? err.message : String(err)}`);
process.exit(1);
}
if (!data.skills || data.skills.length === 0) {
console.log('No skills found in skills.json');
return;
}
console.log(`Processing ${data.skills.length} skills...`);
let updated = 0;
let verified = 0;
for (const skill of data.skills) {
const quality = computeDefaultQuality(skill);
if (skill.quality !== quality) {
skill.quality = quality;
updated++;
}
if (quality >= 85) {
verified++;
}
}
data.updatedAt = new Date().toISOString().split('T')[0];
console.log(`Updated ${updated} skills with quality scores`);
console.log(`Verified sources: ${verified} skills (quality >= 85)`);
const averageQuality = Math.round(
data.skills.reduce((sum, s) => sum + (s.quality ?? 70), 0) / data.skills.length
);
console.log(`Average quality: ${averageQuality}`);
try {
writeFileSync(skillsPath, JSON.stringify(data, null, 2) + '\n');
console.log('Saved skills.json');
} catch (err) {
console.error(`Failed to write skills.json: ${err instanceof Error ? err.message : String(err)}`);
process.exit(1);
}
}
main();