feat(quality): add skill quality evaluation system

- Add quality scoring module with structure, clarity, and specificity metrics
- Evaluate skills for metadata, triggers, boundaries, code examples
- Detect vague terms and penalize overly long skills
- Update validate command to include quality reports
- Add --verbose, --quality, --issues, --json flags
- Quality scores: A (90+), B (80+), C (70+), D (60+), F (<60)
This commit is contained in:
Rohit Ghumare
2026-01-28 11:32:15 +00:00
parent b9a724012a
commit 96f83fdc2e
4 changed files with 858 additions and 53 deletions
+293 -53
View File
@@ -1,87 +1,327 @@
import { existsSync, readdirSync } from 'node:fs';
import { join, basename } from 'node:path';
import chalk from 'chalk';
import { join, basename, resolve } from 'node:path';
import { Command, Option } from 'clipanion';
import { validateSkill } from '@skillkit/core';
import {
validateSkill,
evaluateSkillFile,
evaluateSkillDirectory,
getQualityGrade,
isHighQuality,
findAllSkills,
type QualityScore,
} from '@skillkit/core';
import { getSearchDirs } from '../helpers.js';
import {
colors,
symbols,
spinner,
success,
warn,
error,
header,
} from '../onboarding/index.js';
function formatScore(score: number): string {
if (score >= 80) return colors.success(`${score}`);
if (score >= 60) return colors.warning(`${score}`);
return colors.error(`${score}`);
}
function formatGrade(grade: string): string {
switch (grade) {
case 'A':
case 'B':
return colors.success(grade);
case 'C':
case 'D':
return colors.warning(grade);
default:
return colors.error(grade);
}
}
function printQualityReport(name: string, quality: QualityScore, verbose: boolean): void {
const grade = getQualityGrade(quality.overall);
const gradeDisplay = formatGrade(grade);
const scoreDisplay = formatScore(quality.overall);
console.log(`\n${colors.primary(name)}`);
console.log(` Grade: ${gradeDisplay} Score: ${scoreDisplay}/100`);
if (verbose) {
console.log('');
console.log(` ${colors.muted('Structure:')} ${formatScore(quality.structure.score)}/100`);
console.log(` ${quality.structure.hasMetadata ? colors.success(symbols.success) : colors.error(symbols.error)} Metadata`);
console.log(` ${quality.structure.hasDescription ? colors.success(symbols.success) : colors.error(symbols.error)} Description`);
console.log(` ${quality.structure.hasTriggers ? colors.success(symbols.success) : colors.error(symbols.error)} Trigger conditions`);
console.log(` ${quality.structure.hasExamples ? colors.success(symbols.success) : colors.error(symbols.error)} Code examples`);
console.log(` ${quality.structure.hasBoundaries ? colors.success(symbols.success) : colors.error(symbols.error)} Boundaries defined`);
console.log('');
console.log(` ${colors.muted('Clarity:')} ${formatScore(quality.clarity.score)}/100`);
console.log(` Lines: ${quality.clarity.lineCount}${quality.clarity.lineCount > 300 ? colors.warning(' (consider splitting)') : ''}`);
console.log(` Tokens: ${quality.clarity.tokenCount}${quality.clarity.tokenCount > 2000 ? colors.warning(' (high)') : ''}`);
console.log(` ${quality.clarity.hasHeaders ? colors.success(symbols.success) : colors.error(symbols.error)} Uses headers`);
console.log('');
console.log(` ${colors.muted('Specificity:')} ${formatScore(quality.specificity.score)}/100`);
console.log(` ${quality.specificity.hasConcreteCommands ? colors.success(symbols.success) : colors.error(symbols.error)} Concrete commands`);
console.log(` ${quality.specificity.hasFilePatterns ? colors.success(symbols.success) : colors.error(symbols.error)} File patterns`);
console.log(` ${quality.specificity.hasCodeExamples ? colors.success(symbols.success) : colors.error(symbols.error)} Multiple code examples`);
if (quality.specificity.vagueTermCount > 0) {
console.log(` ${colors.warning(symbols.warning)} ${quality.specificity.vagueTermCount} vague term(s)`);
}
}
if (quality.warnings.length > 0) {
console.log('');
console.log(` ${colors.warning('Warnings:')}`);
for (const w of quality.warnings) {
console.log(` ${colors.warning(symbols.warning)} ${w}`);
}
}
if (quality.suggestions.length > 0 && verbose) {
console.log('');
console.log(` ${colors.muted('Suggestions:')}`);
for (const s of quality.suggestions) {
console.log(` ${colors.muted(symbols.bullet)} ${s}`);
}
}
}
export class ValidateCommand extends Command {
static override paths = [['validate'], ['v']];
static override paths = [['validate']];
static override usage = Command.Usage({
description: 'Validate skill(s) against the Agent Skills specification (agentskills.io)',
description: 'Validate skill format and quality',
details: `
Validates skills against the specification and evaluates quality.
Quality scores are based on structure, clarity, and specificity.
`,
examples: [
['Validate a skill directory', '$0 validate ./my-skill'],
['Validate all skills in a directory', '$0 validate ./skills --all'],
['Validate all installed skills', '$0 validate'],
['Validate specific skill', '$0 validate my-skill'],
['Validate skill file', '$0 validate ./path/to/SKILL.md'],
['Detailed quality report', '$0 validate --verbose'],
['Only show issues', '$0 validate --issues'],
['Quality check only', '$0 validate --quality'],
['JSON output', '$0 validate --json'],
],
});
skillPath = Option.String({ required: true });
targets = Option.Rest();
verbose = Option.Boolean('--verbose,-v', false, {
description: 'Show detailed quality breakdown',
});
quality = Option.Boolean('--quality,-q', false, {
description: 'Only check quality (skip format validation)',
});
issues = Option.Boolean('--issues,-i', false, {
description: 'Only show skills with issues',
});
minScore = Option.String('--min-score,-m', {
description: 'Minimum acceptable score (default: 60)',
});
json = Option.Boolean('--json', false, {
description: 'Output as JSON',
});
all = Option.Boolean('--all,-a', false, {
description: 'Validate all skills in the directory',
description: 'Validate all skills in directory',
});
async execute(): Promise<number> {
const targetPath = this.skillPath;
const minScore = this.minScore ? parseInt(this.minScore, 10) : 60;
const results: Array<{ name: string; quality: QualityScore | null; formatValid: boolean; formatErrors: string[]; path?: string }> = [];
if (!existsSync(targetPath)) {
console.error(chalk.red(`Path does not exist: ${targetPath}`));
return 1;
}
if (this.targets.length > 0) {
for (const target of this.targets) {
const resolved = resolve(target);
const skillPaths: string[] = [];
if (existsSync(resolved)) {
if (this.all) {
const entries = readdirSync(resolved, { withFileTypes: true });
for (const entry of entries) {
if (entry.isDirectory()) {
const skillPath = join(resolved, entry.name);
if (existsSync(join(skillPath, 'SKILL.md'))) {
const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(skillPath);
const quality = evaluateSkillDirectory(skillPath);
results.push({
name: entry.name,
quality,
formatValid: formatResult.valid,
formatErrors: formatResult.errors || [],
path: skillPath,
});
}
}
}
} else {
const isFile = resolved.endsWith('.md') || resolved.endsWith('.mdc');
const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(resolved);
const quality = isFile ? evaluateSkillFile(resolved) : evaluateSkillDirectory(resolved);
results.push({
name: basename(target),
quality,
formatValid: formatResult.valid,
formatErrors: formatResult.errors || [],
path: resolved,
});
}
} else {
const searchDirs = getSearchDirs();
const skills = findAllSkills(searchDirs);
const found = skills.find(s => s.name === target);
if (this.all) {
const entries = readdirSync(targetPath, { withFileTypes: true });
for (const entry of entries) {
if (entry.isDirectory()) {
const skillPath = join(targetPath, entry.name);
if (existsSync(join(skillPath, 'SKILL.md'))) {
skillPaths.push(skillPath);
if (found) {
const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(found.path);
const quality = evaluateSkillDirectory(found.path);
results.push({
name: found.name,
quality,
formatValid: formatResult.valid,
formatErrors: formatResult.errors || [],
path: found.path,
});
} else {
if (!this.json) {
warn(`Skill not found: ${target}`);
}
}
}
}
if (skillPaths.length === 0) {
console.error(chalk.yellow('No skills found in directory'));
return 1;
}
} else {
skillPaths.push(targetPath);
}
const searchDirs = getSearchDirs();
const skills = findAllSkills(searchDirs);
let hasErrors = false;
for (const skillPath of skillPaths) {
const skillName = basename(skillPath);
const result = validateSkill(skillPath);
if (result.valid) {
console.log(chalk.green(`✓ ${skillName}`));
if (result.warnings && result.warnings.length > 0) {
result.warnings.forEach(w => {
console.log(chalk.yellow(` ⚠ ${w}`));
});
if (skills.length === 0) {
if (!this.json) {
warn('No installed skills found');
}
} else {
console.log(chalk.red(`✗ ${skillName}`));
result.errors.forEach(e => {
console.log(chalk.red(` • ${e}`));
return 0;
}
const s = spinner();
if (!this.json) {
s.start(`Evaluating ${skills.length} skill(s)...`);
}
for (const skill of skills) {
const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(skill.path);
const quality = evaluateSkillDirectory(skill.path);
results.push({
name: skill.name,
quality,
formatValid: formatResult.valid,
formatErrors: formatResult.errors || [],
path: skill.path,
});
hasErrors = true;
}
if (!this.json) {
s.stop(`Evaluated ${results.length} skill(s)`);
}
}
console.log();
if (hasErrors) {
console.log(chalk.red('Validation failed'));
console.log(chalk.dim('See https://agentskills.io/specification for the complete format'));
if (results.length === 0) {
if (!this.json) {
error('No skills to validate');
}
return 1;
}
console.log(chalk.green(`Validated ${skillPaths.length} skill(s) successfully`));
if (this.json) {
const output = results.map(r => ({
name: r.name,
path: r.path,
formatValid: r.formatValid,
formatErrors: r.formatErrors,
score: r.quality?.overall ?? null,
grade: r.quality ? getQualityGrade(r.quality.overall) : null,
highQuality: r.quality ? isHighQuality(r.quality) : false,
structure: r.quality?.structure ?? null,
clarity: r.quality?.clarity ?? null,
specificity: r.quality?.specificity ?? null,
warnings: r.quality?.warnings ?? [],
suggestions: r.quality?.suggestions ?? [],
}));
console.log(JSON.stringify(output, null, 2));
return 0;
}
header('Skill Validation Report');
let filteredResults = results;
if (this.issues) {
filteredResults = results.filter(r =>
!r.formatValid ||
(r.quality && r.quality.overall < minScore) ||
(r.quality && r.quality.warnings.length > 0)
);
}
filteredResults.sort((a, b) => (b.quality?.overall ?? 0) - (a.quality?.overall ?? 0));
for (const { name, quality, formatValid, formatErrors } of filteredResults) {
if (!formatValid) {
console.log(`\n${colors.error(symbols.error)} ${colors.primary(name)}`);
console.log(` ${colors.error('Format validation failed:')}`);
for (const err of formatErrors) {
console.log(` ${colors.error(symbols.bullet)} ${err}`);
}
}
if (quality) {
printQualityReport(name, quality, this.verbose);
}
}
console.log('');
const formatFailed = results.filter(r => !r.formatValid).length;
const withQuality = results.filter(r => r.quality !== null);
const highQuality = withQuality.filter(r => r.quality!.overall >= 80).length;
const passing = withQuality.filter(r => r.quality!.overall >= minScore).length;
const failing = withQuality.filter(r => r.quality!.overall < minScore).length;
console.log(colors.muted('Summary:'));
if (!this.quality) {
console.log(` ${formatFailed === 0 ? colors.success(symbols.success) : colors.error(symbols.error)} Format valid: ${results.length - formatFailed}/${results.length}`);
}
console.log(` ${colors.success(symbols.success)} High quality (80+): ${highQuality}`);
console.log(` ${colors.primary(symbols.bullet)} Passing (${minScore}+): ${passing}`);
if (failing > 0) {
console.log(` ${colors.error(symbols.error)} Below threshold: ${failing}`);
}
if (withQuality.length > 0) {
const avgScore = Math.round(withQuality.reduce((sum, r) => sum + r.quality!.overall, 0) / withQuality.length);
console.log(` ${colors.muted('Average score:')} ${formatScore(avgScore)}/100`);
}
if (formatFailed > 0 || failing > 0) {
console.log('');
if (formatFailed > 0) {
warn(`${formatFailed} skill(s) failed format validation`);
}
if (failing > 0) {
warn(`${failing} skill(s) below quality threshold`);
}
return 1;
}
console.log('');
success('All skills validated successfully');
return 0;
}
}
+2
View File
@@ -75,3 +75,5 @@ export * from './agents/index.js';
export * from './skill-translator.js';
export * from './manifest/index.js';
export * from './quality/index.js';
@@ -0,0 +1,203 @@
import { describe, it, expect } from 'vitest';
import {
evaluateSkillContent,
getQualityGrade,
isHighQuality,
} from '../index.js';
describe('Quality Evaluation', () => {
describe('evaluateSkillContent', () => {
it('should score a well-structured skill highly', () => {
const content = `---
name: test-skill
description: A test skill for validation
globs: ["**/*.ts"]
---
# Test Skill
## When to Use
Use this skill when working with TypeScript files.
## Instructions
Always follow these patterns:
\`\`\`typescript
function example(): void {
console.log('example');
}
\`\`\`
## Commands
Run tests with:
\`\`\`bash
npm test
\`\`\`
## Boundaries
Never commit secrets to the repository.
Always run linting before commits.
`;
const result = evaluateSkillContent(content);
expect(result.overall).toBeGreaterThanOrEqual(70);
expect(result.structure.hasMetadata).toBe(true);
expect(result.structure.hasTriggers).toBe(true);
expect(result.structure.hasExamples).toBe(true);
expect(result.structure.hasBoundaries).toBe(true);
expect(result.clarity.hasHeaders).toBe(true);
expect(result.specificity.hasConcreteCommands).toBe(true);
});
it('should score a minimal skill lower', () => {
const content = `# My Skill
Be helpful and assist the user with various tasks.
Try to do your best.
`;
const result = evaluateSkillContent(content);
expect(result.overall).toBeLessThan(50);
expect(result.structure.hasMetadata).toBe(false);
expect(result.structure.hasTriggers).toBe(false);
expect(result.specificity.vagueTermCount).toBeGreaterThan(0);
expect(result.warnings.length).toBeGreaterThan(0);
});
it('should penalize overly long skills', () => {
const longContent = '# Skill\n' + 'Some content.\n'.repeat(600);
const result = evaluateSkillContent(longContent);
expect(result.clarity.lineCount).toBeGreaterThan(500);
expect(result.clarity.score).toBeLessThan(70);
});
it('should detect vague terms', () => {
const content = `# Skill
Be helpful and assist the user.
Try to do your best when appropriate.
Help with various tasks as needed.
`;
const result = evaluateSkillContent(content);
expect(result.specificity.vagueTermCount).toBeGreaterThan(3);
});
it('should detect code examples', () => {
const content = `# Skill
\`\`\`typescript
const x = 1;
\`\`\`
\`\`\`javascript
const y = 2;
\`\`\`
`;
const result = evaluateSkillContent(content);
expect(result.structure.hasExamples).toBe(true);
expect(result.specificity.hasCodeExamples).toBe(true);
});
it('should detect boundary patterns', () => {
const content = `# Skill
## Boundaries
- Never commit secrets
- Always run tests
- Don't modify node_modules
- Avoid using deprecated APIs
`;
const result = evaluateSkillContent(content);
expect(result.structure.hasBoundaries).toBe(true);
});
it('should detect trigger conditions', () => {
const content = `# Skill
## When to Use
Use this skill when:
- Working with React components
- Building forms
`;
const result = evaluateSkillContent(content);
expect(result.structure.hasTriggers).toBe(true);
expect(result.structure.hasWhenToUse).toBe(true);
});
});
describe('getQualityGrade', () => {
it('should return A for scores >= 90', () => {
expect(getQualityGrade(90)).toBe('A');
expect(getQualityGrade(100)).toBe('A');
});
it('should return B for scores >= 80', () => {
expect(getQualityGrade(80)).toBe('B');
expect(getQualityGrade(89)).toBe('B');
});
it('should return C for scores >= 70', () => {
expect(getQualityGrade(70)).toBe('C');
expect(getQualityGrade(79)).toBe('C');
});
it('should return D for scores >= 60', () => {
expect(getQualityGrade(60)).toBe('D');
expect(getQualityGrade(69)).toBe('D');
});
it('should return F for scores < 60', () => {
expect(getQualityGrade(59)).toBe('F');
expect(getQualityGrade(0)).toBe('F');
});
});
describe('isHighQuality', () => {
it('should return true for high scoring skills with few warnings', () => {
const score = {
overall: 80,
structure: { score: 80, hasMetadata: true, hasDescription: true, hasTriggers: true, hasExamples: true, hasBoundaries: true, hasWhenToUse: true },
clarity: { score: 80, lineCount: 100, tokenCount: 500, avgSentenceLength: 15, hasHeaders: true },
specificity: { score: 80, hasConcreteCommands: true, hasFilePatterns: true, hasCodeExamples: true, vagueTermCount: 0 },
warnings: [],
suggestions: [],
};
expect(isHighQuality(score)).toBe(true);
});
it('should return false for low scoring skills', () => {
const score = {
overall: 50,
structure: { score: 50, hasMetadata: false, hasDescription: false, hasTriggers: false, hasExamples: false, hasBoundaries: false, hasWhenToUse: false },
clarity: { score: 50, lineCount: 100, tokenCount: 500, avgSentenceLength: 15, hasHeaders: true },
specificity: { score: 50, hasConcreteCommands: false, hasFilePatterns: false, hasCodeExamples: false, vagueTermCount: 5 },
warnings: ['Missing triggers', 'No examples'],
suggestions: [],
};
expect(isHighQuality(score)).toBe(false);
});
it('should return false for skills with many warnings', () => {
const score = {
overall: 75,
structure: { score: 75, hasMetadata: true, hasDescription: true, hasTriggers: false, hasExamples: false, hasBoundaries: false, hasWhenToUse: false },
clarity: { score: 75, lineCount: 100, tokenCount: 500, avgSentenceLength: 15, hasHeaders: true },
specificity: { score: 75, hasConcreteCommands: true, hasFilePatterns: true, hasCodeExamples: false, vagueTermCount: 0 },
warnings: ['Warning 1', 'Warning 2', 'Warning 3'],
suggestions: [],
};
expect(isHighQuality(score)).toBe(false);
});
});
});
+360
View File
@@ -0,0 +1,360 @@
import { readFileSync, existsSync } from 'node:fs';
import { join, basename } from 'node:path';
export interface QualityScore {
overall: number;
structure: StructureScore;
clarity: ClarityScore;
specificity: SpecificityScore;
warnings: string[];
suggestions: string[];
}
export interface StructureScore {
score: number;
hasMetadata: boolean;
hasDescription: boolean;
hasTriggers: boolean;
hasExamples: boolean;
hasBoundaries: boolean;
hasWhenToUse: boolean;
}
export interface ClarityScore {
score: number;
lineCount: number;
tokenCount: number;
avgSentenceLength: number;
hasHeaders: boolean;
}
export interface SpecificityScore {
score: number;
hasConcreteCommands: boolean;
hasFilePatterns: boolean;
hasCodeExamples: boolean;
vagueTermCount: number;
}
const VAGUE_TERMS = [
'be helpful',
'assist the user',
'help with',
'try to',
'attempt to',
'do your best',
'as needed',
'when appropriate',
'if necessary',
'general purpose',
'various tasks',
'many things',
'etc.',
'and so on',
'stuff like that',
];
const BOUNDARY_PATTERNS = [
/never\s+(?:do|use|commit|push|delete|remove)/i,
/always\s+(?:do|use|check|verify|ensure)/i,
/don'?t\s+(?:do|use|commit|push|delete)/i,
/avoid\s+(?:using|doing|committing)/i,
/must\s+(?:not|always|never)/i,
/forbidden/i,
/prohibited/i,
/required/i,
];
const TRIGGER_PATTERNS = [
/when\s+to\s+use/i,
/use\s+this\s+(?:skill|when)/i,
/triggers?\s*(?:when|:)/i,
/activated?\s+(?:when|by)/i,
/applies?\s+(?:when|to)/i,
/invoke\s+(?:when|this)/i,
];
const COMMAND_PATTERNS = [
/```(?:bash|sh|shell|zsh)[\s\S]*?```/g,
/`(?:npm|pnpm|yarn|bun|npx|git|docker|kubectl)\s+[^`]+`/g,
/\$\s*\w+/g,
];
const CODE_EXAMPLE_PATTERNS = [
/```(?:typescript|javascript|tsx|jsx|python|go|rust|java)[\s\S]*?```/g,
/```[\s\S]{50,}?```/g,
];
const FILE_PATTERN_PATTERNS = [
/\*\*\/\*\.\w+/,
/\.\w+$/m,
/glob[s]?\s*[:=]/i,
/include[s]?\s*[:=]/i,
/pattern[s]?\s*[:=]/i,
];
function extractFrontmatter(content: string): Record<string, unknown> | null {
const match = content.match(/^---\n([\s\S]*?)\n---/);
if (!match) return null;
const frontmatter: Record<string, unknown> = {};
const lines = match[1].split('\n');
for (const line of lines) {
const colonIndex = line.indexOf(':');
if (colonIndex > 0) {
const key = line.slice(0, colonIndex).trim();
const value = line.slice(colonIndex + 1).trim();
frontmatter[key] = value;
}
}
return frontmatter;
}
function countTokens(text: string): number {
return Math.ceil(text.length / 4);
}
function countVagueTerms(content: string): number {
const lowerContent = content.toLowerCase();
let count = 0;
for (const term of VAGUE_TERMS) {
const regex = new RegExp(term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'gi');
const matches = lowerContent.match(regex);
if (matches) count += matches.length;
}
return count;
}
function hasPattern(content: string, patterns: RegExp[]): boolean {
return patterns.some(p => p.test(content));
}
function countMatches(content: string, patterns: RegExp[]): number {
let count = 0;
for (const pattern of patterns) {
const matches = content.match(pattern);
if (matches) count += matches.length;
}
return count;
}
function calculateAvgSentenceLength(content: string): number {
const contentWithoutCode = content.replace(/```[\s\S]*?```/g, '');
const sentences = contentWithoutCode.split(/[.!?]+/).filter(s => s.trim().length > 0);
if (sentences.length === 0) return 0;
const totalWords = sentences.reduce((sum, s) => sum + s.trim().split(/\s+/).length, 0);
return Math.round(totalWords / sentences.length);
}
function evaluateStructure(content: string): StructureScore {
const frontmatter = extractFrontmatter(content);
const hasMetadata = frontmatter !== null && Object.keys(frontmatter).length > 0;
const hasDescription = !!(frontmatter?.description) || /^#+\s*description/im.test(content);
const hasTriggers = hasPattern(content, TRIGGER_PATTERNS);
const hasExamples = countMatches(content, CODE_EXAMPLE_PATTERNS) > 0;
const hasBoundaries = hasPattern(content, BOUNDARY_PATTERNS);
const hasWhenToUse = /when\s+to\s+use|use\s+case|scenario/i.test(content);
let score = 0;
if (hasMetadata) score += 15;
if (hasDescription) score += 10;
if (hasTriggers) score += 20;
if (hasExamples) score += 20;
if (hasBoundaries) score += 20;
if (hasWhenToUse) score += 15;
return {
score: Math.min(100, score),
hasMetadata,
hasDescription,
hasTriggers,
hasExamples,
hasBoundaries,
hasWhenToUse,
};
}
function evaluateClarity(content: string): ClarityScore {
const lines = content.split('\n');
const lineCount = lines.length;
const tokenCount = countTokens(content);
const avgSentenceLength = calculateAvgSentenceLength(content);
const hasHeaders = /^#+\s+/m.test(content);
let score = 100;
if (lineCount > 500) score -= 30;
else if (lineCount > 300) score -= 15;
else if (lineCount > 150) score -= 5;
if (tokenCount > 4000) score -= 30;
else if (tokenCount > 2000) score -= 15;
else if (tokenCount > 1000) score -= 5;
if (avgSentenceLength > 30) score -= 20;
else if (avgSentenceLength > 20) score -= 10;
if (!hasHeaders) score -= 15;
return {
score: Math.max(0, score),
lineCount,
tokenCount,
avgSentenceLength,
hasHeaders,
};
}
function evaluateSpecificity(content: string): SpecificityScore {
const hasConcreteCommands = countMatches(content, COMMAND_PATTERNS) > 0;
const hasFilePatterns = hasPattern(content, FILE_PATTERN_PATTERNS);
const hasCodeExamples = countMatches(content, CODE_EXAMPLE_PATTERNS) >= 2;
const vagueTermCount = countVagueTerms(content);
let score = 0;
if (hasConcreteCommands) score += 30;
if (hasFilePatterns) score += 25;
if (hasCodeExamples) score += 30;
if (vagueTermCount === 0) score += 15;
else if (vagueTermCount <= 2) score += 10;
else if (vagueTermCount <= 5) score += 5;
return {
score: Math.min(100, score),
hasConcreteCommands,
hasFilePatterns,
hasCodeExamples,
vagueTermCount,
};
}
function generateWarnings(
structure: StructureScore,
clarity: ClarityScore,
specificity: SpecificityScore
): string[] {
const warnings: string[] = [];
if (!structure.hasMetadata) {
warnings.push('Missing frontmatter metadata');
}
if (!structure.hasTriggers) {
warnings.push('No trigger conditions defined');
}
if (!structure.hasBoundaries) {
warnings.push('No boundaries or constraints specified');
}
if (clarity.lineCount > 500) {
warnings.push(`Skill is too long (${clarity.lineCount} lines)`);
}
if (clarity.tokenCount > 4000) {
warnings.push(`High token usage (${clarity.tokenCount} tokens)`);
}
if (specificity.vagueTermCount > 5) {
warnings.push(`Contains ${specificity.vagueTermCount} vague terms`);
}
if (!specificity.hasCodeExamples) {
warnings.push('No code examples provided');
}
return warnings;
}
function generateSuggestions(
structure: StructureScore,
clarity: ClarityScore,
specificity: SpecificityScore
): string[] {
const suggestions: string[] = [];
if (!structure.hasMetadata) {
suggestions.push('Add YAML frontmatter with name, description, and globs');
}
if (!structure.hasTriggers) {
suggestions.push('Add a "When to Use" section with specific trigger conditions');
}
if (!structure.hasExamples) {
suggestions.push('Include code examples showing expected output');
}
if (!structure.hasBoundaries) {
suggestions.push('Define boundaries: what the agent should never do');
}
if (clarity.lineCount > 300) {
suggestions.push('Consider splitting into multiple focused skills');
}
if (!clarity.hasHeaders) {
suggestions.push('Use markdown headers to organize content');
}
if (specificity.vagueTermCount > 0) {
suggestions.push('Replace vague terms with specific instructions');
}
if (!specificity.hasConcreteCommands) {
suggestions.push('Add concrete executable commands with flags');
}
return suggestions;
}
export function evaluateSkillContent(content: string): QualityScore {
const structure = evaluateStructure(content);
const clarity = evaluateClarity(content);
const specificity = evaluateSpecificity(content);
const overall = Math.round(
structure.score * 0.4 +
clarity.score * 0.3 +
specificity.score * 0.3
);
const warnings = generateWarnings(structure, clarity, specificity);
const suggestions = generateSuggestions(structure, clarity, specificity);
return {
overall,
structure,
clarity,
specificity,
warnings,
suggestions,
};
}
export function evaluateSkillFile(filePath: string): QualityScore | null {
if (!existsSync(filePath)) return null;
const content = readFileSync(filePath, 'utf-8');
return evaluateSkillContent(content);
}
export function evaluateSkillDirectory(dirPath: string): QualityScore | null {
const skillMdPath = join(dirPath, 'SKILL.md');
if (existsSync(skillMdPath)) {
return evaluateSkillFile(skillMdPath);
}
const mdcFiles = ['index.mdc', `${basename(dirPath)}.mdc`];
for (const file of mdcFiles) {
const mdcPath = join(dirPath, file);
if (existsSync(mdcPath)) {
return evaluateSkillFile(mdcPath);
}
}
return null;
}
export function getQualityGrade(score: number): string {
if (score >= 90) return 'A';
if (score >= 80) return 'B';
if (score >= 70) return 'C';
if (score >= 60) return 'D';
return 'F';
}
export function isHighQuality(score: QualityScore): boolean {
return score.overall >= 70 && score.warnings.length <= 2;
}