mirror of
https://github.com/rohitg00/skillkit.git
synced 2026-11-19 02:13:58 +00:00
feat(quality): add skill quality evaluation system (#20)
* feat(quality): add skill quality evaluation system - Add quality scoring module with structure, clarity, and specificity metrics - Evaluate skills for metadata, triggers, boundaries, code examples - Detect vague terms and penalize overly long skills - Update validate command to include quality reports - Add --verbose, --quality, --issues, --json flags - Quality scores: A (90+), B (80+), C (70+), D (60+), F (<60) * fix: address CodeRabbit and Devin review suggestions - Fix JSON output to return exit code 1 on validation failures - Add --min-score NaN validation with warning message - Add directory guard for --all flag with .mdc file support - Fix CRLF handling in frontmatter extraction - Fix code blocks double-counting in specificity evaluation - Fix warning message inconsistency for code examples - Add try/catch error handling in evaluateSkillFile - Fix isHighQuality threshold to match UI display (80)
This commit is contained in:
@@ -1,87 +1,349 @@
|
||||
import { existsSync, readdirSync } from 'node:fs';
|
||||
import { join, basename } from 'node:path';
|
||||
import chalk from 'chalk';
|
||||
import { existsSync, readdirSync, statSync } from 'node:fs';
|
||||
import { join, basename, resolve } from 'node:path';
|
||||
import { Command, Option } from 'clipanion';
|
||||
import { validateSkill } from '@skillkit/core';
|
||||
import {
|
||||
validateSkill,
|
||||
evaluateSkillFile,
|
||||
evaluateSkillDirectory,
|
||||
getQualityGrade,
|
||||
isHighQuality,
|
||||
findAllSkills,
|
||||
type QualityScore,
|
||||
} from '@skillkit/core';
|
||||
import { getSearchDirs } from '../helpers.js';
|
||||
import {
|
||||
colors,
|
||||
symbols,
|
||||
spinner,
|
||||
success,
|
||||
warn,
|
||||
error,
|
||||
header,
|
||||
} from '../onboarding/index.js';
|
||||
|
||||
function formatScore(score: number): string {
|
||||
if (score >= 80) return colors.success(`${score}`);
|
||||
if (score >= 60) return colors.warning(`${score}`);
|
||||
return colors.error(`${score}`);
|
||||
}
|
||||
|
||||
function formatGrade(grade: string): string {
|
||||
switch (grade) {
|
||||
case 'A':
|
||||
case 'B':
|
||||
return colors.success(grade);
|
||||
case 'C':
|
||||
case 'D':
|
||||
return colors.warning(grade);
|
||||
default:
|
||||
return colors.error(grade);
|
||||
}
|
||||
}
|
||||
|
||||
function printQualityReport(name: string, quality: QualityScore, verbose: boolean): void {
|
||||
const grade = getQualityGrade(quality.overall);
|
||||
const gradeDisplay = formatGrade(grade);
|
||||
const scoreDisplay = formatScore(quality.overall);
|
||||
|
||||
console.log(`\n${colors.primary(name)}`);
|
||||
console.log(` Grade: ${gradeDisplay} Score: ${scoreDisplay}/100`);
|
||||
|
||||
if (verbose) {
|
||||
console.log('');
|
||||
console.log(` ${colors.muted('Structure:')} ${formatScore(quality.structure.score)}/100`);
|
||||
console.log(` ${quality.structure.hasMetadata ? colors.success(symbols.success) : colors.error(symbols.error)} Metadata`);
|
||||
console.log(` ${quality.structure.hasDescription ? colors.success(symbols.success) : colors.error(symbols.error)} Description`);
|
||||
console.log(` ${quality.structure.hasTriggers ? colors.success(symbols.success) : colors.error(symbols.error)} Trigger conditions`);
|
||||
console.log(` ${quality.structure.hasExamples ? colors.success(symbols.success) : colors.error(symbols.error)} Code examples`);
|
||||
console.log(` ${quality.structure.hasBoundaries ? colors.success(symbols.success) : colors.error(symbols.error)} Boundaries defined`);
|
||||
|
||||
console.log('');
|
||||
console.log(` ${colors.muted('Clarity:')} ${formatScore(quality.clarity.score)}/100`);
|
||||
console.log(` Lines: ${quality.clarity.lineCount}${quality.clarity.lineCount > 300 ? colors.warning(' (consider splitting)') : ''}`);
|
||||
console.log(` Tokens: ${quality.clarity.tokenCount}${quality.clarity.tokenCount > 2000 ? colors.warning(' (high)') : ''}`);
|
||||
console.log(` ${quality.clarity.hasHeaders ? colors.success(symbols.success) : colors.error(symbols.error)} Uses headers`);
|
||||
|
||||
console.log('');
|
||||
console.log(` ${colors.muted('Specificity:')} ${formatScore(quality.specificity.score)}/100`);
|
||||
console.log(` ${quality.specificity.hasConcreteCommands ? colors.success(symbols.success) : colors.error(symbols.error)} Concrete commands`);
|
||||
console.log(` ${quality.specificity.hasFilePatterns ? colors.success(symbols.success) : colors.error(symbols.error)} File patterns`);
|
||||
console.log(` ${quality.specificity.hasCodeExamples ? colors.success(symbols.success) : colors.error(symbols.error)} Multiple code examples`);
|
||||
if (quality.specificity.vagueTermCount > 0) {
|
||||
console.log(` ${colors.warning(symbols.warning)} ${quality.specificity.vagueTermCount} vague term(s)`);
|
||||
}
|
||||
}
|
||||
|
||||
if (quality.warnings.length > 0) {
|
||||
console.log('');
|
||||
console.log(` ${colors.warning('Warnings:')}`);
|
||||
for (const w of quality.warnings) {
|
||||
console.log(` ${colors.warning(symbols.warning)} ${w}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (quality.suggestions.length > 0 && verbose) {
|
||||
console.log('');
|
||||
console.log(` ${colors.muted('Suggestions:')}`);
|
||||
for (const s of quality.suggestions) {
|
||||
console.log(` ${colors.muted(symbols.bullet)} ${s}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class ValidateCommand extends Command {
|
||||
static override paths = [['validate'], ['v']];
|
||||
static override paths = [['validate']];
|
||||
|
||||
static override usage = Command.Usage({
|
||||
description: 'Validate skill(s) against the Agent Skills specification (agentskills.io)',
|
||||
description: 'Validate skill format and quality',
|
||||
details: `
|
||||
Validates skills against the specification and evaluates quality.
|
||||
Quality scores are based on structure, clarity, and specificity.
|
||||
`,
|
||||
examples: [
|
||||
['Validate a skill directory', '$0 validate ./my-skill'],
|
||||
['Validate all skills in a directory', '$0 validate ./skills --all'],
|
||||
['Validate all installed skills', '$0 validate'],
|
||||
['Validate specific skill', '$0 validate my-skill'],
|
||||
['Validate skill file', '$0 validate ./path/to/SKILL.md'],
|
||||
['Detailed quality report', '$0 validate --verbose'],
|
||||
['Only show issues', '$0 validate --issues'],
|
||||
['Quality check only', '$0 validate --quality'],
|
||||
['JSON output', '$0 validate --json'],
|
||||
],
|
||||
});
|
||||
|
||||
skillPath = Option.String({ required: true });
|
||||
targets = Option.Rest();
|
||||
|
||||
verbose = Option.Boolean('--verbose,-v', false, {
|
||||
description: 'Show detailed quality breakdown',
|
||||
});
|
||||
|
||||
quality = Option.Boolean('--quality,-q', false, {
|
||||
description: 'Only check quality (skip format validation)',
|
||||
});
|
||||
|
||||
issues = Option.Boolean('--issues,-i', false, {
|
||||
description: 'Only show skills with issues',
|
||||
});
|
||||
|
||||
minScore = Option.String('--min-score,-m', {
|
||||
description: 'Minimum acceptable score (default: 60)',
|
||||
});
|
||||
|
||||
json = Option.Boolean('--json', false, {
|
||||
description: 'Output as JSON',
|
||||
});
|
||||
|
||||
all = Option.Boolean('--all,-a', false, {
|
||||
description: 'Validate all skills in the directory',
|
||||
description: 'Validate all skills in directory',
|
||||
});
|
||||
|
||||
async execute(): Promise<number> {
|
||||
const targetPath = this.skillPath;
|
||||
|
||||
if (!existsSync(targetPath)) {
|
||||
console.error(chalk.red(`Path does not exist: ${targetPath}`));
|
||||
return 1;
|
||||
let minScore = 60;
|
||||
if (this.minScore) {
|
||||
const parsed = parseInt(this.minScore, 10);
|
||||
if (Number.isNaN(parsed) || parsed < 0 || parsed > 100) {
|
||||
if (!this.json) {
|
||||
warn(`Invalid --min-score value "${this.minScore}", using default: 60`);
|
||||
}
|
||||
} else {
|
||||
minScore = parsed;
|
||||
}
|
||||
}
|
||||
const results: Array<{ name: string; quality: QualityScore | null; formatValid: boolean; formatErrors: string[]; path?: string }> = [];
|
||||
|
||||
const skillPaths: string[] = [];
|
||||
if (this.targets.length > 0) {
|
||||
for (const target of this.targets) {
|
||||
const resolved = resolve(target);
|
||||
|
||||
if (this.all) {
|
||||
const entries = readdirSync(targetPath, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory()) {
|
||||
const skillPath = join(targetPath, entry.name);
|
||||
if (existsSync(join(skillPath, 'SKILL.md'))) {
|
||||
skillPaths.push(skillPath);
|
||||
if (existsSync(resolved)) {
|
||||
const isDirectory = statSync(resolved).isDirectory();
|
||||
if (this.all) {
|
||||
if (!isDirectory) {
|
||||
if (!this.json) {
|
||||
warn(`--all requires a directory, but "${target}" is a file`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const entries = readdirSync(resolved, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory()) {
|
||||
const skillPath = join(resolved, entry.name);
|
||||
const hasSkillMd = existsSync(join(skillPath, 'SKILL.md'));
|
||||
const hasIndexMdc = existsSync(join(skillPath, 'index.mdc'));
|
||||
const hasNamedMdc = existsSync(join(skillPath, `${entry.name}.mdc`));
|
||||
if (hasSkillMd || hasIndexMdc || hasNamedMdc) {
|
||||
const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(skillPath);
|
||||
const quality = evaluateSkillDirectory(skillPath);
|
||||
results.push({
|
||||
name: entry.name,
|
||||
quality,
|
||||
formatValid: formatResult.valid,
|
||||
formatErrors: formatResult.errors || [],
|
||||
path: skillPath,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const isFile = resolved.endsWith('.md') || resolved.endsWith('.mdc');
|
||||
const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(resolved);
|
||||
const quality = isFile ? evaluateSkillFile(resolved) : evaluateSkillDirectory(resolved);
|
||||
results.push({
|
||||
name: basename(target),
|
||||
quality,
|
||||
formatValid: formatResult.valid,
|
||||
formatErrors: formatResult.errors || [],
|
||||
path: resolved,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
const searchDirs = getSearchDirs();
|
||||
const skills = findAllSkills(searchDirs);
|
||||
const found = skills.find(s => s.name === target);
|
||||
|
||||
if (found) {
|
||||
const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(found.path);
|
||||
const quality = evaluateSkillDirectory(found.path);
|
||||
results.push({
|
||||
name: found.name,
|
||||
quality,
|
||||
formatValid: formatResult.valid,
|
||||
formatErrors: formatResult.errors || [],
|
||||
path: found.path,
|
||||
});
|
||||
} else {
|
||||
if (!this.json) {
|
||||
warn(`Skill not found: ${target}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (skillPaths.length === 0) {
|
||||
console.error(chalk.yellow('No skills found in directory'));
|
||||
return 1;
|
||||
}
|
||||
} else {
|
||||
skillPaths.push(targetPath);
|
||||
}
|
||||
const searchDirs = getSearchDirs();
|
||||
const skills = findAllSkills(searchDirs);
|
||||
|
||||
let hasErrors = false;
|
||||
|
||||
for (const skillPath of skillPaths) {
|
||||
const skillName = basename(skillPath);
|
||||
const result = validateSkill(skillPath);
|
||||
|
||||
if (result.valid) {
|
||||
console.log(chalk.green(`✓ ${skillName}`));
|
||||
|
||||
if (result.warnings && result.warnings.length > 0) {
|
||||
result.warnings.forEach(w => {
|
||||
console.log(chalk.yellow(` ⚠ ${w}`));
|
||||
});
|
||||
if (skills.length === 0) {
|
||||
if (!this.json) {
|
||||
warn('No installed skills found');
|
||||
}
|
||||
} else {
|
||||
console.log(chalk.red(`✗ ${skillName}`));
|
||||
result.errors.forEach(e => {
|
||||
console.log(chalk.red(` • ${e}`));
|
||||
return 0;
|
||||
}
|
||||
|
||||
const s = spinner();
|
||||
if (!this.json) {
|
||||
s.start(`Evaluating ${skills.length} skill(s)...`);
|
||||
}
|
||||
|
||||
for (const skill of skills) {
|
||||
const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(skill.path);
|
||||
const quality = evaluateSkillDirectory(skill.path);
|
||||
results.push({
|
||||
name: skill.name,
|
||||
quality,
|
||||
formatValid: formatResult.valid,
|
||||
formatErrors: formatResult.errors || [],
|
||||
path: skill.path,
|
||||
});
|
||||
hasErrors = true;
|
||||
}
|
||||
|
||||
if (!this.json) {
|
||||
s.stop(`Evaluated ${results.length} skill(s)`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log();
|
||||
|
||||
if (hasErrors) {
|
||||
console.log(chalk.red('Validation failed'));
|
||||
console.log(chalk.dim('See https://agentskills.io/specification for the complete format'));
|
||||
if (results.length === 0) {
|
||||
if (!this.json) {
|
||||
error('No skills to validate');
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
console.log(chalk.green(`Validated ${skillPaths.length} skill(s) successfully`));
|
||||
if (this.json) {
|
||||
const output = results.map(r => ({
|
||||
name: r.name,
|
||||
path: r.path,
|
||||
formatValid: r.formatValid,
|
||||
formatErrors: r.formatErrors,
|
||||
score: r.quality?.overall ?? null,
|
||||
grade: r.quality ? getQualityGrade(r.quality.overall) : null,
|
||||
highQuality: r.quality ? isHighQuality(r.quality) : false,
|
||||
structure: r.quality?.structure ?? null,
|
||||
clarity: r.quality?.clarity ?? null,
|
||||
specificity: r.quality?.specificity ?? null,
|
||||
warnings: r.quality?.warnings ?? [],
|
||||
suggestions: r.quality?.suggestions ?? [],
|
||||
}));
|
||||
console.log(JSON.stringify(output, null, 2));
|
||||
const hasFormatFailures = results.some(r => !r.formatValid);
|
||||
const hasBelowThreshold = results.some(r => r.quality && r.quality.overall < minScore);
|
||||
return hasFormatFailures || hasBelowThreshold ? 1 : 0;
|
||||
}
|
||||
|
||||
header('Skill Validation Report');
|
||||
|
||||
let filteredResults = results;
|
||||
if (this.issues) {
|
||||
filteredResults = results.filter(r =>
|
||||
!r.formatValid ||
|
||||
(r.quality && r.quality.overall < minScore) ||
|
||||
(r.quality && r.quality.warnings.length > 0)
|
||||
);
|
||||
}
|
||||
|
||||
filteredResults.sort((a, b) => (b.quality?.overall ?? 0) - (a.quality?.overall ?? 0));
|
||||
|
||||
for (const { name, quality, formatValid, formatErrors } of filteredResults) {
|
||||
if (!formatValid) {
|
||||
console.log(`\n${colors.error(symbols.error)} ${colors.primary(name)}`);
|
||||
console.log(` ${colors.error('Format validation failed:')}`);
|
||||
for (const err of formatErrors) {
|
||||
console.log(` ${colors.error(symbols.bullet)} ${err}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (quality) {
|
||||
printQualityReport(name, quality, this.verbose);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('');
|
||||
|
||||
const formatFailed = results.filter(r => !r.formatValid).length;
|
||||
const withQuality = results.filter(r => r.quality !== null);
|
||||
const highQuality = withQuality.filter(r => r.quality!.overall >= 80).length;
|
||||
const passing = withQuality.filter(r => r.quality!.overall >= minScore).length;
|
||||
const failing = withQuality.filter(r => r.quality!.overall < minScore).length;
|
||||
|
||||
console.log(colors.muted('Summary:'));
|
||||
|
||||
if (!this.quality) {
|
||||
console.log(` ${formatFailed === 0 ? colors.success(symbols.success) : colors.error(symbols.error)} Format valid: ${results.length - formatFailed}/${results.length}`);
|
||||
}
|
||||
|
||||
console.log(` ${colors.success(symbols.success)} High quality (80+): ${highQuality}`);
|
||||
console.log(` ${colors.primary(symbols.bullet)} Passing (${minScore}+): ${passing}`);
|
||||
|
||||
if (failing > 0) {
|
||||
console.log(` ${colors.error(symbols.error)} Below threshold: ${failing}`);
|
||||
}
|
||||
|
||||
if (withQuality.length > 0) {
|
||||
const avgScore = Math.round(withQuality.reduce((sum, r) => sum + r.quality!.overall, 0) / withQuality.length);
|
||||
console.log(` ${colors.muted('Average score:')} ${formatScore(avgScore)}/100`);
|
||||
}
|
||||
|
||||
if (formatFailed > 0 || failing > 0) {
|
||||
console.log('');
|
||||
if (formatFailed > 0) {
|
||||
warn(`${formatFailed} skill(s) failed format validation`);
|
||||
}
|
||||
if (failing > 0) {
|
||||
warn(`${failing} skill(s) below quality threshold`);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
console.log('');
|
||||
success('All skills validated successfully');
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,3 +75,5 @@ export * from './agents/index.js';
|
||||
|
||||
export * from './skill-translator.js';
|
||||
export * from './manifest/index.js';
|
||||
|
||||
export * from './quality/index.js';
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
evaluateSkillContent,
|
||||
getQualityGrade,
|
||||
isHighQuality,
|
||||
} from '../index.js';
|
||||
|
||||
describe('Quality Evaluation', () => {
|
||||
describe('evaluateSkillContent', () => {
|
||||
it('should score a well-structured skill highly', () => {
|
||||
const content = `---
|
||||
name: test-skill
|
||||
description: A test skill for validation
|
||||
globs: ["**/*.ts"]
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
|
||||
## When to Use
|
||||
|
||||
Use this skill when working with TypeScript files.
|
||||
|
||||
## Instructions
|
||||
|
||||
Always follow these patterns:
|
||||
|
||||
\`\`\`typescript
|
||||
function example(): void {
|
||||
console.log('example');
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
## Commands
|
||||
|
||||
Run tests with:
|
||||
|
||||
\`\`\`bash
|
||||
npm test
|
||||
\`\`\`
|
||||
|
||||
## Boundaries
|
||||
|
||||
Never commit secrets to the repository.
|
||||
Always run linting before commits.
|
||||
`;
|
||||
|
||||
const result = evaluateSkillContent(content);
|
||||
expect(result.overall).toBeGreaterThanOrEqual(70);
|
||||
expect(result.structure.hasMetadata).toBe(true);
|
||||
expect(result.structure.hasTriggers).toBe(true);
|
||||
expect(result.structure.hasExamples).toBe(true);
|
||||
expect(result.structure.hasBoundaries).toBe(true);
|
||||
expect(result.clarity.hasHeaders).toBe(true);
|
||||
expect(result.specificity.hasConcreteCommands).toBe(true);
|
||||
});
|
||||
|
||||
it('should score a minimal skill lower', () => {
|
||||
const content = `# My Skill
|
||||
|
||||
Be helpful and assist the user with various tasks.
|
||||
Try to do your best.
|
||||
`;
|
||||
|
||||
const result = evaluateSkillContent(content);
|
||||
expect(result.overall).toBeLessThan(50);
|
||||
expect(result.structure.hasMetadata).toBe(false);
|
||||
expect(result.structure.hasTriggers).toBe(false);
|
||||
expect(result.specificity.vagueTermCount).toBeGreaterThan(0);
|
||||
expect(result.warnings.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should penalize overly long skills', () => {
|
||||
const longContent = '# Skill\n' + 'Some content.\n'.repeat(600);
|
||||
const result = evaluateSkillContent(longContent);
|
||||
expect(result.clarity.lineCount).toBeGreaterThan(500);
|
||||
expect(result.clarity.score).toBeLessThan(70);
|
||||
});
|
||||
|
||||
it('should detect vague terms', () => {
|
||||
const content = `# Skill
|
||||
|
||||
Be helpful and assist the user.
|
||||
Try to do your best when appropriate.
|
||||
Help with various tasks as needed.
|
||||
`;
|
||||
|
||||
const result = evaluateSkillContent(content);
|
||||
expect(result.specificity.vagueTermCount).toBeGreaterThan(3);
|
||||
});
|
||||
|
||||
it('should detect code examples', () => {
|
||||
const content = `# Skill
|
||||
|
||||
\`\`\`typescript
|
||||
const x = 1;
|
||||
\`\`\`
|
||||
|
||||
\`\`\`javascript
|
||||
const y = 2;
|
||||
\`\`\`
|
||||
`;
|
||||
|
||||
const result = evaluateSkillContent(content);
|
||||
expect(result.structure.hasExamples).toBe(true);
|
||||
expect(result.specificity.hasCodeExamples).toBe(true);
|
||||
});
|
||||
|
||||
it('should detect boundary patterns', () => {
|
||||
const content = `# Skill
|
||||
|
||||
## Boundaries
|
||||
|
||||
- Never commit secrets
|
||||
- Always run tests
|
||||
- Don't modify node_modules
|
||||
- Avoid using deprecated APIs
|
||||
`;
|
||||
|
||||
const result = evaluateSkillContent(content);
|
||||
expect(result.structure.hasBoundaries).toBe(true);
|
||||
});
|
||||
|
||||
it('should detect trigger conditions', () => {
|
||||
const content = `# Skill
|
||||
|
||||
## When to Use
|
||||
|
||||
Use this skill when:
|
||||
- Working with React components
|
||||
- Building forms
|
||||
`;
|
||||
|
||||
const result = evaluateSkillContent(content);
|
||||
expect(result.structure.hasTriggers).toBe(true);
|
||||
expect(result.structure.hasWhenToUse).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getQualityGrade', () => {
|
||||
it('should return A for scores >= 90', () => {
|
||||
expect(getQualityGrade(90)).toBe('A');
|
||||
expect(getQualityGrade(100)).toBe('A');
|
||||
});
|
||||
|
||||
it('should return B for scores >= 80', () => {
|
||||
expect(getQualityGrade(80)).toBe('B');
|
||||
expect(getQualityGrade(89)).toBe('B');
|
||||
});
|
||||
|
||||
it('should return C for scores >= 70', () => {
|
||||
expect(getQualityGrade(70)).toBe('C');
|
||||
expect(getQualityGrade(79)).toBe('C');
|
||||
});
|
||||
|
||||
it('should return D for scores >= 60', () => {
|
||||
expect(getQualityGrade(60)).toBe('D');
|
||||
expect(getQualityGrade(69)).toBe('D');
|
||||
});
|
||||
|
||||
it('should return F for scores < 60', () => {
|
||||
expect(getQualityGrade(59)).toBe('F');
|
||||
expect(getQualityGrade(0)).toBe('F');
|
||||
});
|
||||
});
|
||||
|
||||
describe('isHighQuality', () => {
|
||||
it('should return true for high scoring skills with few warnings', () => {
|
||||
const score = {
|
||||
overall: 80,
|
||||
structure: { score: 80, hasMetadata: true, hasDescription: true, hasTriggers: true, hasExamples: true, hasBoundaries: true, hasWhenToUse: true },
|
||||
clarity: { score: 80, lineCount: 100, tokenCount: 500, avgSentenceLength: 15, hasHeaders: true },
|
||||
specificity: { score: 80, hasConcreteCommands: true, hasFilePatterns: true, hasCodeExamples: true, vagueTermCount: 0 },
|
||||
warnings: [],
|
||||
suggestions: [],
|
||||
};
|
||||
expect(isHighQuality(score)).toBe(true);
|
||||
});
|
||||
|
||||
it('should return false for low scoring skills', () => {
|
||||
const score = {
|
||||
overall: 50,
|
||||
structure: { score: 50, hasMetadata: false, hasDescription: false, hasTriggers: false, hasExamples: false, hasBoundaries: false, hasWhenToUse: false },
|
||||
clarity: { score: 50, lineCount: 100, tokenCount: 500, avgSentenceLength: 15, hasHeaders: true },
|
||||
specificity: { score: 50, hasConcreteCommands: false, hasFilePatterns: false, hasCodeExamples: false, vagueTermCount: 5 },
|
||||
warnings: ['Missing triggers', 'No examples'],
|
||||
suggestions: [],
|
||||
};
|
||||
expect(isHighQuality(score)).toBe(false);
|
||||
});
|
||||
|
||||
it('should return false for skills with many warnings', () => {
|
||||
const score = {
|
||||
overall: 75,
|
||||
structure: { score: 75, hasMetadata: true, hasDescription: true, hasTriggers: false, hasExamples: false, hasBoundaries: false, hasWhenToUse: false },
|
||||
clarity: { score: 75, lineCount: 100, tokenCount: 500, avgSentenceLength: 15, hasHeaders: true },
|
||||
specificity: { score: 75, hasConcreteCommands: true, hasFilePatterns: true, hasCodeExamples: false, vagueTermCount: 0 },
|
||||
warnings: ['Warning 1', 'Warning 2', 'Warning 3'],
|
||||
suggestions: [],
|
||||
};
|
||||
expect(isHighQuality(score)).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,373 @@
|
||||
import { readFileSync, existsSync } from 'node:fs';
|
||||
import { join, basename } from 'node:path';
|
||||
|
||||
export interface QualityScore {
|
||||
overall: number;
|
||||
structure: StructureScore;
|
||||
clarity: ClarityScore;
|
||||
specificity: SpecificityScore;
|
||||
warnings: string[];
|
||||
suggestions: string[];
|
||||
}
|
||||
|
||||
export interface StructureScore {
|
||||
score: number;
|
||||
hasMetadata: boolean;
|
||||
hasDescription: boolean;
|
||||
hasTriggers: boolean;
|
||||
hasExamples: boolean;
|
||||
hasBoundaries: boolean;
|
||||
hasWhenToUse: boolean;
|
||||
}
|
||||
|
||||
export interface ClarityScore {
|
||||
score: number;
|
||||
lineCount: number;
|
||||
tokenCount: number;
|
||||
avgSentenceLength: number;
|
||||
hasHeaders: boolean;
|
||||
}
|
||||
|
||||
export interface SpecificityScore {
|
||||
score: number;
|
||||
hasConcreteCommands: boolean;
|
||||
hasFilePatterns: boolean;
|
||||
hasCodeExamples: boolean;
|
||||
vagueTermCount: number;
|
||||
}
|
||||
|
||||
const VAGUE_TERMS = [
|
||||
'be helpful',
|
||||
'assist the user',
|
||||
'help with',
|
||||
'try to',
|
||||
'attempt to',
|
||||
'do your best',
|
||||
'as needed',
|
||||
'when appropriate',
|
||||
'if necessary',
|
||||
'general purpose',
|
||||
'various tasks',
|
||||
'many things',
|
||||
'etc.',
|
||||
'and so on',
|
||||
'stuff like that',
|
||||
];
|
||||
|
||||
const BOUNDARY_PATTERNS = [
|
||||
/never\s+(?:do|use|commit|push|delete|remove)/i,
|
||||
/always\s+(?:do|use|check|verify|ensure)/i,
|
||||
/don'?t\s+(?:do|use|commit|push|delete)/i,
|
||||
/avoid\s+(?:using|doing|committing)/i,
|
||||
/must\s+(?:not|always|never)/i,
|
||||
/forbidden/i,
|
||||
/prohibited/i,
|
||||
/required/i,
|
||||
];
|
||||
|
||||
const TRIGGER_PATTERNS = [
|
||||
/when\s+to\s+use/i,
|
||||
/use\s+this\s+(?:skill|when)/i,
|
||||
/triggers?\s*(?:when|:)/i,
|
||||
/activated?\s+(?:when|by)/i,
|
||||
/applies?\s+(?:when|to)/i,
|
||||
/invoke\s+(?:when|this)/i,
|
||||
];
|
||||
|
||||
const COMMAND_PATTERNS = [
|
||||
/```(?:bash|sh|shell|zsh)[\s\S]*?```/g,
|
||||
/`(?:npm|pnpm|yarn|bun|npx|git|docker|kubectl)\s+[^`]+`/g,
|
||||
/\$\s*\w+/g,
|
||||
];
|
||||
|
||||
const CODE_EXAMPLE_PATTERNS = [
|
||||
/```(?:typescript|javascript|tsx|jsx|python|go|rust|java)[\s\S]*?```/g,
|
||||
/```[\s\S]{50,}?```/g,
|
||||
];
|
||||
|
||||
const FILE_PATTERN_PATTERNS = [
|
||||
/\*\*\/\*\.\w+/,
|
||||
/\.\w+$/m,
|
||||
/glob[s]?\s*[:=]/i,
|
||||
/include[s]?\s*[:=]/i,
|
||||
/pattern[s]?\s*[:=]/i,
|
||||
];
|
||||
|
||||
function extractFrontmatter(content: string): Record<string, unknown> | null {
|
||||
const normalizedContent = content.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
||||
const match = normalizedContent.match(/^---\s*\n([\s\S]*?)\n---/);
|
||||
if (!match) return null;
|
||||
|
||||
const frontmatter: Record<string, unknown> = {};
|
||||
const lines = match[1].split('\n');
|
||||
|
||||
for (const line of lines) {
|
||||
const colonIndex = line.indexOf(':');
|
||||
if (colonIndex > 0) {
|
||||
const key = line.slice(0, colonIndex).trim();
|
||||
const value = line.slice(colonIndex + 1).trim();
|
||||
frontmatter[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
return Object.keys(frontmatter).length > 0 ? frontmatter : null;
|
||||
}
|
||||
|
||||
function countTokens(text: string): number {
|
||||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
|
||||
function countVagueTerms(content: string): number {
|
||||
const lowerContent = content.toLowerCase();
|
||||
let count = 0;
|
||||
for (const term of VAGUE_TERMS) {
|
||||
const regex = new RegExp(term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'gi');
|
||||
const matches = lowerContent.match(regex);
|
||||
if (matches) count += matches.length;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
function hasPattern(content: string, patterns: RegExp[]): boolean {
|
||||
return patterns.some(p => p.test(content));
|
||||
}
|
||||
|
||||
function countMatches(content: string, patterns: RegExp[]): number {
|
||||
let count = 0;
|
||||
for (const pattern of patterns) {
|
||||
const matches = content.match(pattern);
|
||||
if (matches) count += matches.length;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
function calculateAvgSentenceLength(content: string): number {
|
||||
const contentWithoutCode = content.replace(/```[\s\S]*?```/g, '');
|
||||
const sentences = contentWithoutCode.split(/[.!?]+/).filter(s => s.trim().length > 0);
|
||||
if (sentences.length === 0) return 0;
|
||||
const totalWords = sentences.reduce((sum, s) => sum + s.trim().split(/\s+/).length, 0);
|
||||
return Math.round(totalWords / sentences.length);
|
||||
}
|
||||
|
||||
function evaluateStructure(content: string): StructureScore {
|
||||
const frontmatter = extractFrontmatter(content);
|
||||
const hasMetadata = frontmatter !== null && Object.keys(frontmatter).length > 0;
|
||||
const hasDescription = !!(frontmatter?.description) || /^#+\s*description/im.test(content);
|
||||
const hasTriggers = hasPattern(content, TRIGGER_PATTERNS);
|
||||
const hasExamples = countMatches(content, CODE_EXAMPLE_PATTERNS) > 0;
|
||||
const hasBoundaries = hasPattern(content, BOUNDARY_PATTERNS);
|
||||
const hasWhenToUse = /when\s+to\s+use|use\s+case|scenario/i.test(content);
|
||||
|
||||
let score = 0;
|
||||
if (hasMetadata) score += 15;
|
||||
if (hasDescription) score += 10;
|
||||
if (hasTriggers) score += 20;
|
||||
if (hasExamples) score += 20;
|
||||
if (hasBoundaries) score += 20;
|
||||
if (hasWhenToUse) score += 15;
|
||||
|
||||
return {
|
||||
score: Math.min(100, score),
|
||||
hasMetadata,
|
||||
hasDescription,
|
||||
hasTriggers,
|
||||
hasExamples,
|
||||
hasBoundaries,
|
||||
hasWhenToUse,
|
||||
};
|
||||
}
|
||||
|
||||
function evaluateClarity(content: string): ClarityScore {
|
||||
const lines = content.split('\n');
|
||||
const lineCount = lines.length;
|
||||
const tokenCount = countTokens(content);
|
||||
const avgSentenceLength = calculateAvgSentenceLength(content);
|
||||
const hasHeaders = /^#+\s+/m.test(content);
|
||||
|
||||
let score = 100;
|
||||
|
||||
if (lineCount > 500) score -= 30;
|
||||
else if (lineCount > 300) score -= 15;
|
||||
else if (lineCount > 150) score -= 5;
|
||||
|
||||
if (tokenCount > 4000) score -= 30;
|
||||
else if (tokenCount > 2000) score -= 15;
|
||||
else if (tokenCount > 1000) score -= 5;
|
||||
|
||||
if (avgSentenceLength > 30) score -= 20;
|
||||
else if (avgSentenceLength > 20) score -= 10;
|
||||
|
||||
if (!hasHeaders) score -= 15;
|
||||
|
||||
return {
|
||||
score: Math.max(0, score),
|
||||
lineCount,
|
||||
tokenCount,
|
||||
avgSentenceLength,
|
||||
hasHeaders,
|
||||
};
|
||||
}
|
||||
|
||||
function countCodeBlocks(content: string): number {
|
||||
const fencedBlocks = content.match(/```[\s\S]*?```/g) || [];
|
||||
return fencedBlocks.length;
|
||||
}
|
||||
|
||||
function evaluateSpecificity(content: string): SpecificityScore {
|
||||
const hasConcreteCommands = countMatches(content, COMMAND_PATTERNS) > 0;
|
||||
const hasFilePatterns = hasPattern(content, FILE_PATTERN_PATTERNS);
|
||||
const codeBlockCount = countCodeBlocks(content);
|
||||
const hasCodeExamples = codeBlockCount >= 2;
|
||||
const vagueTermCount = countVagueTerms(content);
|
||||
|
||||
let score = 0;
|
||||
|
||||
if (hasConcreteCommands) score += 30;
|
||||
if (hasFilePatterns) score += 25;
|
||||
if (hasCodeExamples) score += 30;
|
||||
|
||||
if (vagueTermCount === 0) score += 15;
|
||||
else if (vagueTermCount <= 2) score += 10;
|
||||
else if (vagueTermCount <= 5) score += 5;
|
||||
|
||||
return {
|
||||
score: Math.min(100, score),
|
||||
hasConcreteCommands,
|
||||
hasFilePatterns,
|
||||
hasCodeExamples,
|
||||
vagueTermCount,
|
||||
};
|
||||
}
|
||||
|
||||
function generateWarnings(
|
||||
structure: StructureScore,
|
||||
clarity: ClarityScore,
|
||||
specificity: SpecificityScore
|
||||
): string[] {
|
||||
const warnings: string[] = [];
|
||||
|
||||
if (!structure.hasMetadata) {
|
||||
warnings.push('Missing frontmatter metadata');
|
||||
}
|
||||
if (!structure.hasTriggers) {
|
||||
warnings.push('No trigger conditions defined');
|
||||
}
|
||||
if (!structure.hasBoundaries) {
|
||||
warnings.push('No boundaries or constraints specified');
|
||||
}
|
||||
if (clarity.lineCount > 500) {
|
||||
warnings.push(`Skill is too long (${clarity.lineCount} lines)`);
|
||||
}
|
||||
if (clarity.tokenCount > 4000) {
|
||||
warnings.push(`High token usage (${clarity.tokenCount} tokens)`);
|
||||
}
|
||||
if (specificity.vagueTermCount > 5) {
|
||||
warnings.push(`Contains ${specificity.vagueTermCount} vague terms`);
|
||||
}
|
||||
if (!structure.hasExamples) {
|
||||
warnings.push('No code examples provided');
|
||||
} else if (!specificity.hasCodeExamples) {
|
||||
warnings.push('Only one code example provided');
|
||||
}
|
||||
|
||||
return warnings;
|
||||
}
|
||||
|
||||
function generateSuggestions(
|
||||
structure: StructureScore,
|
||||
clarity: ClarityScore,
|
||||
specificity: SpecificityScore
|
||||
): string[] {
|
||||
const suggestions: string[] = [];
|
||||
|
||||
if (!structure.hasMetadata) {
|
||||
suggestions.push('Add YAML frontmatter with name, description, and globs');
|
||||
}
|
||||
if (!structure.hasTriggers) {
|
||||
suggestions.push('Add a "When to Use" section with specific trigger conditions');
|
||||
}
|
||||
if (!structure.hasExamples) {
|
||||
suggestions.push('Include code examples showing expected output');
|
||||
}
|
||||
if (!structure.hasBoundaries) {
|
||||
suggestions.push('Define boundaries: what the agent should never do');
|
||||
}
|
||||
if (clarity.lineCount > 300) {
|
||||
suggestions.push('Consider splitting into multiple focused skills');
|
||||
}
|
||||
if (!clarity.hasHeaders) {
|
||||
suggestions.push('Use markdown headers to organize content');
|
||||
}
|
||||
if (specificity.vagueTermCount > 0) {
|
||||
suggestions.push('Replace vague terms with specific instructions');
|
||||
}
|
||||
if (!specificity.hasConcreteCommands) {
|
||||
suggestions.push('Add concrete executable commands with flags');
|
||||
}
|
||||
|
||||
return suggestions;
|
||||
}
|
||||
|
||||
export function evaluateSkillContent(content: string): QualityScore {
|
||||
const structure = evaluateStructure(content);
|
||||
const clarity = evaluateClarity(content);
|
||||
const specificity = evaluateSpecificity(content);
|
||||
|
||||
const overall = Math.round(
|
||||
structure.score * 0.4 +
|
||||
clarity.score * 0.3 +
|
||||
specificity.score * 0.3
|
||||
);
|
||||
|
||||
const warnings = generateWarnings(structure, clarity, specificity);
|
||||
const suggestions = generateSuggestions(structure, clarity, specificity);
|
||||
|
||||
return {
|
||||
overall,
|
||||
structure,
|
||||
clarity,
|
||||
specificity,
|
||||
warnings,
|
||||
suggestions,
|
||||
};
|
||||
}
|
||||
|
||||
export function evaluateSkillFile(filePath: string): QualityScore | null {
|
||||
if (!existsSync(filePath)) return null;
|
||||
|
||||
try {
|
||||
const content = readFileSync(filePath, 'utf-8');
|
||||
return evaluateSkillContent(content);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function evaluateSkillDirectory(dirPath: string): QualityScore | null {
|
||||
const skillMdPath = join(dirPath, 'SKILL.md');
|
||||
if (existsSync(skillMdPath)) {
|
||||
return evaluateSkillFile(skillMdPath);
|
||||
}
|
||||
|
||||
const mdcFiles = ['index.mdc', `${basename(dirPath)}.mdc`];
|
||||
for (const file of mdcFiles) {
|
||||
const mdcPath = join(dirPath, file);
|
||||
if (existsSync(mdcPath)) {
|
||||
return evaluateSkillFile(mdcPath);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function getQualityGrade(score: number): string {
|
||||
if (score >= 90) return 'A';
|
||||
if (score >= 80) return 'B';
|
||||
if (score >= 70) return 'C';
|
||||
if (score >= 60) return 'D';
|
||||
return 'F';
|
||||
}
|
||||
|
||||
export function isHighQuality(score: QualityScore): boolean {
|
||||
return score.overall >= 80 && score.warnings.length <= 2;
|
||||
}
|
||||
Reference in New Issue
Block a user