diff --git a/packages/cli/src/commands/validate.ts b/packages/cli/src/commands/validate.ts index c4fc0dd8..6a9b1a8b 100644 --- a/packages/cli/src/commands/validate.ts +++ b/packages/cli/src/commands/validate.ts @@ -1,87 +1,349 @@ -import { existsSync, readdirSync } from 'node:fs'; -import { join, basename } from 'node:path'; -import chalk from 'chalk'; +import { existsSync, readdirSync, statSync } from 'node:fs'; +import { join, basename, resolve } from 'node:path'; import { Command, Option } from 'clipanion'; -import { validateSkill } from '@skillkit/core'; +import { + validateSkill, + evaluateSkillFile, + evaluateSkillDirectory, + getQualityGrade, + isHighQuality, + findAllSkills, + type QualityScore, +} from '@skillkit/core'; +import { getSearchDirs } from '../helpers.js'; +import { + colors, + symbols, + spinner, + success, + warn, + error, + header, +} from '../onboarding/index.js'; + +function formatScore(score: number): string { + if (score >= 80) return colors.success(`${score}`); + if (score >= 60) return colors.warning(`${score}`); + return colors.error(`${score}`); +} + +function formatGrade(grade: string): string { + switch (grade) { + case 'A': + case 'B': + return colors.success(grade); + case 'C': + case 'D': + return colors.warning(grade); + default: + return colors.error(grade); + } +} + +function printQualityReport(name: string, quality: QualityScore, verbose: boolean): void { + const grade = getQualityGrade(quality.overall); + const gradeDisplay = formatGrade(grade); + const scoreDisplay = formatScore(quality.overall); + + console.log(`\n${colors.primary(name)}`); + console.log(` Grade: ${gradeDisplay} Score: ${scoreDisplay}/100`); + + if (verbose) { + console.log(''); + console.log(` ${colors.muted('Structure:')} ${formatScore(quality.structure.score)}/100`); + console.log(` ${quality.structure.hasMetadata ? colors.success(symbols.success) : colors.error(symbols.error)} Metadata`); + console.log(` ${quality.structure.hasDescription ? colors.success(symbols.success) : colors.error(symbols.error)} Description`); + console.log(` ${quality.structure.hasTriggers ? colors.success(symbols.success) : colors.error(symbols.error)} Trigger conditions`); + console.log(` ${quality.structure.hasExamples ? colors.success(symbols.success) : colors.error(symbols.error)} Code examples`); + console.log(` ${quality.structure.hasBoundaries ? colors.success(symbols.success) : colors.error(symbols.error)} Boundaries defined`); + + console.log(''); + console.log(` ${colors.muted('Clarity:')} ${formatScore(quality.clarity.score)}/100`); + console.log(` Lines: ${quality.clarity.lineCount}${quality.clarity.lineCount > 300 ? colors.warning(' (consider splitting)') : ''}`); + console.log(` Tokens: ${quality.clarity.tokenCount}${quality.clarity.tokenCount > 2000 ? colors.warning(' (high)') : ''}`); + console.log(` ${quality.clarity.hasHeaders ? colors.success(symbols.success) : colors.error(symbols.error)} Uses headers`); + + console.log(''); + console.log(` ${colors.muted('Specificity:')} ${formatScore(quality.specificity.score)}/100`); + console.log(` ${quality.specificity.hasConcreteCommands ? colors.success(symbols.success) : colors.error(symbols.error)} Concrete commands`); + console.log(` ${quality.specificity.hasFilePatterns ? colors.success(symbols.success) : colors.error(symbols.error)} File patterns`); + console.log(` ${quality.specificity.hasCodeExamples ? colors.success(symbols.success) : colors.error(symbols.error)} Multiple code examples`); + if (quality.specificity.vagueTermCount > 0) { + console.log(` ${colors.warning(symbols.warning)} ${quality.specificity.vagueTermCount} vague term(s)`); + } + } + + if (quality.warnings.length > 0) { + console.log(''); + console.log(` ${colors.warning('Warnings:')}`); + for (const w of quality.warnings) { + console.log(` ${colors.warning(symbols.warning)} ${w}`); + } + } + + if (quality.suggestions.length > 0 && verbose) { + console.log(''); + console.log(` ${colors.muted('Suggestions:')}`); + for (const s of quality.suggestions) { + console.log(` ${colors.muted(symbols.bullet)} ${s}`); + } + } +} export class ValidateCommand extends Command { - static override paths = [['validate'], ['v']]; + static override paths = [['validate']]; static override usage = Command.Usage({ - description: 'Validate skill(s) against the Agent Skills specification (agentskills.io)', + description: 'Validate skill format and quality', + details: ` + Validates skills against the specification and evaluates quality. + Quality scores are based on structure, clarity, and specificity. + `, examples: [ - ['Validate a skill directory', '$0 validate ./my-skill'], - ['Validate all skills in a directory', '$0 validate ./skills --all'], + ['Validate all installed skills', '$0 validate'], + ['Validate specific skill', '$0 validate my-skill'], + ['Validate skill file', '$0 validate ./path/to/SKILL.md'], + ['Detailed quality report', '$0 validate --verbose'], + ['Only show issues', '$0 validate --issues'], + ['Quality check only', '$0 validate --quality'], + ['JSON output', '$0 validate --json'], ], }); - skillPath = Option.String({ required: true }); + targets = Option.Rest(); + + verbose = Option.Boolean('--verbose,-v', false, { + description: 'Show detailed quality breakdown', + }); + + quality = Option.Boolean('--quality,-q', false, { + description: 'Only check quality (skip format validation)', + }); + + issues = Option.Boolean('--issues,-i', false, { + description: 'Only show skills with issues', + }); + + minScore = Option.String('--min-score,-m', { + description: 'Minimum acceptable score (default: 60)', + }); + + json = Option.Boolean('--json', false, { + description: 'Output as JSON', + }); all = Option.Boolean('--all,-a', false, { - description: 'Validate all skills in the directory', + description: 'Validate all skills in directory', }); async execute(): Promise { - const targetPath = this.skillPath; - - if (!existsSync(targetPath)) { - console.error(chalk.red(`Path does not exist: ${targetPath}`)); - return 1; + let minScore = 60; + if (this.minScore) { + const parsed = parseInt(this.minScore, 10); + if (Number.isNaN(parsed) || parsed < 0 || parsed > 100) { + if (!this.json) { + warn(`Invalid --min-score value "${this.minScore}", using default: 60`); + } + } else { + minScore = parsed; + } } + const results: Array<{ name: string; quality: QualityScore | null; formatValid: boolean; formatErrors: string[]; path?: string }> = []; - const skillPaths: string[] = []; + if (this.targets.length > 0) { + for (const target of this.targets) { + const resolved = resolve(target); - if (this.all) { - const entries = readdirSync(targetPath, { withFileTypes: true }); - for (const entry of entries) { - if (entry.isDirectory()) { - const skillPath = join(targetPath, entry.name); - if (existsSync(join(skillPath, 'SKILL.md'))) { - skillPaths.push(skillPath); + if (existsSync(resolved)) { + const isDirectory = statSync(resolved).isDirectory(); + if (this.all) { + if (!isDirectory) { + if (!this.json) { + warn(`--all requires a directory, but "${target}" is a file`); + } + continue; + } + const entries = readdirSync(resolved, { withFileTypes: true }); + for (const entry of entries) { + if (entry.isDirectory()) { + const skillPath = join(resolved, entry.name); + const hasSkillMd = existsSync(join(skillPath, 'SKILL.md')); + const hasIndexMdc = existsSync(join(skillPath, 'index.mdc')); + const hasNamedMdc = existsSync(join(skillPath, `${entry.name}.mdc`)); + if (hasSkillMd || hasIndexMdc || hasNamedMdc) { + const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(skillPath); + const quality = evaluateSkillDirectory(skillPath); + results.push({ + name: entry.name, + quality, + formatValid: formatResult.valid, + formatErrors: formatResult.errors || [], + path: skillPath, + }); + } + } + } + } else { + const isFile = resolved.endsWith('.md') || resolved.endsWith('.mdc'); + const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(resolved); + const quality = isFile ? evaluateSkillFile(resolved) : evaluateSkillDirectory(resolved); + results.push({ + name: basename(target), + quality, + formatValid: formatResult.valid, + formatErrors: formatResult.errors || [], + path: resolved, + }); + } + } else { + const searchDirs = getSearchDirs(); + const skills = findAllSkills(searchDirs); + const found = skills.find(s => s.name === target); + + if (found) { + const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(found.path); + const quality = evaluateSkillDirectory(found.path); + results.push({ + name: found.name, + quality, + formatValid: formatResult.valid, + formatErrors: formatResult.errors || [], + path: found.path, + }); + } else { + if (!this.json) { + warn(`Skill not found: ${target}`); + } } } } - - if (skillPaths.length === 0) { - console.error(chalk.yellow('No skills found in directory')); - return 1; - } } else { - skillPaths.push(targetPath); - } + const searchDirs = getSearchDirs(); + const skills = findAllSkills(searchDirs); - let hasErrors = false; - - for (const skillPath of skillPaths) { - const skillName = basename(skillPath); - const result = validateSkill(skillPath); - - if (result.valid) { - console.log(chalk.green(`✓ ${skillName}`)); - - if (result.warnings && result.warnings.length > 0) { - result.warnings.forEach(w => { - console.log(chalk.yellow(` ⚠ ${w}`)); - }); + if (skills.length === 0) { + if (!this.json) { + warn('No installed skills found'); } - } else { - console.log(chalk.red(`✗ ${skillName}`)); - result.errors.forEach(e => { - console.log(chalk.red(` • ${e}`)); + return 0; + } + + const s = spinner(); + if (!this.json) { + s.start(`Evaluating ${skills.length} skill(s)...`); + } + + for (const skill of skills) { + const formatResult = this.quality ? { valid: true, errors: [] } : validateSkill(skill.path); + const quality = evaluateSkillDirectory(skill.path); + results.push({ + name: skill.name, + quality, + formatValid: formatResult.valid, + formatErrors: formatResult.errors || [], + path: skill.path, }); - hasErrors = true; + } + + if (!this.json) { + s.stop(`Evaluated ${results.length} skill(s)`); } } - console.log(); - - if (hasErrors) { - console.log(chalk.red('Validation failed')); - console.log(chalk.dim('See https://agentskills.io/specification for the complete format')); + if (results.length === 0) { + if (!this.json) { + error('No skills to validate'); + } return 1; } - console.log(chalk.green(`Validated ${skillPaths.length} skill(s) successfully`)); + if (this.json) { + const output = results.map(r => ({ + name: r.name, + path: r.path, + formatValid: r.formatValid, + formatErrors: r.formatErrors, + score: r.quality?.overall ?? null, + grade: r.quality ? getQualityGrade(r.quality.overall) : null, + highQuality: r.quality ? isHighQuality(r.quality) : false, + structure: r.quality?.structure ?? null, + clarity: r.quality?.clarity ?? null, + specificity: r.quality?.specificity ?? null, + warnings: r.quality?.warnings ?? [], + suggestions: r.quality?.suggestions ?? [], + })); + console.log(JSON.stringify(output, null, 2)); + const hasFormatFailures = results.some(r => !r.formatValid); + const hasBelowThreshold = results.some(r => r.quality && r.quality.overall < minScore); + return hasFormatFailures || hasBelowThreshold ? 1 : 0; + } + + header('Skill Validation Report'); + + let filteredResults = results; + if (this.issues) { + filteredResults = results.filter(r => + !r.formatValid || + (r.quality && r.quality.overall < minScore) || + (r.quality && r.quality.warnings.length > 0) + ); + } + + filteredResults.sort((a, b) => (b.quality?.overall ?? 0) - (a.quality?.overall ?? 0)); + + for (const { name, quality, formatValid, formatErrors } of filteredResults) { + if (!formatValid) { + console.log(`\n${colors.error(symbols.error)} ${colors.primary(name)}`); + console.log(` ${colors.error('Format validation failed:')}`); + for (const err of formatErrors) { + console.log(` ${colors.error(symbols.bullet)} ${err}`); + } + } + + if (quality) { + printQualityReport(name, quality, this.verbose); + } + } + + console.log(''); + + const formatFailed = results.filter(r => !r.formatValid).length; + const withQuality = results.filter(r => r.quality !== null); + const highQuality = withQuality.filter(r => r.quality!.overall >= 80).length; + const passing = withQuality.filter(r => r.quality!.overall >= minScore).length; + const failing = withQuality.filter(r => r.quality!.overall < minScore).length; + + console.log(colors.muted('Summary:')); + + if (!this.quality) { + console.log(` ${formatFailed === 0 ? colors.success(symbols.success) : colors.error(symbols.error)} Format valid: ${results.length - formatFailed}/${results.length}`); + } + + console.log(` ${colors.success(symbols.success)} High quality (80+): ${highQuality}`); + console.log(` ${colors.primary(symbols.bullet)} Passing (${minScore}+): ${passing}`); + + if (failing > 0) { + console.log(` ${colors.error(symbols.error)} Below threshold: ${failing}`); + } + + if (withQuality.length > 0) { + const avgScore = Math.round(withQuality.reduce((sum, r) => sum + r.quality!.overall, 0) / withQuality.length); + console.log(` ${colors.muted('Average score:')} ${formatScore(avgScore)}/100`); + } + + if (formatFailed > 0 || failing > 0) { + console.log(''); + if (formatFailed > 0) { + warn(`${formatFailed} skill(s) failed format validation`); + } + if (failing > 0) { + warn(`${failing} skill(s) below quality threshold`); + } + return 1; + } + + console.log(''); + success('All skills validated successfully'); return 0; } } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 9f82a9dd..d6f17901 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -75,3 +75,5 @@ export * from './agents/index.js'; export * from './skill-translator.js'; export * from './manifest/index.js'; + +export * from './quality/index.js'; diff --git a/packages/core/src/quality/__tests__/quality.test.ts b/packages/core/src/quality/__tests__/quality.test.ts new file mode 100644 index 00000000..a1c97d4d --- /dev/null +++ b/packages/core/src/quality/__tests__/quality.test.ts @@ -0,0 +1,203 @@ +import { describe, it, expect } from 'vitest'; +import { + evaluateSkillContent, + getQualityGrade, + isHighQuality, +} from '../index.js'; + +describe('Quality Evaluation', () => { + describe('evaluateSkillContent', () => { + it('should score a well-structured skill highly', () => { + const content = `--- +name: test-skill +description: A test skill for validation +globs: ["**/*.ts"] +--- + +# Test Skill + +## When to Use + +Use this skill when working with TypeScript files. + +## Instructions + +Always follow these patterns: + +\`\`\`typescript +function example(): void { + console.log('example'); +} +\`\`\` + +## Commands + +Run tests with: + +\`\`\`bash +npm test +\`\`\` + +## Boundaries + +Never commit secrets to the repository. +Always run linting before commits. +`; + + const result = evaluateSkillContent(content); + expect(result.overall).toBeGreaterThanOrEqual(70); + expect(result.structure.hasMetadata).toBe(true); + expect(result.structure.hasTriggers).toBe(true); + expect(result.structure.hasExamples).toBe(true); + expect(result.structure.hasBoundaries).toBe(true); + expect(result.clarity.hasHeaders).toBe(true); + expect(result.specificity.hasConcreteCommands).toBe(true); + }); + + it('should score a minimal skill lower', () => { + const content = `# My Skill + +Be helpful and assist the user with various tasks. +Try to do your best. +`; + + const result = evaluateSkillContent(content); + expect(result.overall).toBeLessThan(50); + expect(result.structure.hasMetadata).toBe(false); + expect(result.structure.hasTriggers).toBe(false); + expect(result.specificity.vagueTermCount).toBeGreaterThan(0); + expect(result.warnings.length).toBeGreaterThan(0); + }); + + it('should penalize overly long skills', () => { + const longContent = '# Skill\n' + 'Some content.\n'.repeat(600); + const result = evaluateSkillContent(longContent); + expect(result.clarity.lineCount).toBeGreaterThan(500); + expect(result.clarity.score).toBeLessThan(70); + }); + + it('should detect vague terms', () => { + const content = `# Skill + +Be helpful and assist the user. +Try to do your best when appropriate. +Help with various tasks as needed. +`; + + const result = evaluateSkillContent(content); + expect(result.specificity.vagueTermCount).toBeGreaterThan(3); + }); + + it('should detect code examples', () => { + const content = `# Skill + +\`\`\`typescript +const x = 1; +\`\`\` + +\`\`\`javascript +const y = 2; +\`\`\` +`; + + const result = evaluateSkillContent(content); + expect(result.structure.hasExamples).toBe(true); + expect(result.specificity.hasCodeExamples).toBe(true); + }); + + it('should detect boundary patterns', () => { + const content = `# Skill + +## Boundaries + +- Never commit secrets +- Always run tests +- Don't modify node_modules +- Avoid using deprecated APIs +`; + + const result = evaluateSkillContent(content); + expect(result.structure.hasBoundaries).toBe(true); + }); + + it('should detect trigger conditions', () => { + const content = `# Skill + +## When to Use + +Use this skill when: +- Working with React components +- Building forms +`; + + const result = evaluateSkillContent(content); + expect(result.structure.hasTriggers).toBe(true); + expect(result.structure.hasWhenToUse).toBe(true); + }); + }); + + describe('getQualityGrade', () => { + it('should return A for scores >= 90', () => { + expect(getQualityGrade(90)).toBe('A'); + expect(getQualityGrade(100)).toBe('A'); + }); + + it('should return B for scores >= 80', () => { + expect(getQualityGrade(80)).toBe('B'); + expect(getQualityGrade(89)).toBe('B'); + }); + + it('should return C for scores >= 70', () => { + expect(getQualityGrade(70)).toBe('C'); + expect(getQualityGrade(79)).toBe('C'); + }); + + it('should return D for scores >= 60', () => { + expect(getQualityGrade(60)).toBe('D'); + expect(getQualityGrade(69)).toBe('D'); + }); + + it('should return F for scores < 60', () => { + expect(getQualityGrade(59)).toBe('F'); + expect(getQualityGrade(0)).toBe('F'); + }); + }); + + describe('isHighQuality', () => { + it('should return true for high scoring skills with few warnings', () => { + const score = { + overall: 80, + structure: { score: 80, hasMetadata: true, hasDescription: true, hasTriggers: true, hasExamples: true, hasBoundaries: true, hasWhenToUse: true }, + clarity: { score: 80, lineCount: 100, tokenCount: 500, avgSentenceLength: 15, hasHeaders: true }, + specificity: { score: 80, hasConcreteCommands: true, hasFilePatterns: true, hasCodeExamples: true, vagueTermCount: 0 }, + warnings: [], + suggestions: [], + }; + expect(isHighQuality(score)).toBe(true); + }); + + it('should return false for low scoring skills', () => { + const score = { + overall: 50, + structure: { score: 50, hasMetadata: false, hasDescription: false, hasTriggers: false, hasExamples: false, hasBoundaries: false, hasWhenToUse: false }, + clarity: { score: 50, lineCount: 100, tokenCount: 500, avgSentenceLength: 15, hasHeaders: true }, + specificity: { score: 50, hasConcreteCommands: false, hasFilePatterns: false, hasCodeExamples: false, vagueTermCount: 5 }, + warnings: ['Missing triggers', 'No examples'], + suggestions: [], + }; + expect(isHighQuality(score)).toBe(false); + }); + + it('should return false for skills with many warnings', () => { + const score = { + overall: 75, + structure: { score: 75, hasMetadata: true, hasDescription: true, hasTriggers: false, hasExamples: false, hasBoundaries: false, hasWhenToUse: false }, + clarity: { score: 75, lineCount: 100, tokenCount: 500, avgSentenceLength: 15, hasHeaders: true }, + specificity: { score: 75, hasConcreteCommands: true, hasFilePatterns: true, hasCodeExamples: false, vagueTermCount: 0 }, + warnings: ['Warning 1', 'Warning 2', 'Warning 3'], + suggestions: [], + }; + expect(isHighQuality(score)).toBe(false); + }); + }); +}); diff --git a/packages/core/src/quality/index.ts b/packages/core/src/quality/index.ts new file mode 100644 index 00000000..24274a99 --- /dev/null +++ b/packages/core/src/quality/index.ts @@ -0,0 +1,373 @@ +import { readFileSync, existsSync } from 'node:fs'; +import { join, basename } from 'node:path'; + +export interface QualityScore { + overall: number; + structure: StructureScore; + clarity: ClarityScore; + specificity: SpecificityScore; + warnings: string[]; + suggestions: string[]; +} + +export interface StructureScore { + score: number; + hasMetadata: boolean; + hasDescription: boolean; + hasTriggers: boolean; + hasExamples: boolean; + hasBoundaries: boolean; + hasWhenToUse: boolean; +} + +export interface ClarityScore { + score: number; + lineCount: number; + tokenCount: number; + avgSentenceLength: number; + hasHeaders: boolean; +} + +export interface SpecificityScore { + score: number; + hasConcreteCommands: boolean; + hasFilePatterns: boolean; + hasCodeExamples: boolean; + vagueTermCount: number; +} + +const VAGUE_TERMS = [ + 'be helpful', + 'assist the user', + 'help with', + 'try to', + 'attempt to', + 'do your best', + 'as needed', + 'when appropriate', + 'if necessary', + 'general purpose', + 'various tasks', + 'many things', + 'etc.', + 'and so on', + 'stuff like that', +]; + +const BOUNDARY_PATTERNS = [ + /never\s+(?:do|use|commit|push|delete|remove)/i, + /always\s+(?:do|use|check|verify|ensure)/i, + /don'?t\s+(?:do|use|commit|push|delete)/i, + /avoid\s+(?:using|doing|committing)/i, + /must\s+(?:not|always|never)/i, + /forbidden/i, + /prohibited/i, + /required/i, +]; + +const TRIGGER_PATTERNS = [ + /when\s+to\s+use/i, + /use\s+this\s+(?:skill|when)/i, + /triggers?\s*(?:when|:)/i, + /activated?\s+(?:when|by)/i, + /applies?\s+(?:when|to)/i, + /invoke\s+(?:when|this)/i, +]; + +const COMMAND_PATTERNS = [ + /```(?:bash|sh|shell|zsh)[\s\S]*?```/g, + /`(?:npm|pnpm|yarn|bun|npx|git|docker|kubectl)\s+[^`]+`/g, + /\$\s*\w+/g, +]; + +const CODE_EXAMPLE_PATTERNS = [ + /```(?:typescript|javascript|tsx|jsx|python|go|rust|java)[\s\S]*?```/g, + /```[\s\S]{50,}?```/g, +]; + +const FILE_PATTERN_PATTERNS = [ + /\*\*\/\*\.\w+/, + /\.\w+$/m, + /glob[s]?\s*[:=]/i, + /include[s]?\s*[:=]/i, + /pattern[s]?\s*[:=]/i, +]; + +function extractFrontmatter(content: string): Record | null { + const normalizedContent = content.replace(/\r\n/g, '\n').replace(/\r/g, '\n'); + const match = normalizedContent.match(/^---\s*\n([\s\S]*?)\n---/); + if (!match) return null; + + const frontmatter: Record = {}; + const lines = match[1].split('\n'); + + for (const line of lines) { + const colonIndex = line.indexOf(':'); + if (colonIndex > 0) { + const key = line.slice(0, colonIndex).trim(); + const value = line.slice(colonIndex + 1).trim(); + frontmatter[key] = value; + } + } + + return Object.keys(frontmatter).length > 0 ? frontmatter : null; +} + +function countTokens(text: string): number { + return Math.ceil(text.length / 4); +} + +function countVagueTerms(content: string): number { + const lowerContent = content.toLowerCase(); + let count = 0; + for (const term of VAGUE_TERMS) { + const regex = new RegExp(term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'gi'); + const matches = lowerContent.match(regex); + if (matches) count += matches.length; + } + return count; +} + +function hasPattern(content: string, patterns: RegExp[]): boolean { + return patterns.some(p => p.test(content)); +} + +function countMatches(content: string, patterns: RegExp[]): number { + let count = 0; + for (const pattern of patterns) { + const matches = content.match(pattern); + if (matches) count += matches.length; + } + return count; +} + +function calculateAvgSentenceLength(content: string): number { + const contentWithoutCode = content.replace(/```[\s\S]*?```/g, ''); + const sentences = contentWithoutCode.split(/[.!?]+/).filter(s => s.trim().length > 0); + if (sentences.length === 0) return 0; + const totalWords = sentences.reduce((sum, s) => sum + s.trim().split(/\s+/).length, 0); + return Math.round(totalWords / sentences.length); +} + +function evaluateStructure(content: string): StructureScore { + const frontmatter = extractFrontmatter(content); + const hasMetadata = frontmatter !== null && Object.keys(frontmatter).length > 0; + const hasDescription = !!(frontmatter?.description) || /^#+\s*description/im.test(content); + const hasTriggers = hasPattern(content, TRIGGER_PATTERNS); + const hasExamples = countMatches(content, CODE_EXAMPLE_PATTERNS) > 0; + const hasBoundaries = hasPattern(content, BOUNDARY_PATTERNS); + const hasWhenToUse = /when\s+to\s+use|use\s+case|scenario/i.test(content); + + let score = 0; + if (hasMetadata) score += 15; + if (hasDescription) score += 10; + if (hasTriggers) score += 20; + if (hasExamples) score += 20; + if (hasBoundaries) score += 20; + if (hasWhenToUse) score += 15; + + return { + score: Math.min(100, score), + hasMetadata, + hasDescription, + hasTriggers, + hasExamples, + hasBoundaries, + hasWhenToUse, + }; +} + +function evaluateClarity(content: string): ClarityScore { + const lines = content.split('\n'); + const lineCount = lines.length; + const tokenCount = countTokens(content); + const avgSentenceLength = calculateAvgSentenceLength(content); + const hasHeaders = /^#+\s+/m.test(content); + + let score = 100; + + if (lineCount > 500) score -= 30; + else if (lineCount > 300) score -= 15; + else if (lineCount > 150) score -= 5; + + if (tokenCount > 4000) score -= 30; + else if (tokenCount > 2000) score -= 15; + else if (tokenCount > 1000) score -= 5; + + if (avgSentenceLength > 30) score -= 20; + else if (avgSentenceLength > 20) score -= 10; + + if (!hasHeaders) score -= 15; + + return { + score: Math.max(0, score), + lineCount, + tokenCount, + avgSentenceLength, + hasHeaders, + }; +} + +function countCodeBlocks(content: string): number { + const fencedBlocks = content.match(/```[\s\S]*?```/g) || []; + return fencedBlocks.length; +} + +function evaluateSpecificity(content: string): SpecificityScore { + const hasConcreteCommands = countMatches(content, COMMAND_PATTERNS) > 0; + const hasFilePatterns = hasPattern(content, FILE_PATTERN_PATTERNS); + const codeBlockCount = countCodeBlocks(content); + const hasCodeExamples = codeBlockCount >= 2; + const vagueTermCount = countVagueTerms(content); + + let score = 0; + + if (hasConcreteCommands) score += 30; + if (hasFilePatterns) score += 25; + if (hasCodeExamples) score += 30; + + if (vagueTermCount === 0) score += 15; + else if (vagueTermCount <= 2) score += 10; + else if (vagueTermCount <= 5) score += 5; + + return { + score: Math.min(100, score), + hasConcreteCommands, + hasFilePatterns, + hasCodeExamples, + vagueTermCount, + }; +} + +function generateWarnings( + structure: StructureScore, + clarity: ClarityScore, + specificity: SpecificityScore +): string[] { + const warnings: string[] = []; + + if (!structure.hasMetadata) { + warnings.push('Missing frontmatter metadata'); + } + if (!structure.hasTriggers) { + warnings.push('No trigger conditions defined'); + } + if (!structure.hasBoundaries) { + warnings.push('No boundaries or constraints specified'); + } + if (clarity.lineCount > 500) { + warnings.push(`Skill is too long (${clarity.lineCount} lines)`); + } + if (clarity.tokenCount > 4000) { + warnings.push(`High token usage (${clarity.tokenCount} tokens)`); + } + if (specificity.vagueTermCount > 5) { + warnings.push(`Contains ${specificity.vagueTermCount} vague terms`); + } + if (!structure.hasExamples) { + warnings.push('No code examples provided'); + } else if (!specificity.hasCodeExamples) { + warnings.push('Only one code example provided'); + } + + return warnings; +} + +function generateSuggestions( + structure: StructureScore, + clarity: ClarityScore, + specificity: SpecificityScore +): string[] { + const suggestions: string[] = []; + + if (!structure.hasMetadata) { + suggestions.push('Add YAML frontmatter with name, description, and globs'); + } + if (!structure.hasTriggers) { + suggestions.push('Add a "When to Use" section with specific trigger conditions'); + } + if (!structure.hasExamples) { + suggestions.push('Include code examples showing expected output'); + } + if (!structure.hasBoundaries) { + suggestions.push('Define boundaries: what the agent should never do'); + } + if (clarity.lineCount > 300) { + suggestions.push('Consider splitting into multiple focused skills'); + } + if (!clarity.hasHeaders) { + suggestions.push('Use markdown headers to organize content'); + } + if (specificity.vagueTermCount > 0) { + suggestions.push('Replace vague terms with specific instructions'); + } + if (!specificity.hasConcreteCommands) { + suggestions.push('Add concrete executable commands with flags'); + } + + return suggestions; +} + +export function evaluateSkillContent(content: string): QualityScore { + const structure = evaluateStructure(content); + const clarity = evaluateClarity(content); + const specificity = evaluateSpecificity(content); + + const overall = Math.round( + structure.score * 0.4 + + clarity.score * 0.3 + + specificity.score * 0.3 + ); + + const warnings = generateWarnings(structure, clarity, specificity); + const suggestions = generateSuggestions(structure, clarity, specificity); + + return { + overall, + structure, + clarity, + specificity, + warnings, + suggestions, + }; +} + +export function evaluateSkillFile(filePath: string): QualityScore | null { + if (!existsSync(filePath)) return null; + + try { + const content = readFileSync(filePath, 'utf-8'); + return evaluateSkillContent(content); + } catch { + return null; + } +} + +export function evaluateSkillDirectory(dirPath: string): QualityScore | null { + const skillMdPath = join(dirPath, 'SKILL.md'); + if (existsSync(skillMdPath)) { + return evaluateSkillFile(skillMdPath); + } + + const mdcFiles = ['index.mdc', `${basename(dirPath)}.mdc`]; + for (const file of mdcFiles) { + const mdcPath = join(dirPath, file); + if (existsSync(mdcPath)) { + return evaluateSkillFile(mdcPath); + } + } + + return null; +} + +export function getQualityGrade(score: number): string { + if (score >= 90) return 'A'; + if (score >= 80) return 'B'; + if (score >= 70) return 'C'; + if (score >= 60) return 'D'; + return 'F'; +} + +export function isHighQuality(score: QualityScore): boolean { + return score.overall >= 80 && score.warnings.length <= 2; +}