Files
skillkit/scripts/compute-quality.ts
Rohit Ghumare b3f561dbc9 feat(quality): Quality Evaluation System Enhancements (#21)
* feat(quality): add skill quality evaluation system

- Add quality scoring module with structure, clarity, and specificity metrics
- Evaluate skills for metadata, triggers, boundaries, code examples
- Detect vague terms and penalize overly long skills
- Update validate command to include quality reports
- Add --verbose, --quality, --issues, --json flags
- Quality scores: A (90+), B (80+), C (70+), D (60+), F (<60)

* fix: address CodeRabbit and Devin review suggestions

- Fix JSON output to return exit code 1 on validation failures
- Add --min-score NaN validation with warning message
- Add directory guard for --all flag with .mdc file support
- Fix CRLF handling in frontmatter extraction
- Fix code blocks double-counting in specificity evaluation
- Fix warning message inconsistency for code examples
- Add try/catch error handling in evaluateSkillFile
- Fix isHighQuality threshold to match UI display (80)

* feat(quality): extend quality evaluation system with badges, deeper analysis, and fix command

- Phase 2: Add quality badges to CLI commands (list, recommend, install)
  - formatQualityBadge() helper for color-coded grade badges
  - --quality flag for list command
  - Low-quality skill warnings during install

- Phase 3: Add TUI quality badges
  - Extended SkillItem with quality, grade, warnings fields
  - Quality evaluation in loadSkills()
  - Grade badges in Installed and Recommend screens

- Phase 4: Deeper quality analysis
  - Deprecated pattern detection (CommonJS, class components, var)
  - Security pattern detection (hardcoded credentials, eval, innerHTML)
  - Conflicting instruction detection
  - Completeness assessment (TODOs, empty sections, example coverage)
  - New weight distribution: Structure 35%, Clarity 25%, Specificity 25%, Advanced 15%

- Phase 1: Pre-compute marketplace quality scores
  - Updated schema.json with quality field (0-100)
  - Created scripts/compute-quality.ts for batch computation

- Phase 5: Benchmark against category leaders
  - New benchmark.ts module with category detection
  - Percentile calculation and comparison notes
  - --benchmark flag for validate command

- Phase 6: Fix command for automated quality fixes
  - skillkit fix command with --dry-run and --interactive modes
  - Auto-generates frontmatter, "When to Use", "Boundaries" sections
  - --fix flag alias on validate command

* fix: address CodeRabbit and Devin review suggestions

- Fix .mdc file name extraction in fix.ts (handles both .md and .mdc)
- Add missing advanced field to test fixtures in quality.test.ts
- Fix duplicate header detection in assessCompleteness using regex.exec
- Fix ESM compatibility in compute-quality.ts using import.meta.url
- Add trailing newline to skills.json output

* fix: address remaining CodeRabbit review suggestions

- Escape regex metacharacters in vague term patterns (fix.ts)
- Improve diff algorithm using set-based comparison (fix.ts)
- Fix validateSkill receiving file path instead of directory (validate.ts)
- Check final section in completeness assessment (index.ts)
- Add error handling and empty skills guard (compute-quality.ts)

* fix: address additional review feedback

- validate.ts: Add error when --fix is passed without targets
- index.ts: Skip headers inside fenced code blocks during completeness check
- compute-quality.ts: Use nullish coalescing (??) to preserve zero quality scores
2026-01-28 14:18:42 +00:00

150 lines
3.3 KiB
JavaScript

#!/usr/bin/env node
/**
* Compute quality scores for marketplace skills
*
* This script assigns default quality scores to skills based on source verification.
* For detailed quality evaluation, skills would need to be fetched and analyzed individually.
*
* Usage:
* npx tsx scripts/compute-quality.ts
* pnpm tsx scripts/compute-quality.ts
*/
import { readFileSync, writeFileSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
interface Skill {
id: string;
name: string;
source: string;
description?: string;
tags: string[];
author?: string;
version?: string;
agents?: string[];
quality?: number;
}
interface SkillsIndex {
version: number;
updatedAt: string;
skills: Skill[];
}
const VERIFIED_SOURCES = [
'anthropics/',
'vercel/',
'vercel-labs/',
'modelcontextprotocol/',
'cursor/',
'google/',
'microsoft/',
'openai/',
];
const TRUSTED_AUTHORS = [
'anthropic',
'vercel',
'google',
'microsoft',
'openai',
];
function computeDefaultQuality(skill: Skill): number {
let score = 70;
for (const verified of VERIFIED_SOURCES) {
if (skill.source.startsWith(verified)) {
score = 85;
break;
}
}
if (skill.author) {
for (const trusted of TRUSTED_AUTHORS) {
if (skill.author.toLowerCase().includes(trusted)) {
score = Math.max(score, 85);
break;
}
}
}
if (skill.description && skill.description.length > 50) {
score += 3;
}
if (skill.tags && skill.tags.length >= 3) {
score += 2;
}
if (skill.version) {
score += 2;
}
if (skill.agents && skill.agents.length >= 2) {
score += 3;
}
return Math.min(100, score);
}
function main(): void {
const skillsPath = join(__dirname, '../marketplace/skills.json');
console.log('Reading skills.json...');
let data: SkillsIndex;
try {
const content = readFileSync(skillsPath, 'utf-8');
data = JSON.parse(content) as SkillsIndex;
} catch (err) {
console.error(`Failed to read or parse skills.json: ${err instanceof Error ? err.message : String(err)}`);
process.exit(1);
}
if (!data.skills || data.skills.length === 0) {
console.log('No skills found in skills.json');
return;
}
console.log(`Processing ${data.skills.length} skills...`);
let updated = 0;
let verified = 0;
for (const skill of data.skills) {
const quality = computeDefaultQuality(skill);
if (skill.quality !== quality) {
skill.quality = quality;
updated++;
}
if (quality >= 85) {
verified++;
}
}
data.updatedAt = new Date().toISOString().split('T')[0];
console.log(`Updated ${updated} skills with quality scores`);
console.log(`Verified sources: ${verified} skills (quality >= 85)`);
const averageQuality = Math.round(
data.skills.reduce((sum, s) => sum + (s.quality ?? 70), 0) / data.skills.length
);
console.log(`Average quality: ${averageQuality}`);
try {
writeFileSync(skillsPath, JSON.stringify(data, null, 2) + '\n');
console.log('Saved skills.json');
} catch (err) {
console.error(`Failed to write skills.json: ${err instanceof Error ? err.message : String(err)}`);
process.exit(1);
}
}
main();