mirror of
https://github.com/rohitg00/skillkit.git
synced 2026-11-20 12:23:29 +00:00
* feat: add session memory system for cross-agent learning
Implements a lightweight, local-first memory system that works across
all 17 AI coding agents. Key features:
- YAML-based storage (no databases or external services)
- Three memory layers: Session, Project, and Global
- AI-powered compression of observations into learnings
- Cross-agent memory injection with relevance matching
- Progressive disclosure for token efficiency
Core modules:
- ObservationStore: Capture raw session observations
- LearningStore: Persist compressed learnings
- MemoryCompressor: AI/rule-based compression engine
- MemoryInjector: Context-aware memory retrieval
- MemoryObserver: Automatic observation capture
CLI commands (skillkit memory):
- status, search, list, show, compress, export
- import, clear, add, rate, config
TUI integration:
- New Memory screen with tabs (Learnings/Observations/Search)
- useMemory hook for state management
- Updated navigation and keybindings
Code quality improvements:
- Replaced nested ternaries with explicit functions
- Added named constants for magic numbers
- Simplified navigation with lookup map
* feat: add marketplace, testing, CI/CD, workflows, and fix dry-run bug
New Features:
- Skill Marketplace: Aggregates skills from GitHub repos with caching
- Skill Testing Framework: 18+ assertion types for skill validation
- CI/CD Integration: Templates for GitHub Actions, GitLab CI, CircleCI
- Workflow Orchestration: Wave-based parallel skill execution
- Session Management: Pause/resume support with state persistence
- Agent Execution: CLI spawning for claude/codex/gemini/opencode/goose
New CLI Commands:
- skillkit marketplace - Browse and install from marketplace
- skillkit test - Run skill tests with assertions
- skillkit run - Execute skills with orchestration
- skillkit status - Show session state
- skillkit pause/resume - Session management
- skillkit workflow - Workflow management
New TUI Screens:
- Marketplace browser with search
- History viewer with session details
- Workflow management screen
Bug Fix:
- Fixed dry-run mode in memory compress command (Devin review)
The --dry-run flag now properly prevents storing learnings
* fix: add missing yaml dependency to @skillkit/cli
* chore: update pnpm-lock.yaml
* fix: address 7 bugs identified by Devin code review
- Fix MemoryIndexStore ignoring isGlobal flag (memory-index.ts)
- Fix ESM runtime crash using require('yaml') in test runner (runner.ts)
- Fix ESM runtime crash using require('node:fs') in MarketplaceAggregator (aggregator.ts)
- Fix ESM runtime crash using require('node:fs') in SessionManager (manager.ts)
- Fix memory rate command parsing rating from --content instead of positional arg (memory.ts)
- Fix SkillExecutionEngine emitting failed status for paused executions (engine.ts)
- Fix run command misclassifying directories as skill markdown files (run.ts)
* fix: address CodeRabbit/Devin review issues (batch 2)
- Fix resume skipping checkpoint handling in engine.ts
- Add paused status guard before completeExecution call
- Fix NaN% progress when totalSteps=0 in Execute.tsx
- Fix effectiveness ?? instead of || to preserve 0% values in compressor.ts
- Clear cache on source change in aggregator.ts addSource/removeSource
* fix: address critical and major CodeRabbit/Devin issues (batch 3)
Security fixes:
- Remove shell: true from spawn to prevent shell injection (agents.ts)
- Add safe regex test with ReDoS protection (engine.ts)
- Add security warning for verification commands (engine.ts)
Bug fixes:
- Fix global memory index path to use ~/.skillkit/ (compressor.ts, memory-index.ts)
- Add task_complete events and verification to resumeExecution (engine.ts)
- Handle absolute URLs in skill entries (aggregator.ts)
- Implement actual port availability check (runner.ts)
- Remove invalid JUnit report format (templates.ts)
- Add crash guard to workflow list command (workflow/list.ts)
* fix: address remaining CodeRabbit/Devin issues (batch 4)
- ReDoS protection: Add pattern validator to reject dangerous regex patterns
(nested quantifiers, overlapping alternations, excessive quantifiers)
- Add missing 'cancelled' status message in execution engine
- Make completedAt/durationMs optional in TaskExecutionResult
- Fix tag filter to skip untagged tests when tags are specified
- Improve error-solution matching: require keyword similarity + solution indicator
- Validate --limit and sanitize --tags in memory/marketplace CLI commands
- Guard against missing arrays when importing YAML learnings
- Escape YAML frontmatter values to prevent invalid output
- Preserve original learning source when merging learnings
- Fix keyword double-counting in memory index search
- Normalize keywords by stripping punctuation for better matching
289 lines
7.9 KiB
TypeScript
289 lines
7.9 KiB
TypeScript
import { Command, Option } from 'clipanion';
|
|
import { resolve, join } from 'node:path';
|
|
import { existsSync, readFileSync, readdirSync } from 'node:fs';
|
|
import chalk from 'chalk';
|
|
import {
|
|
runTestSuite,
|
|
createTestSuiteFromFrontmatter,
|
|
type SkillTestSuite,
|
|
type TestSuiteResult,
|
|
} from '@skillkit/core';
|
|
import { parse as parseYaml } from 'yaml';
|
|
|
|
/**
|
|
* Test command - run skill tests
|
|
*/
|
|
export class TestCommand extends Command {
|
|
static override paths = [['test']];
|
|
|
|
static override usage = Command.Usage({
|
|
description: 'Run skill tests',
|
|
details: `
|
|
The test command runs test cases defined in skill frontmatter.
|
|
|
|
Tests are defined in the skill's YAML frontmatter using the 'tests' key.
|
|
|
|
Example skill with tests:
|
|
\`\`\`yaml
|
|
---
|
|
name: my-skill
|
|
tests:
|
|
- name: "Creates config file"
|
|
assertions:
|
|
- type: file_exists
|
|
target: config.json
|
|
- name: "Types check"
|
|
assertions:
|
|
- type: type_check
|
|
---
|
|
\`\`\`
|
|
`,
|
|
examples: [
|
|
['Run all skill tests', '$0 test'],
|
|
['Run tests for a specific skill', '$0 test my-skill'],
|
|
['Run with verbose output', '$0 test --verbose'],
|
|
['Stop on first failure', '$0 test --bail'],
|
|
['Run tests with specific tags', '$0 test --tags unit,integration'],
|
|
],
|
|
});
|
|
|
|
skill = Option.String({ required: false });
|
|
|
|
verbose = Option.Boolean('--verbose,-v', false, {
|
|
description: 'Verbose output',
|
|
});
|
|
|
|
bail = Option.Boolean('--bail,-b', false, {
|
|
description: 'Stop on first failure',
|
|
});
|
|
|
|
tags = Option.String('--tags,-t', {
|
|
description: 'Only run tests with these tags (comma-separated)',
|
|
});
|
|
|
|
skipTags = Option.String('--skip-tags', {
|
|
description: 'Skip tests with these tags (comma-separated)',
|
|
});
|
|
|
|
json = Option.Boolean('--json,-j', false, {
|
|
description: 'Output results as JSON',
|
|
});
|
|
|
|
projectPath = Option.String('--path,-p', {
|
|
description: 'Project path',
|
|
});
|
|
|
|
timeout = Option.String('--timeout', {
|
|
description: 'Test timeout in milliseconds',
|
|
});
|
|
|
|
async execute(): Promise<number> {
|
|
const targetPath = resolve(this.projectPath || process.cwd());
|
|
|
|
// Find skills to test
|
|
const skillFiles = this.findSkillFiles(targetPath);
|
|
|
|
if (skillFiles.length === 0) {
|
|
if (!this.json) {
|
|
console.log(chalk.yellow('No skills found with tests.'));
|
|
console.log(chalk.dim('Add tests to your skills using YAML frontmatter.'));
|
|
} else {
|
|
console.log(JSON.stringify({ results: [], passed: true, total: 0 }));
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// Filter by skill name if provided
|
|
const filesToTest = this.skill
|
|
? skillFiles.filter((f) => f.name.includes(this.skill!))
|
|
: skillFiles;
|
|
|
|
if (filesToTest.length === 0) {
|
|
if (!this.json) {
|
|
console.log(chalk.yellow(`No skills found matching "${this.skill}"`));
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
if (!this.json) {
|
|
console.log(chalk.bold('Running skill tests...\n'));
|
|
}
|
|
|
|
// Parse tags
|
|
const tags = this.tags?.split(',').map((t) => t.trim());
|
|
const skipTags = this.skipTags?.split(',').map((t) => t.trim());
|
|
const timeout = this.timeout ? parseInt(this.timeout, 10) : undefined;
|
|
|
|
// Run tests for each skill
|
|
const results: TestSuiteResult[] = [];
|
|
let allPassed = true;
|
|
|
|
for (const file of filesToTest) {
|
|
const suite = this.parseSkillTests(file.path, file.name);
|
|
|
|
if (!suite || suite.tests.length === 0) {
|
|
continue;
|
|
}
|
|
|
|
if (!this.json) {
|
|
console.log(chalk.blue(`Testing: ${suite.skillName}`));
|
|
}
|
|
|
|
const result = await runTestSuite(suite, {
|
|
cwd: targetPath,
|
|
verbose: this.verbose,
|
|
bail: this.bail,
|
|
tags,
|
|
skipTags,
|
|
timeout,
|
|
onProgress: (event) => {
|
|
if (this.json || !this.verbose) return;
|
|
|
|
switch (event.type) {
|
|
case 'test_start':
|
|
console.log(chalk.dim(` Running: ${event.testName}`));
|
|
break;
|
|
case 'test_end':
|
|
if (event.passed) {
|
|
console.log(chalk.green(` ✓ ${event.testName}`));
|
|
} else {
|
|
console.log(chalk.red(` ✗ ${event.testName}`));
|
|
if (event.error) {
|
|
console.log(chalk.red(` ${event.error}`));
|
|
}
|
|
}
|
|
break;
|
|
case 'assertion_end':
|
|
if (this.verbose && !event.passed) {
|
|
console.log(chalk.red(` - ${event.assertionType}: ${event.error}`));
|
|
}
|
|
break;
|
|
}
|
|
},
|
|
});
|
|
|
|
results.push(result);
|
|
|
|
if (!result.passed) {
|
|
allPassed = false;
|
|
}
|
|
|
|
if (!this.json) {
|
|
const icon = result.passed ? chalk.green('✓') : chalk.red('✗');
|
|
const status = result.passed ? 'PASSED' : 'FAILED';
|
|
console.log(
|
|
`${icon} ${suite.skillName}: ${result.passedCount}/${result.tests.length} tests ${status} (${result.duration}ms)\n`
|
|
);
|
|
}
|
|
|
|
if (this.bail && !result.passed) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Summary
|
|
if (this.json) {
|
|
console.log(
|
|
JSON.stringify({
|
|
results: results.map((r) => ({
|
|
skill: r.skillName,
|
|
passed: r.passed,
|
|
tests: r.passedCount,
|
|
failed: r.failedCount,
|
|
skipped: r.skippedCount,
|
|
duration: r.duration,
|
|
})),
|
|
passed: allPassed,
|
|
total: results.length,
|
|
})
|
|
);
|
|
} else {
|
|
const totalTests = results.reduce((acc, r) => acc + r.tests.length, 0);
|
|
const passedTests = results.reduce((acc, r) => acc + r.passedCount, 0);
|
|
const failedTests = results.reduce((acc, r) => acc + r.failedCount, 0);
|
|
const skippedTests = results.reduce((acc, r) => acc + r.skippedCount, 0);
|
|
|
|
console.log(chalk.bold('Summary:'));
|
|
console.log(` Skills tested: ${results.length}`);
|
|
console.log(` Total tests: ${totalTests}`);
|
|
console.log(chalk.green(` Passed: ${passedTests}`));
|
|
if (failedTests > 0) {
|
|
console.log(chalk.red(` Failed: ${failedTests}`));
|
|
}
|
|
if (skippedTests > 0) {
|
|
console.log(chalk.yellow(` Skipped: ${skippedTests}`));
|
|
}
|
|
|
|
if (allPassed) {
|
|
console.log(chalk.green('\n✓ All tests passed!'));
|
|
} else {
|
|
console.log(chalk.red('\n✗ Some tests failed.'));
|
|
}
|
|
}
|
|
|
|
return allPassed ? 0 : 1;
|
|
}
|
|
|
|
/**
|
|
* Find skill files with tests
|
|
*/
|
|
private findSkillFiles(projectPath: string): { name: string; path: string }[] {
|
|
const files: { name: string; path: string }[] = [];
|
|
|
|
// Check common skill directories
|
|
const skillDirs = [
|
|
'.claude/skills',
|
|
'.cursor/skills',
|
|
'.skillkit/skills',
|
|
'skills',
|
|
];
|
|
|
|
for (const dir of skillDirs) {
|
|
const fullDir = join(projectPath, dir);
|
|
if (!existsSync(fullDir)) continue;
|
|
|
|
try {
|
|
const entries = readdirSync(fullDir, { withFileTypes: true });
|
|
|
|
for (const entry of entries) {
|
|
if (entry.isFile() && (entry.name.endsWith('.md') || entry.name.endsWith('.mdc'))) {
|
|
files.push({
|
|
name: entry.name.replace(/\.(md|mdc)$/, ''),
|
|
path: join(fullDir, entry.name),
|
|
});
|
|
}
|
|
}
|
|
} catch {
|
|
// Directory not readable
|
|
}
|
|
}
|
|
|
|
return files;
|
|
}
|
|
|
|
/**
|
|
* Parse skill file for tests
|
|
*/
|
|
private parseSkillTests(filePath: string, skillName: string): SkillTestSuite | null {
|
|
try {
|
|
const content = readFileSync(filePath, 'utf-8');
|
|
|
|
// Extract frontmatter
|
|
const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---/);
|
|
if (!frontmatterMatch) {
|
|
return null;
|
|
}
|
|
|
|
const frontmatter = parseYaml(frontmatterMatch[1]) as Record<string, unknown>;
|
|
|
|
if (!frontmatter.tests) {
|
|
return null;
|
|
}
|
|
|
|
return createTestSuiteFromFrontmatter(skillName, frontmatter);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
}
|