Merge pull request #103 from rohitg00/feat/clone-progress-sparse

feat: live clone progress + sparse checkout for large repos
This commit is contained in:
Rohit Ghumare
2026-04-10 22:10:41 +01:00
committed by GitHub
11 changed files with 317 additions and 137 deletions
+40 -17
View File
@@ -4,6 +4,7 @@ import {
cpSync,
rmSync,
symlinkSync,
statSync,
readFileSync,
writeFileSync,
} from "node:fs";
@@ -133,7 +134,7 @@ export class InstallCommand extends Command {
const s = spinner();
try {
if (isInteractive && !this.quiet) {
if (process.stdin.isTTY && !this.quiet) {
welcome();
}
@@ -235,8 +236,13 @@ export class InstallCommand extends Command {
}
if (!result) {
s.start(`Fetching from ${providerAdapter.name}...`);
result = await providerAdapter.clone(this.source, "", { depth: 1 });
s.start(`Cloning from ${providerAdapter.name}...`);
result = await providerAdapter.clone(this.source, "", {
depth: 1,
onProgress: (msg) => {
s.message(`${providerAdapter!.name} — ${msg}`);
},
});
if (!result.success || !result.path) {
s.stop(colors.error(result.error || "Failed to fetch source"));
@@ -466,7 +472,18 @@ export class InstallCommand extends Command {
mkdirSync(installDir, { recursive: true });
}
const targetPath = join(installDir, skillName);
let isStandaloneFile: boolean;
try {
isStandaloneFile = statSync(sourcePath).isFile();
} catch {
if (!this.quiet) {
warn(`Skipping ${skillName} for ${adapter.name} (source not accessible)`);
}
continue;
}
const targetPath = isStandaloneFile
? join(installDir, skillName.endsWith(".md") ? skillName : `${skillName}.md`)
: join(installDir, skillName);
if (existsSync(targetPath) && !this.force) {
if (!this.quiet) {
@@ -492,25 +509,31 @@ export class InstallCommand extends Command {
}
if (useSymlink && primaryPath) {
symlinkSync(primaryPath, targetPath, "dir");
symlinkSync(primaryPath, targetPath, isStandaloneFile ? "file" : "dir");
} else {
cpSync(sourcePath, targetPath, { recursive: true, dereference: true });
if (isStandaloneFile) {
cpSync(sourcePath, targetPath);
} else {
cpSync(sourcePath, targetPath, { recursive: true, dereference: true });
}
if (isSymlinkMode && primaryPath === null) {
primaryPath = targetPath;
}
const packageJsonPath = join(targetPath, "package.json");
if (existsSync(packageJsonPath)) {
s.stop(`Installed ${skillName} to ${adapter.name}`);
s.start(`Installing npm dependencies for ${skillName}...`);
try {
await execFileAsync("npm", ["install", "--omit=dev"], { cwd: targetPath });
s.stop(`Installed dependencies for ${skillName}`);
} catch {
s.stop(colors.warning(`Dependencies failed for ${skillName}`));
console.log(colors.muted("Run manually: npm install in " + targetPath));
if (!isStandaloneFile) {
const packageJsonPath = join(targetPath, "package.json");
if (existsSync(packageJsonPath)) {
s.stop(`Installed ${skillName} to ${adapter.name}`);
s.start(`Installing npm dependencies for ${skillName}...`);
try {
await execFileAsync("npm", ["install", "--omit=dev"], { cwd: targetPath });
s.stop(`Installed dependencies for ${skillName}`);
} catch {
s.stop(colors.warning(`Dependencies failed for ${skillName}`));
console.log(colors.muted("Run manually: npm install in " + targetPath));
}
s.start(`Finishing ${skillName} installation...`);
}
s.start(`Finishing ${skillName} installation...`);
}
}
+60 -15
View File
@@ -1,8 +1,8 @@
import { existsSync, rmSync, readFileSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { join, dirname, basename } from 'node:path';
import { colors, warn, success, error } from '../onboarding/index.js';
import { Command, Option } from 'clipanion';
import { findSkill, AgentsMdParser, AgentsMdGenerator } from '@skillkit/core';
import { findSkill, findAllSkills, loadMetadata, AgentsMdParser, AgentsMdGenerator } from '@skillkit/core';
import { getSearchDirs } from '../helpers.js';
export class RemoveCommand extends Command {
@@ -13,11 +13,21 @@ export class RemoveCommand extends Command {
examples: [
['Remove a skill', '$0 remove pdf'],
['Remove multiple skills', '$0 remove pdf xlsx docx'],
['Force removal without confirmation', '$0 remove pdf --force'],
['Remove all skills from a source', '$0 remove --source iii-hq/iii'],
['Remove all installed skills', '$0 remove --all'],
['Force removal without confirmation', '$0 remove --all --force'],
],
});
skills = Option.Rest({ required: 1 });
skills = Option.Rest({ required: 0 });
all = Option.Boolean('--all,-a', false, {
description: 'Remove all installed skills',
});
source = Option.String('--source,-s', {
description: 'Remove all skills installed from this source (e.g. iii-hq/iii)',
});
force = Option.Boolean('--force,-f', false, {
description: 'Skip confirmation',
@@ -25,24 +35,59 @@ export class RemoveCommand extends Command {
async execute(): Promise<number> {
const searchDirs = getSearchDirs();
if (!this.all && !this.source && this.skills.length === 0) {
error('Provide skill names, --source, or --all');
return 1;
}
let skillsToRemove: Array<{ name: string; path: string }> = [];
if (this.all || this.source) {
const allSkills = findAllSkills(searchDirs);
if (this.source) {
skillsToRemove = allSkills.filter((s) => {
const meta = loadMetadata(s.path);
return meta?.source?.includes(this.source!) ?? false;
}).map((s) => ({ name: s.name, path: s.path }));
if (skillsToRemove.length === 0) {
warn(`No skills found from source: ${this.source}`);
return 0;
}
} else {
skillsToRemove = allSkills.map((s) => ({ name: s.name, path: s.path }));
if (skillsToRemove.length === 0) {
warn('No installed skills found');
return 0;
}
}
console.log(colors.muted(`Found ${skillsToRemove.length} skill(s) to remove`));
} else {
for (const skillName of this.skills) {
const skill = findSkill(skillName, searchDirs);
if (!skill) {
warn(`Skill not found: ${skillName}`);
continue;
}
skillsToRemove.push({ name: skill.name, path: skill.path });
}
}
let removed = 0;
let failed = 0;
for (const skillName of this.skills) {
const skill = findSkill(skillName, searchDirs);
if (!skill) {
warn(`Skill not found: ${skillName}`);
continue;
}
if (!existsSync(skill.path)) {
warn(`Path not found: ${skill.path}`);
for (const { name: skillName, path: skillPath } of skillsToRemove) {
if (!existsSync(skillPath)) {
warn(`Path not found: ${skillPath}`);
continue;
}
try {
rmSync(skill.path, { recursive: true, force: true });
rmSync(skillPath, { recursive: true, force: true });
if (skillPath.endsWith('.md')) {
const metaSibling = join(dirname(skillPath), `.${basename(skillPath, '.md')}.skillkit.json`);
if (existsSync(metaSibling)) rmSync(metaSibling);
}
success(`Removed: ${skillName}`);
removed++;
} catch (err) {
+16 -36
View File
@@ -1,4 +1,3 @@
// Re-export commonly used functions with proper adapters
import {
loadConfig,
getSearchDirs as coreGetSearchDirs,
@@ -8,59 +7,40 @@ import {
loadSkillMetadata as coreLoadSkillMetadata,
saveSkillMetadata as coreSaveSkillMetadata,
} from "@skillkit/core";
import { getAdapter, detectAgent } from "@skillkit/agents";
import { getAdapter, detectAgent, getAllAdapters } from "@skillkit/agents";
import type { AgentType, AgentAdapterInfo } from "@skillkit/core";
// Re-export metadata functions directly (they don't need adapter bridging)
export const loadSkillMetadata = coreLoadSkillMetadata;
export const saveSkillMetadata = coreSaveSkillMetadata;
function toAdapterInfo(adapter: { type: AgentType; name: string; skillsDir: string; configFile: string }): AgentAdapterInfo {
return { type: adapter.type, name: adapter.name, skillsDir: adapter.skillsDir, configFile: adapter.configFile };
}
export function getSearchDirs(agentType?: AgentType): string[] {
const type = agentType || loadConfig().agent;
const adapter = getAdapter(type);
const adapterInfo: AgentAdapterInfo = {
type: adapter.type,
name: adapter.name,
skillsDir: adapter.skillsDir,
configFile: adapter.configFile,
};
return coreGetSearchDirs(adapterInfo);
const dirs = new Set<string>();
const adapters = agentType ? [getAdapter(agentType)] : getAllAdapters();
for (const adapter of adapters) {
for (const dir of coreGetSearchDirs(toAdapterInfo(adapter))) {
dirs.add(dir);
}
}
return [...dirs];
}
export function getInstallDir(global = false, agentType?: AgentType): string {
const type = agentType || loadConfig().agent;
const adapter = getAdapter(type);
const adapterInfo: AgentAdapterInfo = {
type: adapter.type,
name: adapter.name,
skillsDir: adapter.skillsDir,
configFile: adapter.configFile,
};
return coreGetInstallDir(adapterInfo, global);
return coreGetInstallDir(toAdapterInfo(getAdapter(type)), global);
}
export function getAgentConfigPath(agentType?: AgentType): string {
const type = agentType || loadConfig().agent;
const adapter = getAdapter(type);
const adapterInfo: AgentAdapterInfo = {
type: adapter.type,
name: adapter.name,
skillsDir: adapter.skillsDir,
configFile: adapter.configFile,
};
return coreGetAgentConfigPath(adapterInfo);
return coreGetAgentConfigPath(toAdapterInfo(getAdapter(type)));
}
export async function initProject(agentType?: AgentType): Promise<void> {
const type = agentType || (await detectAgent());
const adapter = getAdapter(type);
const adapterInfo: AgentAdapterInfo = {
type: adapter.type,
name: adapter.name,
skillsDir: adapter.skillsDir,
configFile: adapter.configFile,
};
return coreInitProject(type, adapterInfo);
return coreInitProject(type, toAdapterInfo(getAdapter(type)));
}
export function formatCount(count: number): string {
+11 -4
View File
@@ -1,5 +1,5 @@
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { existsSync, readFileSync, writeFileSync, mkdirSync, statSync } from 'node:fs';
import { join, dirname, basename } from 'node:path';
import { homedir } from 'node:os';
import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
import { SkillkitConfig, type AgentType, type SkillMetadata, type AgentAdapterInfo } from './types.js';
@@ -7,6 +7,13 @@ import { SkillkitConfig, type AgentType, type SkillMetadata, type AgentAdapterIn
const CONFIG_FILE = 'skillkit.yaml';
const METADATA_FILE = '.skillkit.json';
function metadataPathFor(skillPath: string): string {
if (skillPath.endsWith('.md') && existsSync(skillPath) && statSync(skillPath).isFile()) {
return join(dirname(skillPath), `.${basename(skillPath, '.md')}.skillkit.json`);
}
return join(skillPath, METADATA_FILE);
}
export function getProjectConfigPath(): string {
return join(process.cwd(), CONFIG_FILE);
}
@@ -110,12 +117,12 @@ export function getAgentConfigPath(adapter: AgentAdapterInfo): string {
}
export function saveSkillMetadata(skillPath: string, metadata: SkillMetadata): void {
const metadataPath = join(skillPath, METADATA_FILE);
const metadataPath = metadataPathFor(skillPath);
writeFileSync(metadataPath, JSON.stringify(metadata, null, 2), 'utf-8');
}
export function loadSkillMetadata(skillPath: string): SkillMetadata | null {
const metadataPath = join(skillPath, METADATA_FILE);
const metadataPath = metadataPathFor(skillPath);
if (!existsSync(metadataPath)) {
return null;
+121
View File
@@ -1,4 +1,7 @@
import { spawn, execFileSync } from 'node:child_process';
import { existsSync, rmSync } from 'node:fs';
import type { GitProvider, CloneResult } from '../types.js';
import { SKILL_DISCOVERY_PATHS } from '../skills.js';
export interface GitProviderAdapter {
readonly type: GitProvider;
@@ -16,6 +19,124 @@ export interface CloneOptions {
depth?: number;
branch?: string;
ssh?: boolean;
subpath?: string;
onProgress?: (message: string) => void;
}
function parseGitProgress(text: string): string | null {
const lines = text.split(/[\r\n]+/);
for (let i = lines.length - 1; i >= 0; i--) {
const line = lines[i].trim();
if (!line) continue;
const pctMatch = line.match(/([\w\s]+):\s*(\d+)%\s*\((\d+)\/(\d+)\)(?:,\s*(.+?))?(?:\s*$|\s*\r)/);
if (pctMatch) {
const phase = pctMatch[1].replace(/^remote:\s*/, '').trim();
const pct = pctMatch[2];
const size = pctMatch[5]?.replace(/,?\s*done\.?$/, '').trim();
return size ? `${phase} ${pct}% ${size}` : `${phase} ${pct}%`;
}
const remoteMatch = line.match(/remote:\s*([\w\s]+):\s*(\d+)%/);
if (remoteMatch) {
return `${remoteMatch[1].trim()} ${remoteMatch[2]}%`;
}
const remoteDone = line.match(/remote:\s*([\w\s]+):\s*\d+.*done/);
if (remoteDone) {
return `${remoteDone[1].trim()}`;
}
}
return null;
}
function spawnGit(args: string[], onProgress?: (msg: string) => void): Promise<void> {
return new Promise<void>((resolve, reject) => {
const proc = spawn('git', args, { stdio: ['pipe', 'ignore', 'pipe'] });
let stderr = '';
proc.stderr?.on('data', (data: Buffer) => {
const text = data.toString();
stderr = (stderr + text).slice(-4096);
if (onProgress) {
const msg = parseGitProgress(text);
if (msg) onProgress(msg);
}
});
proc.on('close', (code) => {
if (code === 0) resolve();
else reject(new Error(stderr.slice(-500)));
});
proc.on('error', reject);
});
}
export async function cloneRepo(
cloneUrl: string,
tempDir: string,
options: CloneOptions = {},
): Promise<void> {
const args = ['clone', '--progress', '--single-branch', '--no-tags'];
if (options.depth) {
args.push('--depth', String(options.depth));
}
if (options.branch) {
args.push('--branch', options.branch);
}
const tryPartialClone = !options.branch;
if (tryPartialClone) {
const partialArgs = [...args, '--filter=blob:none', '--no-checkout', cloneUrl, tempDir];
let partialCloneOk = false;
try {
await spawnGit(partialArgs, options.onProgress);
partialCloneOk = true;
} catch {
options.onProgress?.('Partial clone unsupported, falling back to full clone');
if (existsSync(tempDir)) {
rmSync(tempDir, { recursive: true, force: true });
}
}
if (partialCloneOk) {
try {
execFileSync('git', ['-C', tempDir, 'sparse-checkout', 'init', '--cone'], {
stdio: ['pipe', 'pipe', 'pipe'],
});
const sparsePaths = [...SKILL_DISCOVERY_PATHS];
if (options.subpath) {
sparsePaths.push(options.subpath);
for (const dp of SKILL_DISCOVERY_PATHS) {
sparsePaths.push(`${options.subpath}/${dp}`);
}
}
execFileSync(
'git',
['-C', tempDir, 'sparse-checkout', 'set', '--', ...sparsePaths],
{ stdio: ['pipe', 'pipe', 'pipe'] },
);
options.onProgress?.('Checking out files');
await spawnGit(['-C', tempDir, 'checkout'], options.onProgress);
return;
} catch {
options.onProgress?.('Sparse checkout failed, falling back to full clone');
if (existsSync(tempDir)) {
rmSync(tempDir, { recursive: true, force: true });
}
}
}
}
args.push(cloneUrl, tempDir);
await spawnGit(args, options.onProgress);
}
export function parseShorthand(source: string): { owner: string; repo: string; subpath?: string } | null {
+2 -15
View File
@@ -1,10 +1,9 @@
import { execFileSync } from "node:child_process";
import { existsSync, rmSync } from "node:fs";
import { join, basename } from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
import type { GitProviderAdapter, CloneOptions } from "./base.js";
import { parseShorthand } from "./base.js";
import { parseShorthand, cloneRepo } from "./base.js";
import type { GitProvider, CloneResult } from "../types.js";
import { discoverSkills } from "../skills.js";
@@ -76,19 +75,7 @@ export class BitbucketProvider implements GitProviderAdapter {
const tempDir = join(tmpdir(), `skillkit-${randomUUID()}`);
try {
const args = ["clone"];
if (options.depth) {
args.push("--depth", String(options.depth));
}
if (options.branch) {
args.push("--branch", options.branch);
}
args.push(cloneUrl, tempDir);
execFileSync("git", args, {
stdio: ["pipe", "pipe", "pipe"],
encoding: "utf-8",
});
await cloneRepo(cloneUrl, tempDir, { ...options, subpath });
const searchDir = subpath ? join(tempDir, subpath) : tempDir;
const skills = discoverSkills(searchDir);
+2 -15
View File
@@ -1,10 +1,9 @@
import { execFileSync } from "node:child_process";
import { existsSync, rmSync } from "node:fs";
import { join, basename } from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
import type { GitProviderAdapter, CloneOptions } from "./base.js";
import { parseShorthand, isGitUrl } from "./base.js";
import { parseShorthand, isGitUrl, cloneRepo } from "./base.js";
import type { GitProvider, CloneResult } from "../types.js";
import { discoverSkills } from "../skills.js";
@@ -69,19 +68,7 @@ export class GitHubProvider implements GitProviderAdapter {
const tempDir = join(tmpdir(), `skillkit-${randomUUID()}`);
try {
const args = ["clone"];
if (options.depth) {
args.push("--depth", String(options.depth));
}
if (options.branch) {
args.push("--branch", options.branch);
}
args.push(cloneUrl, tempDir);
execFileSync("git", args, {
stdio: ["pipe", "pipe", "pipe"],
encoding: "utf-8",
});
await cloneRepo(cloneUrl, tempDir, { ...options, subpath });
const searchDir = subpath ? join(tempDir, subpath) : tempDir;
const skills = discoverSkills(searchDir);
+2 -15
View File
@@ -1,10 +1,9 @@
import { execFileSync } from "node:child_process";
import { existsSync, rmSync } from "node:fs";
import { join, basename } from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
import type { GitProviderAdapter, CloneOptions } from "./base.js";
import { parseShorthand } from "./base.js";
import { parseShorthand, cloneRepo } from "./base.js";
import type { GitProvider, CloneResult } from "../types.js";
import { discoverSkills } from "../skills.js";
@@ -74,19 +73,7 @@ export class GitLabProvider implements GitProviderAdapter {
const tempDir = join(tmpdir(), `skillkit-${randomUUID()}`);
try {
const args = ["clone"];
if (options.depth) {
args.push("--depth", String(options.depth));
}
if (options.branch) {
args.push("--branch", options.branch);
}
args.push(cloneUrl, tempDir);
execFileSync("git", args, {
stdio: ["pipe", "pipe", "pipe"],
encoding: "utf-8",
});
await cloneRepo(cloneUrl, tempDir, { ...options, subpath });
const searchDir = subpath ? join(tempDir, subpath) : tempDir;
const skills = discoverSkills(searchDir);
+2 -14
View File
@@ -1,9 +1,9 @@
import { execFileSync } from "node:child_process";
import { existsSync, rmSync } from "node:fs";
import { join, basename, resolve } from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
import type { GitProviderAdapter, CloneOptions } from "./base.js";
import { cloneRepo } from "./base.js";
import type { GitProvider, CloneResult } from "../types.js";
import { discoverSkills } from "../skills.js";
@@ -65,19 +65,7 @@ export class SkillsShProvider implements GitProviderAdapter {
const tempDir = join(tmpdir(), `skillkit-skillssh-${randomUUID()}`);
try {
const args = ["clone"];
if (options.depth) {
args.push("--depth", String(options.depth));
}
if (options.branch) {
args.push("--branch", options.branch);
}
args.push(cloneUrl, tempDir);
execFileSync("git", args, {
stdio: ["pipe", "pipe", "pipe"],
encoding: "utf-8",
});
await cloneRepo(cloneUrl, tempDir, { ...options, subpath });
const searchDir = subpath ? join(tempDir, subpath) : tempDir;
@@ -58,6 +58,47 @@ function ruleMatchesFileType(rule: SecurityRule, fileType: string | undefined):
return rule.fileTypes.includes(fileType);
}
function stripMarkdownCodeAndFrontmatter(content: string): string {
const lines = content.split('\n');
let fenceDelimiter: string | null = null;
let inFrontmatter = false;
let frontmatterDone = false;
if (lines.length > 0 && /^---\s*$/.test(lines[0].trim())) {
const hasCloser = lines.slice(1).some((l) => /^---\s*$/.test(l.trim()));
if (hasCloser) {
inFrontmatter = true;
}
}
return lines.map((line, i) => {
const trimmed = line.trim();
if (inFrontmatter && !frontmatterDone) {
if (i > 0 && /^---\s*$/.test(trimmed)) {
inFrontmatter = false;
frontmatterDone = true;
}
return '';
}
const fenceMatch = trimmed.match(/^(`{3,}|~{3,})/);
if (fenceMatch) {
const char = fenceMatch[1][0];
if (fenceDelimiter === null) {
fenceDelimiter = char;
return line;
} else if (char === fenceDelimiter) {
fenceDelimiter = null;
return line;
}
}
if (fenceDelimiter !== null) return '';
return line;
}).join('\n');
}
function matchesExcludePatterns(line: string, rule: SecurityRule): boolean {
if (!rule.excludePatterns) return false;
return rule.excludePatterns.some((p) => p.test(line));
@@ -95,6 +136,10 @@ export class StaticAnalyzer implements Analyzer {
continue;
}
if (fileType === 'markdown') {
content = stripMarkdownCodeAndFrontmatter(content);
}
const lines = content.split('\n');
for (const rule of applicableRules) {
+16 -6
View File
@@ -1,5 +1,5 @@
import { existsSync, readdirSync, readFileSync } from 'node:fs';
import { join, basename, resolve, sep } from 'node:path';
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
import { join, basename, dirname, resolve, sep } from 'node:path';
import { parse as parseYaml } from 'yaml';
import { SkillFrontmatter, SkillMetadata, type Skill, type SkillLocation } from './types.js';
@@ -274,7 +274,10 @@ export function extractField(content: string, field: string): string | null {
}
export function loadMetadata(skillPath: string): SkillMetadata | null {
const metadataPath = join(skillPath, '.skillkit.json');
const isFile = skillPath.endsWith('.md') && existsSync(skillPath) && statSync(skillPath).isFile();
const metadataPath = isFile
? join(dirname(skillPath), `.${basename(skillPath, '.md')}.skillkit.json`)
: join(skillPath, '.skillkit.json');
if (!existsSync(metadataPath)) {
return null;
@@ -311,10 +314,17 @@ export function findSkill(name: string, searchDirs: string[]): Skill | null {
for (const dir of searchDirs) {
if (!existsSync(dir)) continue;
const location: SkillLocation = isPathInside(dir, process.cwd()) ? 'project' : 'global';
const skillPath = join(dir, name);
if (existsSync(skillPath)) {
const location: SkillLocation = dir.includes(process.cwd()) ? 'project' : 'global';
return parseSkill(skillPath, location);
const skill = parseSkill(skillPath, location);
if (skill) return skill;
}
const mdPath = join(dir, name.endsWith('.md') ? name : `${name}.md`);
if (existsSync(mdPath)) {
return parseStandaloneSkill(mdPath, location);
}
}
@@ -328,7 +338,7 @@ export function findAllSkills(searchDirs: string[]): Skill[] {
for (const dir of searchDirs) {
if (!existsSync(dir)) continue;
const location: SkillLocation = dir.includes(process.cwd()) ? 'project' : 'global';
const location: SkillLocation = isPathInside(dir, process.cwd()) ? 'project' : 'global';
const discovered = discoverSkills(dir);
for (const skill of discovered) {