feat: add live clone progress and sparse checkout for large repos

Replace silent execFileSync with async spawn + --progress flag so the
spinner shows real download progress (e.g. "Receiving objects 45%").
For large monorepos, try partial clone (--filter=blob:none) with sparse
checkout of SKILL_DISCOVERY_PATHS first, falling back to regular
--depth 1 clone if the server doesn't support it. Also adds
--single-branch --no-tags to all clones.

Shared cloneRepo() helper in base.ts eliminates duplicate clone logic
across all four git providers.
This commit is contained in:
Rohit Ghumare
2026-04-10 20:28:38 +01:00
parent 172602f173
commit 1fb95fd57e
6 changed files with 120 additions and 61 deletions
+7 -2
View File
@@ -235,8 +235,13 @@ export class InstallCommand extends Command {
}
if (!result) {
s.start(`Fetching from ${providerAdapter.name}...`);
result = await providerAdapter.clone(this.source, "", { depth: 1 });
s.start(`Cloning from ${providerAdapter.name}...`);
result = await providerAdapter.clone(this.source, "", {
depth: 1,
onProgress: (msg) => {
s.message(`${providerAdapter!.name} — ${msg}`);
},
});
if (!result.success || !result.path) {
s.stop(colors.error(result.error || "Failed to fetch source"));
+105
View File
@@ -1,4 +1,7 @@
import { spawn, execFileSync } from 'node:child_process';
import { existsSync, rmSync } from 'node:fs';
import type { GitProvider, CloneResult } from '../types.js';
import { SKILL_DISCOVERY_PATHS } from '../skills.js';
export interface GitProviderAdapter {
readonly type: GitProvider;
@@ -16,6 +19,108 @@ export interface CloneOptions {
depth?: number;
branch?: string;
ssh?: boolean;
onProgress?: (message: string) => void;
}
function parseGitProgress(text: string): string | null {
const lines = text.split(/[\r\n]+/);
for (let i = lines.length - 1; i >= 0; i--) {
const line = lines[i].trim();
if (!line) continue;
const pctMatch = line.match(/([\w\s]+):\s*(\d+)%\s*\((\d+)\/(\d+)\)(?:,\s*(.+?))?(?:\s*$|\s*\r)/);
if (pctMatch) {
const phase = pctMatch[1].replace(/^remote:\s*/, '').trim();
const pct = pctMatch[2];
const size = pctMatch[5]?.replace(/,?\s*done\.?$/, '').trim();
return size ? `${phase} ${pct}% ${size}` : `${phase} ${pct}%`;
}
const remoteMatch = line.match(/remote:\s*([\w\s]+):\s*(\d+)%/);
if (remoteMatch) {
return `${remoteMatch[1].trim()} ${remoteMatch[2]}%`;
}
const remoteDone = line.match(/remote:\s*([\w\s]+):\s*\d+.*done/);
if (remoteDone) {
return `${remoteDone[1].trim()}`;
}
}
return null;
}
function spawnGit(args: string[]): Promise<void>;
function spawnGit(args: string[], onProgress: (msg: string) => void): Promise<void>;
function spawnGit(args: string[], onProgress?: (msg: string) => void): Promise<void> {
return new Promise<void>((resolve, reject) => {
const proc = spawn('git', args, { stdio: ['pipe', 'pipe', 'pipe'] });
let stderr = '';
proc.stderr?.on('data', (data: Buffer) => {
const text = data.toString();
stderr += text;
if (onProgress) {
const msg = parseGitProgress(text);
if (msg) onProgress(msg);
}
});
proc.on('close', (code) => {
if (code === 0) resolve();
else reject(new Error(stderr.slice(-500)));
});
proc.on('error', reject);
});
}
export async function cloneRepo(
cloneUrl: string,
tempDir: string,
options: CloneOptions = {},
): Promise<void> {
const args = ['clone', '--progress', '--single-branch', '--no-tags'];
if (options.depth) {
args.push('--depth', String(options.depth));
}
if (options.branch) {
args.push('--branch', options.branch);
}
const tryPartialClone = !options.branch;
if (tryPartialClone) {
const partialArgs = [...args, '--filter=blob:none', '--no-checkout', cloneUrl, tempDir];
try {
await spawnGit(partialArgs, options.onProgress ?? (() => {}));
execFileSync('git', ['-C', tempDir, 'sparse-checkout', 'init', '--cone'], {
stdio: ['pipe', 'pipe', 'pipe'],
});
const sparsePaths = SKILL_DISCOVERY_PATHS.filter((p) => !p.includes('/'));
const deepPaths = SKILL_DISCOVERY_PATHS.filter((p) => p.includes('/'));
execFileSync(
'git',
['-C', tempDir, 'sparse-checkout', 'set', ...sparsePaths, ...deepPaths],
{ stdio: ['pipe', 'pipe', 'pipe'] },
);
options.onProgress?.('Checking out files');
await spawnGit(['-C', tempDir, 'checkout'], options.onProgress ?? (() => {}));
return;
} catch {
if (existsSync(tempDir)) {
rmSync(tempDir, { recursive: true, force: true });
}
}
}
args.push(cloneUrl, tempDir);
await spawnGit(args, options.onProgress ?? (() => {}));
}
export function parseShorthand(source: string): { owner: string; repo: string; subpath?: string } | null {
+2 -15
View File
@@ -1,10 +1,9 @@
import { execFileSync } from "node:child_process";
import { existsSync, rmSync } from "node:fs";
import { join, basename } from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
import type { GitProviderAdapter, CloneOptions } from "./base.js";
import { parseShorthand } from "./base.js";
import { parseShorthand, cloneRepo } from "./base.js";
import type { GitProvider, CloneResult } from "../types.js";
import { discoverSkills } from "../skills.js";
@@ -76,19 +75,7 @@ export class BitbucketProvider implements GitProviderAdapter {
const tempDir = join(tmpdir(), `skillkit-${randomUUID()}`);
try {
const args = ["clone"];
if (options.depth) {
args.push("--depth", String(options.depth));
}
if (options.branch) {
args.push("--branch", options.branch);
}
args.push(cloneUrl, tempDir);
execFileSync("git", args, {
stdio: ["pipe", "pipe", "pipe"],
encoding: "utf-8",
});
await cloneRepo(cloneUrl, tempDir, options);
const searchDir = subpath ? join(tempDir, subpath) : tempDir;
const skills = discoverSkills(searchDir);
+2 -15
View File
@@ -1,10 +1,9 @@
import { execFileSync } from "node:child_process";
import { existsSync, rmSync } from "node:fs";
import { join, basename } from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
import type { GitProviderAdapter, CloneOptions } from "./base.js";
import { parseShorthand, isGitUrl } from "./base.js";
import { parseShorthand, isGitUrl, cloneRepo } from "./base.js";
import type { GitProvider, CloneResult } from "../types.js";
import { discoverSkills } from "../skills.js";
@@ -69,19 +68,7 @@ export class GitHubProvider implements GitProviderAdapter {
const tempDir = join(tmpdir(), `skillkit-${randomUUID()}`);
try {
const args = ["clone"];
if (options.depth) {
args.push("--depth", String(options.depth));
}
if (options.branch) {
args.push("--branch", options.branch);
}
args.push(cloneUrl, tempDir);
execFileSync("git", args, {
stdio: ["pipe", "pipe", "pipe"],
encoding: "utf-8",
});
await cloneRepo(cloneUrl, tempDir, options);
const searchDir = subpath ? join(tempDir, subpath) : tempDir;
const skills = discoverSkills(searchDir);
+2 -15
View File
@@ -1,10 +1,9 @@
import { execFileSync } from "node:child_process";
import { existsSync, rmSync } from "node:fs";
import { join, basename } from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
import type { GitProviderAdapter, CloneOptions } from "./base.js";
import { parseShorthand } from "./base.js";
import { parseShorthand, cloneRepo } from "./base.js";
import type { GitProvider, CloneResult } from "../types.js";
import { discoverSkills } from "../skills.js";
@@ -74,19 +73,7 @@ export class GitLabProvider implements GitProviderAdapter {
const tempDir = join(tmpdir(), `skillkit-${randomUUID()}`);
try {
const args = ["clone"];
if (options.depth) {
args.push("--depth", String(options.depth));
}
if (options.branch) {
args.push("--branch", options.branch);
}
args.push(cloneUrl, tempDir);
execFileSync("git", args, {
stdio: ["pipe", "pipe", "pipe"],
encoding: "utf-8",
});
await cloneRepo(cloneUrl, tempDir, options);
const searchDir = subpath ? join(tempDir, subpath) : tempDir;
const skills = discoverSkills(searchDir);
+2 -14
View File
@@ -1,9 +1,9 @@
import { execFileSync } from "node:child_process";
import { existsSync, rmSync } from "node:fs";
import { join, basename, resolve } from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
import type { GitProviderAdapter, CloneOptions } from "./base.js";
import { cloneRepo } from "./base.js";
import type { GitProvider, CloneResult } from "../types.js";
import { discoverSkills } from "../skills.js";
@@ -65,19 +65,7 @@ export class SkillsShProvider implements GitProviderAdapter {
const tempDir = join(tmpdir(), `skillkit-skillssh-${randomUUID()}`);
try {
const args = ["clone"];
if (options.depth) {
args.push("--depth", String(options.depth));
}
if (options.branch) {
args.push("--branch", options.branch);
}
args.push(cloneUrl, tempDir);
execFileSync("git", args, {
stdio: ["pipe", "pipe", "pipe"],
encoding: "utf-8",
});
await cloneRepo(cloneUrl, tempDir, options);
const searchDir = subpath ? join(tempDir, subpath) : tempDir;