mirror of
https://github.com/rohitg00/skillkit.git
synced 2026-11-18 18:04:26 +00:00
feat: add live clone progress and sparse checkout for large repos
Replace silent execFileSync with async spawn + --progress flag so the spinner shows real download progress (e.g. "Receiving objects 45%"). For large monorepos, try partial clone (--filter=blob:none) with sparse checkout of SKILL_DISCOVERY_PATHS first, falling back to regular --depth 1 clone if the server doesn't support it. Also adds --single-branch --no-tags to all clones. Shared cloneRepo() helper in base.ts eliminates duplicate clone logic across all four git providers.
This commit is contained in:
@@ -235,8 +235,13 @@ export class InstallCommand extends Command {
|
||||
}
|
||||
|
||||
if (!result) {
|
||||
s.start(`Fetching from ${providerAdapter.name}...`);
|
||||
result = await providerAdapter.clone(this.source, "", { depth: 1 });
|
||||
s.start(`Cloning from ${providerAdapter.name}...`);
|
||||
result = await providerAdapter.clone(this.source, "", {
|
||||
depth: 1,
|
||||
onProgress: (msg) => {
|
||||
s.message(`${providerAdapter!.name} — ${msg}`);
|
||||
},
|
||||
});
|
||||
|
||||
if (!result.success || !result.path) {
|
||||
s.stop(colors.error(result.error || "Failed to fetch source"));
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import { spawn, execFileSync } from 'node:child_process';
|
||||
import { existsSync, rmSync } from 'node:fs';
|
||||
import type { GitProvider, CloneResult } from '../types.js';
|
||||
import { SKILL_DISCOVERY_PATHS } from '../skills.js';
|
||||
|
||||
export interface GitProviderAdapter {
|
||||
readonly type: GitProvider;
|
||||
@@ -16,6 +19,108 @@ export interface CloneOptions {
|
||||
depth?: number;
|
||||
branch?: string;
|
||||
ssh?: boolean;
|
||||
onProgress?: (message: string) => void;
|
||||
}
|
||||
|
||||
function parseGitProgress(text: string): string | null {
|
||||
const lines = text.split(/[\r\n]+/);
|
||||
for (let i = lines.length - 1; i >= 0; i--) {
|
||||
const line = lines[i].trim();
|
||||
if (!line) continue;
|
||||
|
||||
const pctMatch = line.match(/([\w\s]+):\s*(\d+)%\s*\((\d+)\/(\d+)\)(?:,\s*(.+?))?(?:\s*$|\s*\r)/);
|
||||
if (pctMatch) {
|
||||
const phase = pctMatch[1].replace(/^remote:\s*/, '').trim();
|
||||
const pct = pctMatch[2];
|
||||
const size = pctMatch[5]?.replace(/,?\s*done\.?$/, '').trim();
|
||||
return size ? `${phase} ${pct}% ${size}` : `${phase} ${pct}%`;
|
||||
}
|
||||
|
||||
const remoteMatch = line.match(/remote:\s*([\w\s]+):\s*(\d+)%/);
|
||||
if (remoteMatch) {
|
||||
return `${remoteMatch[1].trim()} ${remoteMatch[2]}%`;
|
||||
}
|
||||
|
||||
const remoteDone = line.match(/remote:\s*([\w\s]+):\s*\d+.*done/);
|
||||
if (remoteDone) {
|
||||
return `${remoteDone[1].trim()}`;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function spawnGit(args: string[]): Promise<void>;
|
||||
function spawnGit(args: string[], onProgress: (msg: string) => void): Promise<void>;
|
||||
function spawnGit(args: string[], onProgress?: (msg: string) => void): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const proc = spawn('git', args, { stdio: ['pipe', 'pipe', 'pipe'] });
|
||||
let stderr = '';
|
||||
|
||||
proc.stderr?.on('data', (data: Buffer) => {
|
||||
const text = data.toString();
|
||||
stderr += text;
|
||||
if (onProgress) {
|
||||
const msg = parseGitProgress(text);
|
||||
if (msg) onProgress(msg);
|
||||
}
|
||||
});
|
||||
|
||||
proc.on('close', (code) => {
|
||||
if (code === 0) resolve();
|
||||
else reject(new Error(stderr.slice(-500)));
|
||||
});
|
||||
|
||||
proc.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
export async function cloneRepo(
|
||||
cloneUrl: string,
|
||||
tempDir: string,
|
||||
options: CloneOptions = {},
|
||||
): Promise<void> {
|
||||
const args = ['clone', '--progress', '--single-branch', '--no-tags'];
|
||||
|
||||
if (options.depth) {
|
||||
args.push('--depth', String(options.depth));
|
||||
}
|
||||
if (options.branch) {
|
||||
args.push('--branch', options.branch);
|
||||
}
|
||||
|
||||
const tryPartialClone = !options.branch;
|
||||
|
||||
if (tryPartialClone) {
|
||||
const partialArgs = [...args, '--filter=blob:none', '--no-checkout', cloneUrl, tempDir];
|
||||
|
||||
try {
|
||||
await spawnGit(partialArgs, options.onProgress ?? (() => {}));
|
||||
|
||||
execFileSync('git', ['-C', tempDir, 'sparse-checkout', 'init', '--cone'], {
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
});
|
||||
|
||||
const sparsePaths = SKILL_DISCOVERY_PATHS.filter((p) => !p.includes('/'));
|
||||
const deepPaths = SKILL_DISCOVERY_PATHS.filter((p) => p.includes('/'));
|
||||
execFileSync(
|
||||
'git',
|
||||
['-C', tempDir, 'sparse-checkout', 'set', ...sparsePaths, ...deepPaths],
|
||||
{ stdio: ['pipe', 'pipe', 'pipe'] },
|
||||
);
|
||||
|
||||
options.onProgress?.('Checking out files');
|
||||
await spawnGit(['-C', tempDir, 'checkout'], options.onProgress ?? (() => {}));
|
||||
|
||||
return;
|
||||
} catch {
|
||||
if (existsSync(tempDir)) {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
args.push(cloneUrl, tempDir);
|
||||
await spawnGit(args, options.onProgress ?? (() => {}));
|
||||
}
|
||||
|
||||
export function parseShorthand(source: string): { owner: string; repo: string; subpath?: string } | null {
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, rmSync } from "node:fs";
|
||||
import { join, basename } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { GitProviderAdapter, CloneOptions } from "./base.js";
|
||||
import { parseShorthand } from "./base.js";
|
||||
import { parseShorthand, cloneRepo } from "./base.js";
|
||||
import type { GitProvider, CloneResult } from "../types.js";
|
||||
import { discoverSkills } from "../skills.js";
|
||||
|
||||
@@ -76,19 +75,7 @@ export class BitbucketProvider implements GitProviderAdapter {
|
||||
const tempDir = join(tmpdir(), `skillkit-${randomUUID()}`);
|
||||
|
||||
try {
|
||||
const args = ["clone"];
|
||||
if (options.depth) {
|
||||
args.push("--depth", String(options.depth));
|
||||
}
|
||||
if (options.branch) {
|
||||
args.push("--branch", options.branch);
|
||||
}
|
||||
args.push(cloneUrl, tempDir);
|
||||
|
||||
execFileSync("git", args, {
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
encoding: "utf-8",
|
||||
});
|
||||
await cloneRepo(cloneUrl, tempDir, options);
|
||||
|
||||
const searchDir = subpath ? join(tempDir, subpath) : tempDir;
|
||||
const skills = discoverSkills(searchDir);
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, rmSync } from "node:fs";
|
||||
import { join, basename } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { GitProviderAdapter, CloneOptions } from "./base.js";
|
||||
import { parseShorthand, isGitUrl } from "./base.js";
|
||||
import { parseShorthand, isGitUrl, cloneRepo } from "./base.js";
|
||||
import type { GitProvider, CloneResult } from "../types.js";
|
||||
import { discoverSkills } from "../skills.js";
|
||||
|
||||
@@ -69,19 +68,7 @@ export class GitHubProvider implements GitProviderAdapter {
|
||||
const tempDir = join(tmpdir(), `skillkit-${randomUUID()}`);
|
||||
|
||||
try {
|
||||
const args = ["clone"];
|
||||
if (options.depth) {
|
||||
args.push("--depth", String(options.depth));
|
||||
}
|
||||
if (options.branch) {
|
||||
args.push("--branch", options.branch);
|
||||
}
|
||||
args.push(cloneUrl, tempDir);
|
||||
|
||||
execFileSync("git", args, {
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
encoding: "utf-8",
|
||||
});
|
||||
await cloneRepo(cloneUrl, tempDir, options);
|
||||
|
||||
const searchDir = subpath ? join(tempDir, subpath) : tempDir;
|
||||
const skills = discoverSkills(searchDir);
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, rmSync } from "node:fs";
|
||||
import { join, basename } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { GitProviderAdapter, CloneOptions } from "./base.js";
|
||||
import { parseShorthand } from "./base.js";
|
||||
import { parseShorthand, cloneRepo } from "./base.js";
|
||||
import type { GitProvider, CloneResult } from "../types.js";
|
||||
import { discoverSkills } from "../skills.js";
|
||||
|
||||
@@ -74,19 +73,7 @@ export class GitLabProvider implements GitProviderAdapter {
|
||||
const tempDir = join(tmpdir(), `skillkit-${randomUUID()}`);
|
||||
|
||||
try {
|
||||
const args = ["clone"];
|
||||
if (options.depth) {
|
||||
args.push("--depth", String(options.depth));
|
||||
}
|
||||
if (options.branch) {
|
||||
args.push("--branch", options.branch);
|
||||
}
|
||||
args.push(cloneUrl, tempDir);
|
||||
|
||||
execFileSync("git", args, {
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
encoding: "utf-8",
|
||||
});
|
||||
await cloneRepo(cloneUrl, tempDir, options);
|
||||
|
||||
const searchDir = subpath ? join(tempDir, subpath) : tempDir;
|
||||
const skills = discoverSkills(searchDir);
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, rmSync } from "node:fs";
|
||||
import { join, basename, resolve } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { GitProviderAdapter, CloneOptions } from "./base.js";
|
||||
import { cloneRepo } from "./base.js";
|
||||
import type { GitProvider, CloneResult } from "../types.js";
|
||||
import { discoverSkills } from "../skills.js";
|
||||
|
||||
@@ -65,19 +65,7 @@ export class SkillsShProvider implements GitProviderAdapter {
|
||||
const tempDir = join(tmpdir(), `skillkit-skillssh-${randomUUID()}`);
|
||||
|
||||
try {
|
||||
const args = ["clone"];
|
||||
if (options.depth) {
|
||||
args.push("--depth", String(options.depth));
|
||||
}
|
||||
if (options.branch) {
|
||||
args.push("--branch", options.branch);
|
||||
}
|
||||
args.push(cloneUrl, tempDir);
|
||||
|
||||
execFileSync("git", args, {
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
encoding: "utf-8",
|
||||
});
|
||||
await cloneRepo(cloneUrl, tempDir, options);
|
||||
|
||||
const searchDir = subpath ? join(tempDir, subpath) : tempDir;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user