From 51d2e3a6c5b178a0b5b0a5cdafc9ec6ef07a36ee Mon Sep 17 00:00:00 2001 From: Willie Date: Sat, 28 Feb 2026 02:17:16 +0800 Subject: [PATCH] perf(update): speed up grouped updates and unstick progress bar Optimize grouped repo updates by skipping reinstall when installed metadata already matches remote state (commit match, or subdir tree-hash match when HEAD moved). Emit per-skill grouped progress events so the batch progress bar advances continuously instead of appearing stuck at 0%. When skipping by tree-hash match, best-effort refresh meta.Version to latest commit to keep status/version output current. Add unit tests covering commit/tree-hash skip behavior and metadata version refresh. --- CHANGELOG.md | 4 ++ cmd/skillshare/update_batch.go | 26 +++++++- internal/install/batch_update.go | 61 ++++++++++++++++-- internal/install/batch_update_test.go | 92 ++++++++++++++++++++++++++- website/src/pages/changelog.md | 4 ++ 5 files changed, 177 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8615d8827..e405ed988 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,10 @@ ### Performance - **Batch gitignore operations** — `.gitignore` updates during `install` reconciliation and `uninstall` are now batched into a single file read/write instead of one per skill; eliminates hang when `.gitignore` grows large (100K+ lines) +- **`update --all` grouped skip** — skills from the same repo are now skipped when installed metadata already matches remote state (commit or tree-hash match), avoiding redundant reinstall/copy; on large repos this eliminates the majority of work +- **`update --all` batch speed** — removed a fixed 50ms per-skill delay in grouped batch iteration that dominated runtime on large skill sets (~90 min at 108K skills → seconds) +- **`update --all` progress visibility** — batch progress bar now advances per-skill instead of per-repo, so it no longer appears stuck at 0% during large grouped updates; a scanning spinner and phase headers (`[1/3] Pulling N tracked repos...`) show which stage is running +- **`collect` scan speed** — directory size calculation is no longer run eagerly during skill discovery; deferred to the Web UI handler where it is actually needed ### Fixed diff --git a/cmd/skillshare/update_batch.go b/cmd/skillshare/update_batch.go index eb6eafd1c..888a07018 100644 --- a/cmd/skillshare/update_batch.go +++ b/cmd/skillshare/update_batch.go @@ -167,6 +167,9 @@ func executeBatchUpdate(uc *updateContext, targets []updateTarget) (updateResult batchOpts := uc.makeInstallOpts() if ui.IsTTY() { batchOpts.OnProgress = func(line string) { + if handleGroupedBatchProgress(progressBar, line) { + return + } progressBar.UpdateTitle(line) } } @@ -183,7 +186,6 @@ func executeBatchUpdate(uc *updateContext, targets []updateTarget) (updateResult for subdir := range skillTargetMap { t := pathToTarget[subdir] - progressBar.UpdateTitle(fmt.Sprintf("Updating %s", t.name)) if err := batchResult.Errors[subdir]; err != nil { if isStaleError(err) { @@ -210,7 +212,6 @@ func executeBatchUpdate(uc *updateContext, targets []updateTarget) (updateResult } else { result.skipped++ } - progressBar.Increment() } } @@ -292,6 +293,27 @@ func executeBatchUpdate(uc *updateContext, targets []updateTarget) (updateResult return result, nil } +func handleGroupedBatchProgress(progressBar *ui.ProgressBar, line string) bool { + if !strings.HasPrefix(line, install.BatchUpdateProgressPrefix) { + return false + } + payload := strings.TrimSpace(strings.TrimPrefix(line, install.BatchUpdateProgressPrefix)) + if payload == "" { + progressBar.UpdateTitle("Updating skills") + progressBar.Increment() + return true + } + + parts := strings.SplitN(payload, " ", 2) + if len(parts) == 2 && strings.TrimSpace(parts[1]) != "" { + progressBar.UpdateTitle(fmt.Sprintf("Updating %s", strings.TrimSpace(parts[1]))) + } else { + progressBar.UpdateTitle("Updating skills") + } + progressBar.Increment() + return true +} + // isStaleError returns true if the error indicates a skill path was deleted // from the upstream repository. func isStaleError(err error) bool { diff --git a/internal/install/batch_update.go b/internal/install/batch_update.go index cd347e8ad..a5d196bff 100644 --- a/internal/install/batch_update.go +++ b/internal/install/batch_update.go @@ -6,6 +6,10 @@ import ( "strings" ) +// BatchUpdateProgressPrefix marks per-skill grouped update progress messages +// emitted through InstallOptions.OnProgress. +const BatchUpdateProgressPrefix = "skillshare:batch-update:" + // BatchUpdateResult holds results for a batch of skills from the same repository type BatchUpdateResult struct { RepoURL string @@ -56,6 +60,8 @@ func UpdateSkillsFromRepo(repoURL string, skillTargets map[string]string, opts I discoveryMap[s.Path] = s } treeHashCache := make(map[string]string) + totalSkills := len(skillTargets) + processedSkills := 0 // 2. Install/Update each requested skill from the discovered repo for repoSubdir, destPath := range skillTargets { @@ -66,6 +72,8 @@ func UpdateSkillsFromRepo(repoURL string, skillTargets map[string]string, opts I skillInfo, ok := discoveryMap[lookupKey] if !ok { result.Errors[repoSubdir] = fmt.Errorf("skill path %q not found in repository", repoSubdir) + processedSkills++ + emitBatchSkillProgress(opts.OnProgress, processedSkills, totalSkills, repoSubdir) continue } // Fast path for update --all on huge inventories: @@ -73,6 +81,12 @@ func UpdateSkillsFromRepo(repoURL string, skillTargets map[string]string, opts I // skip reinstall/copy entirely. if opts.Update && !opts.DryRun && isSkillCurrentAtRepoState(destPath, repoSubdir, result.CommitHash, repoPath, treeHashCache) { + // Best-effort metadata refresh: + // when skipping by tree-hash match on a moved HEAD, keep meta.Version + // aligned to latest commit to avoid stale version reporting. + _ = refreshSkillMetaVersionIfNeeded(destPath, result.CommitHash) + processedSkills++ + emitBatchSkillProgress(opts.OnProgress, processedSkills, totalSkills, repoSubdir) continue } @@ -83,9 +97,13 @@ func UpdateSkillsFromRepo(repoURL string, skillTargets map[string]string, opts I installResult, err := InstallFromDiscovery(discovery, skillInfo, destPath, skillOpts) if err != nil { result.Errors[repoSubdir] = err + processedSkills++ + emitBatchSkillProgress(opts.OnProgress, processedSkills, totalSkills, repoSubdir) continue } result.Results[repoSubdir] = installResult + processedSkills++ + emitBatchSkillProgress(opts.OnProgress, processedSkills, totalSkills, repoSubdir) } return result, nil @@ -101,15 +119,21 @@ func isSkillCurrentAtRepoState(destPath, repoSubdir, commitHash, repoPath string if err != nil || meta == nil { return false } - if strings.TrimSpace(meta.Version) != strings.TrimSpace(commitHash) { - return false - } subdir := strings.TrimSpace(repoSubdir) - if subdir == "" || subdir == "." { - // Root installs don't have a stable tree hash field; commit match is enough. + metaVersion := strings.TrimSpace(meta.Version) + commitHash = strings.TrimSpace(commitHash) + + // Fast path: exact repo commit match means installed content is already current. + // This avoids per-skill git subtree lookups on very large batches. + if metaVersion != "" && metaVersion == commitHash { return true } + + if subdir == "" || subdir == "." { + // Root installs don't have a stable tree hash field; require commit match. + return false + } if strings.TrimSpace(meta.TreeHash) == "" { return false } @@ -121,3 +145,30 @@ func isSkillCurrentAtRepoState(destPath, repoSubdir, commitHash, repoPath string } return treeHash != "" && meta.TreeHash == treeHash } + +func emitBatchSkillProgress(onProgress ProgressCallback, done, total int, repoSubdir string) { + if onProgress == nil || total <= 0 { + return + } + label := strings.TrimSpace(repoSubdir) + if label == "" { + label = "." + } + onProgress(fmt.Sprintf("%s%d/%d %s", BatchUpdateProgressPrefix, done, total, label)) +} + +func refreshSkillMetaVersionIfNeeded(destPath, commitHash string) error { + commitHash = strings.TrimSpace(commitHash) + if commitHash == "" { + return nil + } + meta, err := ReadMeta(destPath) + if err != nil || meta == nil { + return err + } + if strings.TrimSpace(meta.Version) == commitHash { + return nil + } + meta.Version = commitHash + return WriteMeta(destPath, meta) +} diff --git a/internal/install/batch_update_test.go b/internal/install/batch_update_test.go index 4241b389c..286552530 100644 --- a/internal/install/batch_update_test.go +++ b/internal/install/batch_update_test.go @@ -73,7 +73,7 @@ func TestIsSkillCurrentAtRepoState_SubdirTreeHashMatch(t *testing.T) { } } -func TestIsSkillCurrentAtRepoState_SubdirMissingTreeHashDoesNotSkip(t *testing.T) { +func TestIsSkillCurrentAtRepoState_SubdirMissingTreeHashCommitMatchSkips(t *testing.T) { repo := initTestRepo(t) subdir := filepath.Join(repo, "skills", "my-skill") if err := os.MkdirAll(subdir, 0755); err != nil { @@ -101,8 +101,8 @@ func TestIsSkillCurrentAtRepoState_SubdirMissingTreeHashDoesNotSkip(t *testing.T t.Fatalf("WriteMeta failed: %v", err) } - if isSkillCurrentAtRepoState(dest, "skills/my-skill", commit, repo, map[string]string{}) { - t.Fatal("expected subdir skill without tree hash to require reinstall") + if !isSkillCurrentAtRepoState(dest, "skills/my-skill", commit, repo, map[string]string{}) { + t.Fatal("expected subdir skill without tree hash to be up-to-date when commit matches") } } @@ -133,3 +133,89 @@ func TestIsSkillCurrentAtRepoState_CommitMismatchDoesNotSkip(t *testing.T) { t.Fatal("expected commit mismatch to require reinstall") } } + +func TestIsSkillCurrentAtRepoState_SubdirTreeHashMatchSkipsOnCommitMismatch(t *testing.T) { + repo := initTestRepo(t) + subdir := filepath.Join(repo, "skills", "my-skill") + if err := os.MkdirAll(subdir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(subdir, "SKILL.md"), []byte("# Skill"), 0644); err != nil { + t.Fatal(err) + } + gitAdd(t, repo, ".") + gitCommit(t, repo, "add subdir skill") + + originalCommit, err := getGitCommit(repo) + if err != nil || originalCommit == "" { + t.Fatalf("getGitCommit failed: %v, commit=%q", err, originalCommit) + } + tree := getSubdirTreeHash(repo, "skills/my-skill") + if tree == "" { + t.Fatal("expected non-empty tree hash") + } + + // Commit changes outside the skill subdir. + if err := os.WriteFile(filepath.Join(repo, "README.md"), []byte("updated"), 0644); err != nil { + t.Fatal(err) + } + gitAdd(t, repo, ".") + gitCommit(t, repo, "update unrelated file") + + newCommit, err := getGitCommit(repo) + if err != nil || newCommit == "" { + t.Fatalf("getGitCommit failed: %v, commit=%q", err, newCommit) + } + if newCommit == originalCommit { + t.Fatal("expected commit to change after unrelated update") + } + + dest := t.TempDir() + if err := WriteMeta(dest, &SkillMeta{ + Source: "https://example.com/repo/skills/my-skill", + Type: "github", + InstalledAt: time.Now(), + Version: originalCommit, + TreeHash: tree, + Subdir: "skills/my-skill", + }); err != nil { + t.Fatalf("WriteMeta failed: %v", err) + } + + if !isSkillCurrentAtRepoState(dest, "skills/my-skill", newCommit, repo, map[string]string{}) { + t.Fatal("expected subdir skill to skip when tree hash matches despite commit mismatch") + } +} + +func TestRefreshSkillMetaVersionIfNeeded(t *testing.T) { + dest := t.TempDir() + meta := &SkillMeta{ + Source: "https://example.com/repo/skills/my-skill", + Type: "github", + InstalledAt: time.Now(), + Version: "old-commit", + TreeHash: "tree-hash", + Subdir: "skills/my-skill", + } + if err := WriteMeta(dest, meta); err != nil { + t.Fatalf("WriteMeta failed: %v", err) + } + + if err := refreshSkillMetaVersionIfNeeded(dest, "new-commit"); err != nil { + t.Fatalf("refreshSkillMetaVersionIfNeeded failed: %v", err) + } + + updated, err := ReadMeta(dest) + if err != nil { + t.Fatalf("ReadMeta failed: %v", err) + } + if updated == nil { + t.Fatal("expected metadata to exist") + } + if updated.Version != "new-commit" { + t.Fatalf("expected version to be refreshed, got %q", updated.Version) + } + if updated.TreeHash != "tree-hash" { + t.Fatalf("expected tree hash to be preserved, got %q", updated.TreeHash) + } +} diff --git a/website/src/pages/changelog.md b/website/src/pages/changelog.md index 2e11c940c..7f9a32be5 100644 --- a/website/src/pages/changelog.md +++ b/website/src/pages/changelog.md @@ -32,6 +32,10 @@ All notable changes to skillshare are documented here. For the full commit histo ### Performance - **Batch gitignore operations** — `.gitignore` updates during `install` reconciliation and `uninstall` are now batched into a single file read/write instead of one per skill; eliminates hang when `.gitignore` grows large (100K+ lines) +- **`update --all` grouped skip** — skills from the same repo are now skipped when installed metadata already matches remote state (commit or tree-hash match), avoiding redundant reinstall/copy; on large repos this eliminates the majority of work +- **`update --all` batch speed** — removed a fixed 50ms per-skill delay in grouped batch iteration that dominated runtime on large skill sets (~90 min at 108K skills → seconds) +- **`update --all` progress visibility** — batch progress bar now advances per-skill instead of per-repo, so it no longer appears stuck at 0% during large grouped updates; a scanning spinner and phase headers (`[1/3] Pulling N tracked repos...`) show which stage is running +- **`collect` scan speed** — directory size calculation is no longer run eagerly during skill discovery; deferred to the Web UI handler where it is actually needed ### Fixed