perf(update): speed up grouped updates and unstick progress bar

Optimize grouped repo updates by skipping reinstall when installed metadata already matches remote state (commit match, or subdir tree-hash match when HEAD moved).

Emit per-skill grouped progress events so the batch progress bar advances continuously instead of appearing stuck at 0%.

When skipping by tree-hash match, best-effort refresh meta.Version to latest commit to keep status/version output current.

Add unit tests covering commit/tree-hash skip behavior and metadata version refresh.
This commit is contained in:
Willie
2026-02-28 03:44:02 +08:00
parent b4a4969ff6
commit 51d2e3a6c5
5 changed files with 177 additions and 10 deletions
+4
View File
@@ -23,6 +23,10 @@
### Performance
- **Batch gitignore operations** — `.gitignore` updates during `install` reconciliation and `uninstall` are now batched into a single file read/write instead of one per skill; eliminates hang when `.gitignore` grows large (100K+ lines)
- **`update --all` grouped skip** — skills from the same repo are now skipped when installed metadata already matches remote state (commit or tree-hash match), avoiding redundant reinstall/copy; on large repos this eliminates the majority of work
- **`update --all` batch speed** — removed a fixed 50ms per-skill delay in grouped batch iteration that dominated runtime on large skill sets (~90 min at 108K skills → seconds)
- **`update --all` progress visibility** — batch progress bar now advances per-skill instead of per-repo, so it no longer appears stuck at 0% during large grouped updates; a scanning spinner and phase headers (`[1/3] Pulling N tracked repos...`) show which stage is running
- **`collect` scan speed** — directory size calculation is no longer run eagerly during skill discovery; deferred to the Web UI handler where it is actually needed
### Fixed
+24 -2
View File
@@ -167,6 +167,9 @@ func executeBatchUpdate(uc *updateContext, targets []updateTarget) (updateResult
batchOpts := uc.makeInstallOpts()
if ui.IsTTY() {
batchOpts.OnProgress = func(line string) {
if handleGroupedBatchProgress(progressBar, line) {
return
}
progressBar.UpdateTitle(line)
}
}
@@ -183,7 +186,6 @@ func executeBatchUpdate(uc *updateContext, targets []updateTarget) (updateResult
for subdir := range skillTargetMap {
t := pathToTarget[subdir]
progressBar.UpdateTitle(fmt.Sprintf("Updating %s", t.name))
if err := batchResult.Errors[subdir]; err != nil {
if isStaleError(err) {
@@ -210,7 +212,6 @@ func executeBatchUpdate(uc *updateContext, targets []updateTarget) (updateResult
} else {
result.skipped++
}
progressBar.Increment()
}
}
@@ -292,6 +293,27 @@ func executeBatchUpdate(uc *updateContext, targets []updateTarget) (updateResult
return result, nil
}
func handleGroupedBatchProgress(progressBar *ui.ProgressBar, line string) bool {
if !strings.HasPrefix(line, install.BatchUpdateProgressPrefix) {
return false
}
payload := strings.TrimSpace(strings.TrimPrefix(line, install.BatchUpdateProgressPrefix))
if payload == "" {
progressBar.UpdateTitle("Updating skills")
progressBar.Increment()
return true
}
parts := strings.SplitN(payload, " ", 2)
if len(parts) == 2 && strings.TrimSpace(parts[1]) != "" {
progressBar.UpdateTitle(fmt.Sprintf("Updating %s", strings.TrimSpace(parts[1])))
} else {
progressBar.UpdateTitle("Updating skills")
}
progressBar.Increment()
return true
}
// isStaleError returns true if the error indicates a skill path was deleted
// from the upstream repository.
func isStaleError(err error) bool {
+56 -5
View File
@@ -6,6 +6,10 @@ import (
"strings"
)
// BatchUpdateProgressPrefix marks per-skill grouped update progress messages
// emitted through InstallOptions.OnProgress.
const BatchUpdateProgressPrefix = "skillshare:batch-update:"
// BatchUpdateResult holds results for a batch of skills from the same repository
type BatchUpdateResult struct {
RepoURL string
@@ -56,6 +60,8 @@ func UpdateSkillsFromRepo(repoURL string, skillTargets map[string]string, opts I
discoveryMap[s.Path] = s
}
treeHashCache := make(map[string]string)
totalSkills := len(skillTargets)
processedSkills := 0
// 2. Install/Update each requested skill from the discovered repo
for repoSubdir, destPath := range skillTargets {
@@ -66,6 +72,8 @@ func UpdateSkillsFromRepo(repoURL string, skillTargets map[string]string, opts I
skillInfo, ok := discoveryMap[lookupKey]
if !ok {
result.Errors[repoSubdir] = fmt.Errorf("skill path %q not found in repository", repoSubdir)
processedSkills++
emitBatchSkillProgress(opts.OnProgress, processedSkills, totalSkills, repoSubdir)
continue
}
// Fast path for update --all on huge inventories:
@@ -73,6 +81,12 @@ func UpdateSkillsFromRepo(repoURL string, skillTargets map[string]string, opts I
// skip reinstall/copy entirely.
if opts.Update && !opts.DryRun &&
isSkillCurrentAtRepoState(destPath, repoSubdir, result.CommitHash, repoPath, treeHashCache) {
// Best-effort metadata refresh:
// when skipping by tree-hash match on a moved HEAD, keep meta.Version
// aligned to latest commit to avoid stale version reporting.
_ = refreshSkillMetaVersionIfNeeded(destPath, result.CommitHash)
processedSkills++
emitBatchSkillProgress(opts.OnProgress, processedSkills, totalSkills, repoSubdir)
continue
}
@@ -83,9 +97,13 @@ func UpdateSkillsFromRepo(repoURL string, skillTargets map[string]string, opts I
installResult, err := InstallFromDiscovery(discovery, skillInfo, destPath, skillOpts)
if err != nil {
result.Errors[repoSubdir] = err
processedSkills++
emitBatchSkillProgress(opts.OnProgress, processedSkills, totalSkills, repoSubdir)
continue
}
result.Results[repoSubdir] = installResult
processedSkills++
emitBatchSkillProgress(opts.OnProgress, processedSkills, totalSkills, repoSubdir)
}
return result, nil
@@ -101,15 +119,21 @@ func isSkillCurrentAtRepoState(destPath, repoSubdir, commitHash, repoPath string
if err != nil || meta == nil {
return false
}
if strings.TrimSpace(meta.Version) != strings.TrimSpace(commitHash) {
return false
}
subdir := strings.TrimSpace(repoSubdir)
if subdir == "" || subdir == "." {
// Root installs don't have a stable tree hash field; commit match is enough.
metaVersion := strings.TrimSpace(meta.Version)
commitHash = strings.TrimSpace(commitHash)
// Fast path: exact repo commit match means installed content is already current.
// This avoids per-skill git subtree lookups on very large batches.
if metaVersion != "" && metaVersion == commitHash {
return true
}
if subdir == "" || subdir == "." {
// Root installs don't have a stable tree hash field; require commit match.
return false
}
if strings.TrimSpace(meta.TreeHash) == "" {
return false
}
@@ -121,3 +145,30 @@ func isSkillCurrentAtRepoState(destPath, repoSubdir, commitHash, repoPath string
}
return treeHash != "" && meta.TreeHash == treeHash
}
func emitBatchSkillProgress(onProgress ProgressCallback, done, total int, repoSubdir string) {
if onProgress == nil || total <= 0 {
return
}
label := strings.TrimSpace(repoSubdir)
if label == "" {
label = "."
}
onProgress(fmt.Sprintf("%s%d/%d %s", BatchUpdateProgressPrefix, done, total, label))
}
func refreshSkillMetaVersionIfNeeded(destPath, commitHash string) error {
commitHash = strings.TrimSpace(commitHash)
if commitHash == "" {
return nil
}
meta, err := ReadMeta(destPath)
if err != nil || meta == nil {
return err
}
if strings.TrimSpace(meta.Version) == commitHash {
return nil
}
meta.Version = commitHash
return WriteMeta(destPath, meta)
}
+89 -3
View File
@@ -73,7 +73,7 @@ func TestIsSkillCurrentAtRepoState_SubdirTreeHashMatch(t *testing.T) {
}
}
func TestIsSkillCurrentAtRepoState_SubdirMissingTreeHashDoesNotSkip(t *testing.T) {
func TestIsSkillCurrentAtRepoState_SubdirMissingTreeHashCommitMatchSkips(t *testing.T) {
repo := initTestRepo(t)
subdir := filepath.Join(repo, "skills", "my-skill")
if err := os.MkdirAll(subdir, 0755); err != nil {
@@ -101,8 +101,8 @@ func TestIsSkillCurrentAtRepoState_SubdirMissingTreeHashDoesNotSkip(t *testing.T
t.Fatalf("WriteMeta failed: %v", err)
}
if isSkillCurrentAtRepoState(dest, "skills/my-skill", commit, repo, map[string]string{}) {
t.Fatal("expected subdir skill without tree hash to require reinstall")
if !isSkillCurrentAtRepoState(dest, "skills/my-skill", commit, repo, map[string]string{}) {
t.Fatal("expected subdir skill without tree hash to be up-to-date when commit matches")
}
}
@@ -133,3 +133,89 @@ func TestIsSkillCurrentAtRepoState_CommitMismatchDoesNotSkip(t *testing.T) {
t.Fatal("expected commit mismatch to require reinstall")
}
}
func TestIsSkillCurrentAtRepoState_SubdirTreeHashMatchSkipsOnCommitMismatch(t *testing.T) {
repo := initTestRepo(t)
subdir := filepath.Join(repo, "skills", "my-skill")
if err := os.MkdirAll(subdir, 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(subdir, "SKILL.md"), []byte("# Skill"), 0644); err != nil {
t.Fatal(err)
}
gitAdd(t, repo, ".")
gitCommit(t, repo, "add subdir skill")
originalCommit, err := getGitCommit(repo)
if err != nil || originalCommit == "" {
t.Fatalf("getGitCommit failed: %v, commit=%q", err, originalCommit)
}
tree := getSubdirTreeHash(repo, "skills/my-skill")
if tree == "" {
t.Fatal("expected non-empty tree hash")
}
// Commit changes outside the skill subdir.
if err := os.WriteFile(filepath.Join(repo, "README.md"), []byte("updated"), 0644); err != nil {
t.Fatal(err)
}
gitAdd(t, repo, ".")
gitCommit(t, repo, "update unrelated file")
newCommit, err := getGitCommit(repo)
if err != nil || newCommit == "" {
t.Fatalf("getGitCommit failed: %v, commit=%q", err, newCommit)
}
if newCommit == originalCommit {
t.Fatal("expected commit to change after unrelated update")
}
dest := t.TempDir()
if err := WriteMeta(dest, &SkillMeta{
Source: "https://example.com/repo/skills/my-skill",
Type: "github",
InstalledAt: time.Now(),
Version: originalCommit,
TreeHash: tree,
Subdir: "skills/my-skill",
}); err != nil {
t.Fatalf("WriteMeta failed: %v", err)
}
if !isSkillCurrentAtRepoState(dest, "skills/my-skill", newCommit, repo, map[string]string{}) {
t.Fatal("expected subdir skill to skip when tree hash matches despite commit mismatch")
}
}
func TestRefreshSkillMetaVersionIfNeeded(t *testing.T) {
dest := t.TempDir()
meta := &SkillMeta{
Source: "https://example.com/repo/skills/my-skill",
Type: "github",
InstalledAt: time.Now(),
Version: "old-commit",
TreeHash: "tree-hash",
Subdir: "skills/my-skill",
}
if err := WriteMeta(dest, meta); err != nil {
t.Fatalf("WriteMeta failed: %v", err)
}
if err := refreshSkillMetaVersionIfNeeded(dest, "new-commit"); err != nil {
t.Fatalf("refreshSkillMetaVersionIfNeeded failed: %v", err)
}
updated, err := ReadMeta(dest)
if err != nil {
t.Fatalf("ReadMeta failed: %v", err)
}
if updated == nil {
t.Fatal("expected metadata to exist")
}
if updated.Version != "new-commit" {
t.Fatalf("expected version to be refreshed, got %q", updated.Version)
}
if updated.TreeHash != "tree-hash" {
t.Fatalf("expected tree hash to be preserved, got %q", updated.TreeHash)
}
}
+4
View File
@@ -32,6 +32,10 @@ All notable changes to skillshare are documented here. For the full commit histo
### Performance
- **Batch gitignore operations** — `.gitignore` updates during `install` reconciliation and `uninstall` are now batched into a single file read/write instead of one per skill; eliminates hang when `.gitignore` grows large (100K+ lines)
- **`update --all` grouped skip** — skills from the same repo are now skipped when installed metadata already matches remote state (commit or tree-hash match), avoiding redundant reinstall/copy; on large repos this eliminates the majority of work
- **`update --all` batch speed** — removed a fixed 50ms per-skill delay in grouped batch iteration that dominated runtime on large skill sets (~90 min at 108K skills → seconds)
- **`update --all` progress visibility** — batch progress bar now advances per-skill instead of per-repo, so it no longer appears stuck at 0% during large grouped updates; a scanning spinner and phase headers (`[1/3] Pulling N tracked repos...`) show which stage is running
- **`collect` scan speed** — directory size calculation is no longer run eagerly during skill discovery; deferred to the Web UI handler where it is actually needed
### Fixed