mirror of
https://github.com/runkids/skillshare.git
synced 2026-11-20 20:34:19 +00:00
perf(update): speed up grouped updates and unstick progress bar
Optimize grouped repo updates by skipping reinstall when installed metadata already matches remote state (commit match, or subdir tree-hash match when HEAD moved). Emit per-skill grouped progress events so the batch progress bar advances continuously instead of appearing stuck at 0%. When skipping by tree-hash match, best-effort refresh meta.Version to latest commit to keep status/version output current. Add unit tests covering commit/tree-hash skip behavior and metadata version refresh.
This commit is contained in:
@@ -23,6 +23,10 @@
|
||||
### Performance
|
||||
|
||||
- **Batch gitignore operations** — `.gitignore` updates during `install` reconciliation and `uninstall` are now batched into a single file read/write instead of one per skill; eliminates hang when `.gitignore` grows large (100K+ lines)
|
||||
- **`update --all` grouped skip** — skills from the same repo are now skipped when installed metadata already matches remote state (commit or tree-hash match), avoiding redundant reinstall/copy; on large repos this eliminates the majority of work
|
||||
- **`update --all` batch speed** — removed a fixed 50ms per-skill delay in grouped batch iteration that dominated runtime on large skill sets (~90 min at 108K skills → seconds)
|
||||
- **`update --all` progress visibility** — batch progress bar now advances per-skill instead of per-repo, so it no longer appears stuck at 0% during large grouped updates; a scanning spinner and phase headers (`[1/3] Pulling N tracked repos...`) show which stage is running
|
||||
- **`collect` scan speed** — directory size calculation is no longer run eagerly during skill discovery; deferred to the Web UI handler where it is actually needed
|
||||
|
||||
### Fixed
|
||||
|
||||
|
||||
@@ -167,6 +167,9 @@ func executeBatchUpdate(uc *updateContext, targets []updateTarget) (updateResult
|
||||
batchOpts := uc.makeInstallOpts()
|
||||
if ui.IsTTY() {
|
||||
batchOpts.OnProgress = func(line string) {
|
||||
if handleGroupedBatchProgress(progressBar, line) {
|
||||
return
|
||||
}
|
||||
progressBar.UpdateTitle(line)
|
||||
}
|
||||
}
|
||||
@@ -183,7 +186,6 @@ func executeBatchUpdate(uc *updateContext, targets []updateTarget) (updateResult
|
||||
|
||||
for subdir := range skillTargetMap {
|
||||
t := pathToTarget[subdir]
|
||||
progressBar.UpdateTitle(fmt.Sprintf("Updating %s", t.name))
|
||||
|
||||
if err := batchResult.Errors[subdir]; err != nil {
|
||||
if isStaleError(err) {
|
||||
@@ -210,7 +212,6 @@ func executeBatchUpdate(uc *updateContext, targets []updateTarget) (updateResult
|
||||
} else {
|
||||
result.skipped++
|
||||
}
|
||||
progressBar.Increment()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -292,6 +293,27 @@ func executeBatchUpdate(uc *updateContext, targets []updateTarget) (updateResult
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func handleGroupedBatchProgress(progressBar *ui.ProgressBar, line string) bool {
|
||||
if !strings.HasPrefix(line, install.BatchUpdateProgressPrefix) {
|
||||
return false
|
||||
}
|
||||
payload := strings.TrimSpace(strings.TrimPrefix(line, install.BatchUpdateProgressPrefix))
|
||||
if payload == "" {
|
||||
progressBar.UpdateTitle("Updating skills")
|
||||
progressBar.Increment()
|
||||
return true
|
||||
}
|
||||
|
||||
parts := strings.SplitN(payload, " ", 2)
|
||||
if len(parts) == 2 && strings.TrimSpace(parts[1]) != "" {
|
||||
progressBar.UpdateTitle(fmt.Sprintf("Updating %s", strings.TrimSpace(parts[1])))
|
||||
} else {
|
||||
progressBar.UpdateTitle("Updating skills")
|
||||
}
|
||||
progressBar.Increment()
|
||||
return true
|
||||
}
|
||||
|
||||
// isStaleError returns true if the error indicates a skill path was deleted
|
||||
// from the upstream repository.
|
||||
func isStaleError(err error) bool {
|
||||
|
||||
@@ -6,6 +6,10 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// BatchUpdateProgressPrefix marks per-skill grouped update progress messages
|
||||
// emitted through InstallOptions.OnProgress.
|
||||
const BatchUpdateProgressPrefix = "skillshare:batch-update:"
|
||||
|
||||
// BatchUpdateResult holds results for a batch of skills from the same repository
|
||||
type BatchUpdateResult struct {
|
||||
RepoURL string
|
||||
@@ -56,6 +60,8 @@ func UpdateSkillsFromRepo(repoURL string, skillTargets map[string]string, opts I
|
||||
discoveryMap[s.Path] = s
|
||||
}
|
||||
treeHashCache := make(map[string]string)
|
||||
totalSkills := len(skillTargets)
|
||||
processedSkills := 0
|
||||
|
||||
// 2. Install/Update each requested skill from the discovered repo
|
||||
for repoSubdir, destPath := range skillTargets {
|
||||
@@ -66,6 +72,8 @@ func UpdateSkillsFromRepo(repoURL string, skillTargets map[string]string, opts I
|
||||
skillInfo, ok := discoveryMap[lookupKey]
|
||||
if !ok {
|
||||
result.Errors[repoSubdir] = fmt.Errorf("skill path %q not found in repository", repoSubdir)
|
||||
processedSkills++
|
||||
emitBatchSkillProgress(opts.OnProgress, processedSkills, totalSkills, repoSubdir)
|
||||
continue
|
||||
}
|
||||
// Fast path for update --all on huge inventories:
|
||||
@@ -73,6 +81,12 @@ func UpdateSkillsFromRepo(repoURL string, skillTargets map[string]string, opts I
|
||||
// skip reinstall/copy entirely.
|
||||
if opts.Update && !opts.DryRun &&
|
||||
isSkillCurrentAtRepoState(destPath, repoSubdir, result.CommitHash, repoPath, treeHashCache) {
|
||||
// Best-effort metadata refresh:
|
||||
// when skipping by tree-hash match on a moved HEAD, keep meta.Version
|
||||
// aligned to latest commit to avoid stale version reporting.
|
||||
_ = refreshSkillMetaVersionIfNeeded(destPath, result.CommitHash)
|
||||
processedSkills++
|
||||
emitBatchSkillProgress(opts.OnProgress, processedSkills, totalSkills, repoSubdir)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -83,9 +97,13 @@ func UpdateSkillsFromRepo(repoURL string, skillTargets map[string]string, opts I
|
||||
installResult, err := InstallFromDiscovery(discovery, skillInfo, destPath, skillOpts)
|
||||
if err != nil {
|
||||
result.Errors[repoSubdir] = err
|
||||
processedSkills++
|
||||
emitBatchSkillProgress(opts.OnProgress, processedSkills, totalSkills, repoSubdir)
|
||||
continue
|
||||
}
|
||||
result.Results[repoSubdir] = installResult
|
||||
processedSkills++
|
||||
emitBatchSkillProgress(opts.OnProgress, processedSkills, totalSkills, repoSubdir)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
@@ -101,15 +119,21 @@ func isSkillCurrentAtRepoState(destPath, repoSubdir, commitHash, repoPath string
|
||||
if err != nil || meta == nil {
|
||||
return false
|
||||
}
|
||||
if strings.TrimSpace(meta.Version) != strings.TrimSpace(commitHash) {
|
||||
return false
|
||||
}
|
||||
|
||||
subdir := strings.TrimSpace(repoSubdir)
|
||||
if subdir == "" || subdir == "." {
|
||||
// Root installs don't have a stable tree hash field; commit match is enough.
|
||||
metaVersion := strings.TrimSpace(meta.Version)
|
||||
commitHash = strings.TrimSpace(commitHash)
|
||||
|
||||
// Fast path: exact repo commit match means installed content is already current.
|
||||
// This avoids per-skill git subtree lookups on very large batches.
|
||||
if metaVersion != "" && metaVersion == commitHash {
|
||||
return true
|
||||
}
|
||||
|
||||
if subdir == "" || subdir == "." {
|
||||
// Root installs don't have a stable tree hash field; require commit match.
|
||||
return false
|
||||
}
|
||||
if strings.TrimSpace(meta.TreeHash) == "" {
|
||||
return false
|
||||
}
|
||||
@@ -121,3 +145,30 @@ func isSkillCurrentAtRepoState(destPath, repoSubdir, commitHash, repoPath string
|
||||
}
|
||||
return treeHash != "" && meta.TreeHash == treeHash
|
||||
}
|
||||
|
||||
func emitBatchSkillProgress(onProgress ProgressCallback, done, total int, repoSubdir string) {
|
||||
if onProgress == nil || total <= 0 {
|
||||
return
|
||||
}
|
||||
label := strings.TrimSpace(repoSubdir)
|
||||
if label == "" {
|
||||
label = "."
|
||||
}
|
||||
onProgress(fmt.Sprintf("%s%d/%d %s", BatchUpdateProgressPrefix, done, total, label))
|
||||
}
|
||||
|
||||
func refreshSkillMetaVersionIfNeeded(destPath, commitHash string) error {
|
||||
commitHash = strings.TrimSpace(commitHash)
|
||||
if commitHash == "" {
|
||||
return nil
|
||||
}
|
||||
meta, err := ReadMeta(destPath)
|
||||
if err != nil || meta == nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(meta.Version) == commitHash {
|
||||
return nil
|
||||
}
|
||||
meta.Version = commitHash
|
||||
return WriteMeta(destPath, meta)
|
||||
}
|
||||
|
||||
@@ -73,7 +73,7 @@ func TestIsSkillCurrentAtRepoState_SubdirTreeHashMatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsSkillCurrentAtRepoState_SubdirMissingTreeHashDoesNotSkip(t *testing.T) {
|
||||
func TestIsSkillCurrentAtRepoState_SubdirMissingTreeHashCommitMatchSkips(t *testing.T) {
|
||||
repo := initTestRepo(t)
|
||||
subdir := filepath.Join(repo, "skills", "my-skill")
|
||||
if err := os.MkdirAll(subdir, 0755); err != nil {
|
||||
@@ -101,8 +101,8 @@ func TestIsSkillCurrentAtRepoState_SubdirMissingTreeHashDoesNotSkip(t *testing.T
|
||||
t.Fatalf("WriteMeta failed: %v", err)
|
||||
}
|
||||
|
||||
if isSkillCurrentAtRepoState(dest, "skills/my-skill", commit, repo, map[string]string{}) {
|
||||
t.Fatal("expected subdir skill without tree hash to require reinstall")
|
||||
if !isSkillCurrentAtRepoState(dest, "skills/my-skill", commit, repo, map[string]string{}) {
|
||||
t.Fatal("expected subdir skill without tree hash to be up-to-date when commit matches")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,3 +133,89 @@ func TestIsSkillCurrentAtRepoState_CommitMismatchDoesNotSkip(t *testing.T) {
|
||||
t.Fatal("expected commit mismatch to require reinstall")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsSkillCurrentAtRepoState_SubdirTreeHashMatchSkipsOnCommitMismatch(t *testing.T) {
|
||||
repo := initTestRepo(t)
|
||||
subdir := filepath.Join(repo, "skills", "my-skill")
|
||||
if err := os.MkdirAll(subdir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(subdir, "SKILL.md"), []byte("# Skill"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gitAdd(t, repo, ".")
|
||||
gitCommit(t, repo, "add subdir skill")
|
||||
|
||||
originalCommit, err := getGitCommit(repo)
|
||||
if err != nil || originalCommit == "" {
|
||||
t.Fatalf("getGitCommit failed: %v, commit=%q", err, originalCommit)
|
||||
}
|
||||
tree := getSubdirTreeHash(repo, "skills/my-skill")
|
||||
if tree == "" {
|
||||
t.Fatal("expected non-empty tree hash")
|
||||
}
|
||||
|
||||
// Commit changes outside the skill subdir.
|
||||
if err := os.WriteFile(filepath.Join(repo, "README.md"), []byte("updated"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gitAdd(t, repo, ".")
|
||||
gitCommit(t, repo, "update unrelated file")
|
||||
|
||||
newCommit, err := getGitCommit(repo)
|
||||
if err != nil || newCommit == "" {
|
||||
t.Fatalf("getGitCommit failed: %v, commit=%q", err, newCommit)
|
||||
}
|
||||
if newCommit == originalCommit {
|
||||
t.Fatal("expected commit to change after unrelated update")
|
||||
}
|
||||
|
||||
dest := t.TempDir()
|
||||
if err := WriteMeta(dest, &SkillMeta{
|
||||
Source: "https://example.com/repo/skills/my-skill",
|
||||
Type: "github",
|
||||
InstalledAt: time.Now(),
|
||||
Version: originalCommit,
|
||||
TreeHash: tree,
|
||||
Subdir: "skills/my-skill",
|
||||
}); err != nil {
|
||||
t.Fatalf("WriteMeta failed: %v", err)
|
||||
}
|
||||
|
||||
if !isSkillCurrentAtRepoState(dest, "skills/my-skill", newCommit, repo, map[string]string{}) {
|
||||
t.Fatal("expected subdir skill to skip when tree hash matches despite commit mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefreshSkillMetaVersionIfNeeded(t *testing.T) {
|
||||
dest := t.TempDir()
|
||||
meta := &SkillMeta{
|
||||
Source: "https://example.com/repo/skills/my-skill",
|
||||
Type: "github",
|
||||
InstalledAt: time.Now(),
|
||||
Version: "old-commit",
|
||||
TreeHash: "tree-hash",
|
||||
Subdir: "skills/my-skill",
|
||||
}
|
||||
if err := WriteMeta(dest, meta); err != nil {
|
||||
t.Fatalf("WriteMeta failed: %v", err)
|
||||
}
|
||||
|
||||
if err := refreshSkillMetaVersionIfNeeded(dest, "new-commit"); err != nil {
|
||||
t.Fatalf("refreshSkillMetaVersionIfNeeded failed: %v", err)
|
||||
}
|
||||
|
||||
updated, err := ReadMeta(dest)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadMeta failed: %v", err)
|
||||
}
|
||||
if updated == nil {
|
||||
t.Fatal("expected metadata to exist")
|
||||
}
|
||||
if updated.Version != "new-commit" {
|
||||
t.Fatalf("expected version to be refreshed, got %q", updated.Version)
|
||||
}
|
||||
if updated.TreeHash != "tree-hash" {
|
||||
t.Fatalf("expected tree hash to be preserved, got %q", updated.TreeHash)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,10 @@ All notable changes to skillshare are documented here. For the full commit histo
|
||||
### Performance
|
||||
|
||||
- **Batch gitignore operations** — `.gitignore` updates during `install` reconciliation and `uninstall` are now batched into a single file read/write instead of one per skill; eliminates hang when `.gitignore` grows large (100K+ lines)
|
||||
- **`update --all` grouped skip** — skills from the same repo are now skipped when installed metadata already matches remote state (commit or tree-hash match), avoiding redundant reinstall/copy; on large repos this eliminates the majority of work
|
||||
- **`update --all` batch speed** — removed a fixed 50ms per-skill delay in grouped batch iteration that dominated runtime on large skill sets (~90 min at 108K skills → seconds)
|
||||
- **`update --all` progress visibility** — batch progress bar now advances per-skill instead of per-repo, so it no longer appears stuck at 0% during large grouped updates; a scanning spinner and phase headers (`[1/3] Pulling N tracked repos...`) show which stage is running
|
||||
- **`collect` scan speed** — directory size calculation is no longer run eagerly during skill discovery; deferred to the Web UI handler where it is actually needed
|
||||
|
||||
### Fixed
|
||||
|
||||
|
||||
Reference in New Issue
Block a user