Files
skillshare/internal/sync/context.go
T
Willie 1fb23f1e59 feat(analyze): measure what each target actually loads
The analyze numbers were drifting from what tools really see. Symlink-mode
targets expose the whole source folder, so .skillignore'd skills still load
but were left out; merge/copy targets often hold skills the user dropped in
by hand that skillshare never counted; and chars/4 undercounted CJK text by
roughly four times.

Route the CLI, TUI, web handler and sync summary through one
sync.TargetSkills so the three copies of the filter logic cannot disagree
again. Disabled skills stay in the discovery with Disabled=true and are
priced for symlink targets; unmanaged folders in a target are reported as
Local. Token estimates now charge one token per wide rune.

The dashboard keeps the chosen target in the URL so target and project
pages can deep-link into the analysis, lists project targets after own
targets with their tool icon, and tags local/disabled rows. Select gives
single-select icons a key to silence a React warning.
2026-09-21 04:17:26 +08:00

114 lines
3.4 KiB
Go

package sync
import (
"os"
"path/filepath"
"strings"
"unicode/utf8"
"gopkg.in/yaml.v3"
)
// EstimateTokens approximates how many tokens s costs. ASCII text runs about
// four characters per token; anything wider (CJK, emoji, accented letters)
// is counted as one token per rune, which is what tokenizers charge for it.
// ponytail: two-bucket heuristic. Swap in a real tokenizer if a model's count
// ever needs to be exact rather than comparable.
func EstimateTokens(s string) int {
var ascii, wide int
for _, r := range s {
if r < utf8.RuneSelf {
ascii++
} else {
wide++
}
}
return ascii/4 + wide
}
// skillContext is the context cost of one SKILL.md: the always-loaded layer
// (name + description) and the on-demand layer (body after the frontmatter).
type skillContext struct {
Name string
Description string
DescChars int // rune count of name + description
BodyChars int // rune count of body after frontmatter
DescTokens int
BodyTokens int
}
// CalcSkillContext reads a skill's SKILL.md once and returns rune counts for:
// - descChars: name + description (always loaded into context for skill matching)
// - bodyChars: everything after the frontmatter closing --- (loaded on demand)
//
// Returns (0, 0, nil) if SKILL.md does not exist or is empty.
func CalcSkillContext(skillPath string) (descChars, bodyChars int, description string, err error) {
skillFile := filepath.Join(skillPath, "SKILL.md")
content, err := os.ReadFile(skillFile)
if err != nil {
return 0, 0, "", nil
}
sc, _ := calcContextFromContent(content)
return sc.DescChars, sc.BodyChars, sc.Description, nil
}
// calcContextFromContent parses frontmatter name+description and body from
// pre-read SKILL.md content, returning rune and token counts for each layer.
// yamlErr is non-nil when the frontmatter YAML between --- delimiters cannot
// be parsed; body metrics are still valid in that case.
func calcContextFromContent(content []byte) (sc skillContext, yamlErr error) {
s := string(content)
if len(s) == 0 {
return sc, nil
}
// Find frontmatter boundaries (between --- delimiters)
trimmed := strings.TrimSpace(s)
if !strings.HasPrefix(trimmed, "---") {
// No frontmatter — entire content is body
body := strings.TrimSpace(s)
sc.BodyChars = utf8.RuneCountInString(body)
sc.BodyTokens = EstimateTokens(body)
return sc, nil
}
// Find closing ---
rest := trimmed[3:] // skip opening ---
rest = strings.TrimLeft(rest, " \t")
if len(rest) > 0 && rest[0] == '\n' {
rest = rest[1:]
} else if len(rest) > 1 && rest[0] == '\r' && rest[1] == '\n' {
rest = rest[2:]
}
closingIdx := strings.Index(rest, "\n---")
if closingIdx < 0 {
// Malformed frontmatter — no closing ---
return sc, nil
}
fmRaw := rest[:closingIdx]
body := strings.TrimSpace(rest[closingIdx+4:]) // skip \n---
// Parse name and description from frontmatter YAML
var fm struct {
Name string `yaml:"name"`
Description string `yaml:"description"`
}
yamlErr = yaml.Unmarshal([]byte(fmRaw), &fm)
// Build always-loaded string
alwaysLoaded := fm.Name
if fm.Description != "" {
alwaysLoaded = fm.Name + " " + fm.Description
}
sc.Name = fm.Name
sc.Description = fm.Description
sc.DescChars = utf8.RuneCountInString(alwaysLoaded)
sc.BodyChars = utf8.RuneCountInString(body)
sc.DescTokens = EstimateTokens(alwaysLoaded)
sc.BodyTokens = EstimateTokens(body)
return sc, yamlErr
}