Files
skillshare/internal/sync/context.go
T
Willie 9c994faa17 fix(frontmatter): read and rewrite SKILL.md that starts with a UTF-8 BOM (#505)
* fix(frontmatter): read and rewrite SKILL.md that starts with a UTF-8 BOM

Windows editors such as Notepad write a BOM before the opening ---. The
lenient readers (ParseSkillName, ParseFrontmatterField/Fields/List, ReadSkillBody)
did not skip it, so under target_naming standard/prefixed the skill had no
readable name and was silently skipped. Strip the BOM in the shared frontmatter
locator and the line scanner, and in the hub, search, audit and sync/context
readers that split lines themselves (utils.TrimBOM).

The writers kept working on the unstripped text, so SetFrontmatterValue
prepended a second block and ToggleFrontmatterFlag/RewriteFrontmatterList
could leave a BOM in the middle of the file. They now split the BOM off and
put it back in front, rewriting only the name value (or flag/list) as before.

* fix(frontmatter): cover the dashboard parser and stop auditing a leading BOM

A consumer sweep after the first BOM commit found two more places that
mishandled a SKILL.md written by Notepad:

- audit reported the leading BOM as a HIGH hidden-unicode finding (U+FEFF is
  in the zero-width rule), so every such skill showed a risk score. A BOM at
  the very start of a file is its encoding mark and hides nothing; one past
  the start is still reported.
- the dashboard's parseSkillMarkdown saw no frontmatter, so the skill editor
  would treat the whole file as body and write a second block on save.

The audit-engine page and its four translations note the exemption.

* fix(ui): keep the BOM when the skill editor saves a SKILL.md

parseSkillMarkdown dropped the BOM so the editor could see the frontmatter,
but SkillEditor writes the composed text back through PUT .../content, so every
dashboard edit silently removed the marker from a Notepad-written file. The
parse result now carries the BOM and composeSkillMarkdown puts it back in front,
matching the Go rewrite paths.
2026-10-09 23:58:56 +08:00

116 lines
3.4 KiB
Go

package sync
import (
"os"
"path/filepath"
"strings"
"unicode/utf8"
"gopkg.in/yaml.v3"
"skillshare/internal/utils"
)
// EstimateTokens approximates how many tokens s costs. ASCII text runs about
// four characters per token; anything wider (CJK, emoji, accented letters)
// is counted as one token per rune, which is what tokenizers charge for it.
// ponytail: two-bucket heuristic. Swap in a real tokenizer if a model's count
// ever needs to be exact rather than comparable.
func EstimateTokens(s string) int {
var ascii, wide int
for _, r := range s {
if r < utf8.RuneSelf {
ascii++
} else {
wide++
}
}
return ascii/4 + wide
}
// skillContext is the context cost of one SKILL.md: the always-loaded layer
// (name + description) and the on-demand layer (body after the frontmatter).
type skillContext struct {
Name string
Description string
DescChars int // rune count of name + description
BodyChars int // rune count of body after frontmatter
DescTokens int
BodyTokens int
}
// CalcSkillContext reads a skill's SKILL.md once and returns rune counts for:
// - descChars: name + description (always loaded into context for skill matching)
// - bodyChars: everything after the frontmatter closing --- (loaded on demand)
//
// Returns (0, 0, nil) if SKILL.md does not exist or is empty.
func CalcSkillContext(skillPath string) (descChars, bodyChars int, description string, err error) {
skillFile := filepath.Join(skillPath, "SKILL.md")
content, err := os.ReadFile(skillFile)
if err != nil {
return 0, 0, "", nil
}
sc, _ := calcContextFromContent(content)
return sc.DescChars, sc.BodyChars, sc.Description, nil
}
// calcContextFromContent parses frontmatter name+description and body from
// pre-read SKILL.md content, returning rune and token counts for each layer.
// yamlErr is non-nil when the frontmatter YAML between --- delimiters cannot
// be parsed; body metrics are still valid in that case.
func calcContextFromContent(content []byte) (sc skillContext, yamlErr error) {
s := utils.TrimBOM(string(content))
if len(s) == 0 {
return sc, nil
}
// Find frontmatter boundaries (between --- delimiters)
trimmed := strings.TrimSpace(s)
if !strings.HasPrefix(trimmed, "---") {
// No frontmatter — entire content is body
body := strings.TrimSpace(s)
sc.BodyChars = utf8.RuneCountInString(body)
sc.BodyTokens = EstimateTokens(body)
return sc, nil
}
// Find closing ---
rest := trimmed[3:] // skip opening ---
rest = strings.TrimLeft(rest, " \t")
if len(rest) > 0 && rest[0] == '\n' {
rest = rest[1:]
} else if len(rest) > 1 && rest[0] == '\r' && rest[1] == '\n' {
rest = rest[2:]
}
closingIdx := strings.Index(rest, "\n---")
if closingIdx < 0 {
// Malformed frontmatter — no closing ---
return sc, nil
}
fmRaw := rest[:closingIdx]
body := strings.TrimSpace(rest[closingIdx+4:]) // skip \n---
// Parse name and description from frontmatter YAML
var fm struct {
Name string `yaml:"name"`
Description string `yaml:"description"`
}
yamlErr = yaml.Unmarshal([]byte(fmRaw), &fm)
// Build always-loaded string
alwaysLoaded := fm.Name
if fm.Description != "" {
alwaysLoaded = fm.Name + " " + fm.Description
}
sc.Name = fm.Name
sc.Description = fm.Description
sc.DescChars = utf8.RuneCountInString(alwaysLoaded)
sc.BodyChars = utf8.RuneCountInString(body)
sc.DescTokens = EstimateTokens(alwaysLoaded)
sc.BodyTokens = EstimateTokens(body)
return sc, yamlErr
}