Files
ProxyPilot/internal/memory/semantic.go

482 lines
11 KiB
Go

package memory
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"math"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
)
type SemanticRecord struct {
TS time.Time `json:"ts"`
Role string `json:"role,omitempty"`
Text string `json:"text"`
Vec []float32 `json:"vec"`
Norm float32 `json:"norm"`
Source string `json:"source,omitempty"`
Session string `json:"session,omitempty"`
Repo string `json:"repo,omitempty"`
}
type SemanticNamespaceInfo struct {
Key string `json:"key"`
Namespace string `json:"namespace"`
UpdatedAt time.Time `json:"updated_at"`
SizeBytes int64 `json:"size_bytes"`
ItemsBytes int64 `json:"items_bytes"`
}
func (s *FileStore) AppendSemantic(namespace string, records []SemanticRecord) error {
if s == nil || s.BaseDir == "" {
return errors.New("memory store not configured")
}
if namespace == "" || len(records) == 0 {
return nil
}
dir := s.semanticDir(namespace)
if err := os.MkdirAll(dir, 0o755); err != nil {
return err
}
_ = s.writeSemanticNamespace(dir, namespace)
path := filepath.Join(dir, "items.jsonl")
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
if err != nil {
return err
}
defer func() { _ = f.Close() }()
seen := make(map[string]struct{}, len(records))
for i := range records {
r := records[i]
if r.TS.IsZero() {
r.TS = time.Now()
}
r.Text = strings.TrimSpace(RedactText(r.Text))
if r.Text == "" || len(r.Vec) == 0 {
continue
}
if _, ok := seen[r.Text]; ok {
continue
}
seen[r.Text] = struct{}{}
if r.Norm <= 0 {
r.Norm = vectorNorm(r.Vec)
}
if r.Norm <= 0 {
continue
}
b, err := json.Marshal(r)
if err != nil {
continue
}
_, _ = f.Write(b)
_, _ = f.WriteString("\n")
}
return nil
}
func (s *FileStore) SearchSemantic(namespace string, query []float32, maxChars int, maxSnippets int) ([]string, error) {
return s.SearchSemanticWithText(namespace, query, "", maxChars, maxSnippets)
}
func (s *FileStore) SearchSemanticWithText(namespace string, query []float32, queryText string, maxChars int, maxSnippets int) ([]string, error) {
if s == nil || s.BaseDir == "" {
return nil, errors.New("memory store not configured")
}
if namespace == "" || len(query) == 0 {
return nil, nil
}
if maxChars <= 0 {
maxChars = 3000
}
if maxSnippets <= 0 {
maxSnippets = 4
}
dir := s.semanticDir(namespace)
path := filepath.Join(dir, "items.jsonl")
data, err := readTailBytes(path, 2*1024*1024)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
lines := bytes.Split(data, []byte("\n"))
if len(lines) == 0 {
return nil, nil
}
qn := vectorNorm(query)
if qn <= 0 {
return nil, nil
}
tokens := []string{}
if semanticRerankEnabled() && strings.TrimSpace(queryText) != "" {
tokens = tokenizeSemanticQuery(queryText, 12)
}
now := time.Now()
maxAgeDays := semanticMaxAgeDays()
recencyWindow := semanticRecencyWindowDays(maxAgeDays)
keywordBoost := semanticKeywordBoost()
recencyBoost := semanticRecencyBoost()
type scored struct {
score float32
text string
ts time.Time
}
scoredSnips := make([]scored, 0, len(lines))
for i := range lines {
line := bytes.TrimSpace(lines[i])
if len(line) == 0 {
continue
}
var r SemanticRecord
if err := json.Unmarshal(line, &r); err != nil {
continue
}
if len(r.Vec) == 0 || r.Norm <= 0 {
continue
}
if maxAgeDays > 0 && !r.TS.IsZero() {
if now.Sub(r.TS) > time.Duration(maxAgeDays)*24*time.Hour {
continue
}
}
score := cosineSim(query, qn, r.Vec, r.Norm)
if score <= 0 {
continue
}
if len(tokens) > 0 {
overlap := semanticTokenOverlap(tokens, r.Text)
if overlap > 0 {
score *= 1 + keywordBoost*float32(overlap)/float32(len(tokens))
}
}
if recencyBoost > 0 {
ageBoost := semanticRecencyScore(now, r.TS, recencyWindow)
if ageBoost > 0 {
score *= 1 + recencyBoost*ageBoost
}
}
txt := strings.TrimSpace(r.Text)
if txt == "" {
continue
}
scoredSnips = append(scoredSnips, scored{score: score, text: txt, ts: r.TS})
}
if len(scoredSnips) == 0 {
return nil, nil
}
sort.Slice(scoredSnips, func(i, j int) bool { return scoredSnips[i].score > scoredSnips[j].score })
out := make([]string, 0, maxSnippets)
chars := 0
seen := make(map[string]struct{}, maxSnippets*2)
for _, s := range scoredSnips {
if len(out) >= maxSnippets {
break
}
key := s.text
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
snip := s.text
if len(snip) > 1200 {
snip = snip[:1200] + "\n...[truncated]..."
}
if chars+len(snip) > maxChars {
break
}
out = append(out, snip)
chars += len(snip) + 4
}
return out, nil
}
func (s *FileStore) ReadSemanticTail(namespace string, limit int) ([]SemanticRecord, error) {
if s == nil || s.BaseDir == "" {
return nil, errors.New("memory store not configured")
}
if namespace == "" {
return nil, nil
}
if limit <= 0 {
limit = 50
}
if limit > 200 {
limit = 200
}
dir := s.semanticDir(namespace)
path := filepath.Join(dir, "items.jsonl")
data, err := readTailBytes(path, 2*1024*1024)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
lines := bytes.Split(data, []byte("\n"))
if len(lines) == 0 {
return nil, nil
}
out := make([]SemanticRecord, 0, limit)
for i := len(lines) - 1; i >= 0; i-- {
if len(out) >= limit {
break
}
line := bytes.TrimSpace(lines[i])
if len(line) == 0 {
continue
}
var r SemanticRecord
if err := json.Unmarshal(line, &r); err != nil {
continue
}
if strings.TrimSpace(r.Text) == "" {
continue
}
out = append(out, r)
}
// Reverse to chronological order
for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 {
out[i], out[j] = out[j], out[i]
}
return out, nil
}
func (s *FileStore) ListSemanticNamespaces(limit int) ([]SemanticNamespaceInfo, error) {
if s == nil || s.BaseDir == "" {
return nil, errors.New("memory store not configured")
}
root := filepath.Join(s.BaseDir, "semantic")
entries, err := os.ReadDir(root)
if err != nil {
if os.IsNotExist(err) {
return []SemanticNamespaceInfo{}, nil
}
return nil, err
}
out := make([]SemanticNamespaceInfo, 0, len(entries))
for _, e := range entries {
if !e.IsDir() {
continue
}
dir := filepath.Join(root, e.Name())
itemsPath := filepath.Join(dir, "items.jsonl")
fi, err := os.Stat(itemsPath)
if err != nil {
continue
}
ns := strings.TrimSpace(s.readSmallTextFile(filepath.Join(dir, "namespace.txt"), 2048))
info := SemanticNamespaceInfo{
Key: e.Name(),
Namespace: ns,
UpdatedAt: fi.ModTime(),
ItemsBytes: fi.Size(),
}
var total int64
if entries2, err := os.ReadDir(dir); err == nil {
for _, e2 := range entries2 {
if e2.IsDir() {
continue
}
if fi2, err := e2.Info(); err == nil {
total += fi2.Size()
}
}
}
info.SizeBytes = total
out = append(out, info)
}
sort.Slice(out, func(i, j int) bool {
return out[i].UpdatedAt.After(out[j].UpdatedAt)
})
if limit > 0 && len(out) > limit {
out = out[:limit]
}
return out, nil
}
func (s *FileStore) semanticDir(namespace string) string {
key := namespaceKey(namespace)
return filepath.Join(s.BaseDir, "semantic", key)
}
func (s *FileStore) writeSemanticNamespace(dir string, namespace string) error {
if strings.TrimSpace(namespace) == "" {
return nil
}
path := filepath.Join(dir, "namespace.txt")
if _, err := os.Stat(path); err == nil {
return nil
}
return os.WriteFile(path, []byte(namespace), 0o644)
}
func namespaceKey(namespace string) string {
h := sha256Hash(namespace)
return h[:16]
}
func sha256Hash(v string) string {
sum := sha256.Sum256([]byte(v))
return hex.EncodeToString(sum[:])
}
func vectorNorm(vec []float32) float32 {
var sum float64
for i := range vec {
v := float64(vec[i])
sum += v * v
}
if sum <= 0 {
return 0
}
return float32(math.Sqrt(sum))
}
func cosineSim(a []float32, aNorm float32, b []float32, bNorm float32) float32 {
if aNorm <= 0 || bNorm <= 0 {
return 0
}
n := len(a)
if len(b) < n {
n = len(b)
}
if n == 0 {
return 0
}
var dot float64
for i := 0; i < n; i++ {
dot += float64(a[i]) * float64(b[i])
}
return float32(dot) / (aNorm * bNorm)
}
func semanticMaxAgeDays() int {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_AGE_DAYS")); v != "" {
if n, err := strconv.Atoi(v); err == nil {
return n
}
}
return 0
}
func semanticRecencyWindowDays(maxAgeDays int) int {
if maxAgeDays > 0 {
return maxAgeDays
}
return 30
}
func semanticKeywordBoost() float32 {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_KEYWORD_BOOST")); v != "" {
if f, err := strconv.ParseFloat(v, 32); err == nil {
return float32(f)
}
}
return 0.25
}
func semanticRecencyBoost() float32 {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_RECENCY_BOOST")); v != "" {
if f, err := strconv.ParseFloat(v, 32); err == nil {
return float32(f)
}
}
return 0.15
}
func semanticRerankEnabled() bool {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_RERANK")); v != "" {
if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") {
return false
}
}
return true
}
func tokenizeSemanticQuery(q string, max int) []string {
q = strings.ToLower(q)
q = strings.Map(func(r rune) rune {
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' || r == '-' {
return r
}
return ' '
}, q)
parts := strings.Fields(q)
if len(parts) == 0 {
return nil
}
stop := map[string]struct{}{
"the": {}, "and": {}, "for": {}, "with": {}, "that": {}, "this": {}, "from": {}, "into": {}, "what": {}, "how": {},
"you": {}, "your": {}, "are": {}, "was": {}, "were": {}, "can": {}, "could": {}, "should": {}, "would": {}, "will": {},
}
out := make([]string, 0, max)
seen := make(map[string]struct{}, max*2)
for _, p := range parts {
if len(p) < 3 {
continue
}
if _, ok := stop[p]; ok {
continue
}
if _, ok := seen[p]; ok {
continue
}
seen[p] = struct{}{}
out = append(out, p)
if len(out) >= max {
break
}
}
return out
}
func semanticTokenOverlap(tokens []string, text string) int {
if len(tokens) == 0 || strings.TrimSpace(text) == "" {
return 0
}
low := strings.ToLower(text)
count := 0
for _, t := range tokens {
if t == "" {
continue
}
if strings.Contains(low, t) {
count++
}
}
return count
}
func semanticRecencyScore(now time.Time, ts time.Time, windowDays int) float32 {
if ts.IsZero() || windowDays <= 0 {
return 0
}
age := now.Sub(ts)
if age <= 0 {
return 1
}
window := time.Duration(windowDays) * 24 * time.Hour
if age >= window {
return 0
}
return float32(1 - float64(age)/float64(window))
}