mirror of
https://github.com/Finesssee/ProxyPilot.git
synced 2026-11-18 18:04:20 +00:00
482 lines
11 KiB
Go
482 lines
11 KiB
Go
package memory
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"math"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
type SemanticRecord struct {
|
|
TS time.Time `json:"ts"`
|
|
Role string `json:"role,omitempty"`
|
|
Text string `json:"text"`
|
|
Vec []float32 `json:"vec"`
|
|
Norm float32 `json:"norm"`
|
|
Source string `json:"source,omitempty"`
|
|
Session string `json:"session,omitempty"`
|
|
Repo string `json:"repo,omitempty"`
|
|
}
|
|
|
|
type SemanticNamespaceInfo struct {
|
|
Key string `json:"key"`
|
|
Namespace string `json:"namespace"`
|
|
UpdatedAt time.Time `json:"updated_at"`
|
|
SizeBytes int64 `json:"size_bytes"`
|
|
ItemsBytes int64 `json:"items_bytes"`
|
|
}
|
|
|
|
func (s *FileStore) AppendSemantic(namespace string, records []SemanticRecord) error {
|
|
if s == nil || s.BaseDir == "" {
|
|
return errors.New("memory store not configured")
|
|
}
|
|
if namespace == "" || len(records) == 0 {
|
|
return nil
|
|
}
|
|
dir := s.semanticDir(namespace)
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
return err
|
|
}
|
|
_ = s.writeSemanticNamespace(dir, namespace)
|
|
path := filepath.Join(dir, "items.jsonl")
|
|
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = f.Close() }()
|
|
|
|
seen := make(map[string]struct{}, len(records))
|
|
for i := range records {
|
|
r := records[i]
|
|
if r.TS.IsZero() {
|
|
r.TS = time.Now()
|
|
}
|
|
r.Text = strings.TrimSpace(RedactText(r.Text))
|
|
if r.Text == "" || len(r.Vec) == 0 {
|
|
continue
|
|
}
|
|
if _, ok := seen[r.Text]; ok {
|
|
continue
|
|
}
|
|
seen[r.Text] = struct{}{}
|
|
if r.Norm <= 0 {
|
|
r.Norm = vectorNorm(r.Vec)
|
|
}
|
|
if r.Norm <= 0 {
|
|
continue
|
|
}
|
|
b, err := json.Marshal(r)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
_, _ = f.Write(b)
|
|
_, _ = f.WriteString("\n")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *FileStore) SearchSemantic(namespace string, query []float32, maxChars int, maxSnippets int) ([]string, error) {
|
|
return s.SearchSemanticWithText(namespace, query, "", maxChars, maxSnippets)
|
|
}
|
|
|
|
func (s *FileStore) SearchSemanticWithText(namespace string, query []float32, queryText string, maxChars int, maxSnippets int) ([]string, error) {
|
|
if s == nil || s.BaseDir == "" {
|
|
return nil, errors.New("memory store not configured")
|
|
}
|
|
if namespace == "" || len(query) == 0 {
|
|
return nil, nil
|
|
}
|
|
if maxChars <= 0 {
|
|
maxChars = 3000
|
|
}
|
|
if maxSnippets <= 0 {
|
|
maxSnippets = 4
|
|
}
|
|
|
|
dir := s.semanticDir(namespace)
|
|
path := filepath.Join(dir, "items.jsonl")
|
|
data, err := readTailBytes(path, 2*1024*1024)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil, nil
|
|
}
|
|
return nil, err
|
|
}
|
|
lines := bytes.Split(data, []byte("\n"))
|
|
if len(lines) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
qn := vectorNorm(query)
|
|
if qn <= 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
tokens := []string{}
|
|
if semanticRerankEnabled() && strings.TrimSpace(queryText) != "" {
|
|
tokens = tokenizeSemanticQuery(queryText, 12)
|
|
}
|
|
now := time.Now()
|
|
maxAgeDays := semanticMaxAgeDays()
|
|
recencyWindow := semanticRecencyWindowDays(maxAgeDays)
|
|
keywordBoost := semanticKeywordBoost()
|
|
recencyBoost := semanticRecencyBoost()
|
|
|
|
type scored struct {
|
|
score float32
|
|
text string
|
|
ts time.Time
|
|
}
|
|
scoredSnips := make([]scored, 0, len(lines))
|
|
for i := range lines {
|
|
line := bytes.TrimSpace(lines[i])
|
|
if len(line) == 0 {
|
|
continue
|
|
}
|
|
var r SemanticRecord
|
|
if err := json.Unmarshal(line, &r); err != nil {
|
|
continue
|
|
}
|
|
if len(r.Vec) == 0 || r.Norm <= 0 {
|
|
continue
|
|
}
|
|
if maxAgeDays > 0 && !r.TS.IsZero() {
|
|
if now.Sub(r.TS) > time.Duration(maxAgeDays)*24*time.Hour {
|
|
continue
|
|
}
|
|
}
|
|
score := cosineSim(query, qn, r.Vec, r.Norm)
|
|
if score <= 0 {
|
|
continue
|
|
}
|
|
if len(tokens) > 0 {
|
|
overlap := semanticTokenOverlap(tokens, r.Text)
|
|
if overlap > 0 {
|
|
score *= 1 + keywordBoost*float32(overlap)/float32(len(tokens))
|
|
}
|
|
}
|
|
if recencyBoost > 0 {
|
|
ageBoost := semanticRecencyScore(now, r.TS, recencyWindow)
|
|
if ageBoost > 0 {
|
|
score *= 1 + recencyBoost*ageBoost
|
|
}
|
|
}
|
|
txt := strings.TrimSpace(r.Text)
|
|
if txt == "" {
|
|
continue
|
|
}
|
|
scoredSnips = append(scoredSnips, scored{score: score, text: txt, ts: r.TS})
|
|
}
|
|
|
|
if len(scoredSnips) == 0 {
|
|
return nil, nil
|
|
}
|
|
sort.Slice(scoredSnips, func(i, j int) bool { return scoredSnips[i].score > scoredSnips[j].score })
|
|
|
|
out := make([]string, 0, maxSnippets)
|
|
chars := 0
|
|
seen := make(map[string]struct{}, maxSnippets*2)
|
|
for _, s := range scoredSnips {
|
|
if len(out) >= maxSnippets {
|
|
break
|
|
}
|
|
key := s.text
|
|
if _, ok := seen[key]; ok {
|
|
continue
|
|
}
|
|
seen[key] = struct{}{}
|
|
snip := s.text
|
|
if len(snip) > 1200 {
|
|
snip = snip[:1200] + "\n...[truncated]..."
|
|
}
|
|
if chars+len(snip) > maxChars {
|
|
break
|
|
}
|
|
out = append(out, snip)
|
|
chars += len(snip) + 4
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (s *FileStore) ReadSemanticTail(namespace string, limit int) ([]SemanticRecord, error) {
|
|
if s == nil || s.BaseDir == "" {
|
|
return nil, errors.New("memory store not configured")
|
|
}
|
|
if namespace == "" {
|
|
return nil, nil
|
|
}
|
|
if limit <= 0 {
|
|
limit = 50
|
|
}
|
|
if limit > 200 {
|
|
limit = 200
|
|
}
|
|
|
|
dir := s.semanticDir(namespace)
|
|
path := filepath.Join(dir, "items.jsonl")
|
|
data, err := readTailBytes(path, 2*1024*1024)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil, nil
|
|
}
|
|
return nil, err
|
|
}
|
|
lines := bytes.Split(data, []byte("\n"))
|
|
if len(lines) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
out := make([]SemanticRecord, 0, limit)
|
|
for i := len(lines) - 1; i >= 0; i-- {
|
|
if len(out) >= limit {
|
|
break
|
|
}
|
|
line := bytes.TrimSpace(lines[i])
|
|
if len(line) == 0 {
|
|
continue
|
|
}
|
|
var r SemanticRecord
|
|
if err := json.Unmarshal(line, &r); err != nil {
|
|
continue
|
|
}
|
|
if strings.TrimSpace(r.Text) == "" {
|
|
continue
|
|
}
|
|
out = append(out, r)
|
|
}
|
|
// Reverse to chronological order
|
|
for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 {
|
|
out[i], out[j] = out[j], out[i]
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (s *FileStore) ListSemanticNamespaces(limit int) ([]SemanticNamespaceInfo, error) {
|
|
if s == nil || s.BaseDir == "" {
|
|
return nil, errors.New("memory store not configured")
|
|
}
|
|
root := filepath.Join(s.BaseDir, "semantic")
|
|
entries, err := os.ReadDir(root)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return []SemanticNamespaceInfo{}, nil
|
|
}
|
|
return nil, err
|
|
}
|
|
out := make([]SemanticNamespaceInfo, 0, len(entries))
|
|
for _, e := range entries {
|
|
if !e.IsDir() {
|
|
continue
|
|
}
|
|
dir := filepath.Join(root, e.Name())
|
|
itemsPath := filepath.Join(dir, "items.jsonl")
|
|
fi, err := os.Stat(itemsPath)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
ns := strings.TrimSpace(s.readSmallTextFile(filepath.Join(dir, "namespace.txt"), 2048))
|
|
info := SemanticNamespaceInfo{
|
|
Key: e.Name(),
|
|
Namespace: ns,
|
|
UpdatedAt: fi.ModTime(),
|
|
ItemsBytes: fi.Size(),
|
|
}
|
|
var total int64
|
|
if entries2, err := os.ReadDir(dir); err == nil {
|
|
for _, e2 := range entries2 {
|
|
if e2.IsDir() {
|
|
continue
|
|
}
|
|
if fi2, err := e2.Info(); err == nil {
|
|
total += fi2.Size()
|
|
}
|
|
}
|
|
}
|
|
info.SizeBytes = total
|
|
out = append(out, info)
|
|
}
|
|
sort.Slice(out, func(i, j int) bool {
|
|
return out[i].UpdatedAt.After(out[j].UpdatedAt)
|
|
})
|
|
if limit > 0 && len(out) > limit {
|
|
out = out[:limit]
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (s *FileStore) semanticDir(namespace string) string {
|
|
key := namespaceKey(namespace)
|
|
return filepath.Join(s.BaseDir, "semantic", key)
|
|
}
|
|
|
|
func (s *FileStore) writeSemanticNamespace(dir string, namespace string) error {
|
|
if strings.TrimSpace(namespace) == "" {
|
|
return nil
|
|
}
|
|
path := filepath.Join(dir, "namespace.txt")
|
|
if _, err := os.Stat(path); err == nil {
|
|
return nil
|
|
}
|
|
return os.WriteFile(path, []byte(namespace), 0o644)
|
|
}
|
|
|
|
func namespaceKey(namespace string) string {
|
|
h := sha256Hash(namespace)
|
|
return h[:16]
|
|
}
|
|
|
|
func sha256Hash(v string) string {
|
|
sum := sha256.Sum256([]byte(v))
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func vectorNorm(vec []float32) float32 {
|
|
var sum float64
|
|
for i := range vec {
|
|
v := float64(vec[i])
|
|
sum += v * v
|
|
}
|
|
if sum <= 0 {
|
|
return 0
|
|
}
|
|
return float32(math.Sqrt(sum))
|
|
}
|
|
|
|
func cosineSim(a []float32, aNorm float32, b []float32, bNorm float32) float32 {
|
|
if aNorm <= 0 || bNorm <= 0 {
|
|
return 0
|
|
}
|
|
n := len(a)
|
|
if len(b) < n {
|
|
n = len(b)
|
|
}
|
|
if n == 0 {
|
|
return 0
|
|
}
|
|
var dot float64
|
|
for i := 0; i < n; i++ {
|
|
dot += float64(a[i]) * float64(b[i])
|
|
}
|
|
return float32(dot) / (aNorm * bNorm)
|
|
}
|
|
|
|
func semanticMaxAgeDays() int {
|
|
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_AGE_DAYS")); v != "" {
|
|
if n, err := strconv.Atoi(v); err == nil {
|
|
return n
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func semanticRecencyWindowDays(maxAgeDays int) int {
|
|
if maxAgeDays > 0 {
|
|
return maxAgeDays
|
|
}
|
|
return 30
|
|
}
|
|
|
|
func semanticKeywordBoost() float32 {
|
|
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_KEYWORD_BOOST")); v != "" {
|
|
if f, err := strconv.ParseFloat(v, 32); err == nil {
|
|
return float32(f)
|
|
}
|
|
}
|
|
return 0.25
|
|
}
|
|
|
|
func semanticRecencyBoost() float32 {
|
|
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_RECENCY_BOOST")); v != "" {
|
|
if f, err := strconv.ParseFloat(v, 32); err == nil {
|
|
return float32(f)
|
|
}
|
|
}
|
|
return 0.15
|
|
}
|
|
|
|
func semanticRerankEnabled() bool {
|
|
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_RERANK")); v != "" {
|
|
if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func tokenizeSemanticQuery(q string, max int) []string {
|
|
q = strings.ToLower(q)
|
|
q = strings.Map(func(r rune) rune {
|
|
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' || r == '-' {
|
|
return r
|
|
}
|
|
return ' '
|
|
}, q)
|
|
parts := strings.Fields(q)
|
|
if len(parts) == 0 {
|
|
return nil
|
|
}
|
|
stop := map[string]struct{}{
|
|
"the": {}, "and": {}, "for": {}, "with": {}, "that": {}, "this": {}, "from": {}, "into": {}, "what": {}, "how": {},
|
|
"you": {}, "your": {}, "are": {}, "was": {}, "were": {}, "can": {}, "could": {}, "should": {}, "would": {}, "will": {},
|
|
}
|
|
out := make([]string, 0, max)
|
|
seen := make(map[string]struct{}, max*2)
|
|
for _, p := range parts {
|
|
if len(p) < 3 {
|
|
continue
|
|
}
|
|
if _, ok := stop[p]; ok {
|
|
continue
|
|
}
|
|
if _, ok := seen[p]; ok {
|
|
continue
|
|
}
|
|
seen[p] = struct{}{}
|
|
out = append(out, p)
|
|
if len(out) >= max {
|
|
break
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func semanticTokenOverlap(tokens []string, text string) int {
|
|
if len(tokens) == 0 || strings.TrimSpace(text) == "" {
|
|
return 0
|
|
}
|
|
low := strings.ToLower(text)
|
|
count := 0
|
|
for _, t := range tokens {
|
|
if t == "" {
|
|
continue
|
|
}
|
|
if strings.Contains(low, t) {
|
|
count++
|
|
}
|
|
}
|
|
return count
|
|
}
|
|
|
|
func semanticRecencyScore(now time.Time, ts time.Time, windowDays int) float32 {
|
|
if ts.IsZero() || windowDays <= 0 {
|
|
return 0
|
|
}
|
|
age := now.Sub(ts)
|
|
if age <= 0 {
|
|
return 1
|
|
}
|
|
window := time.Duration(windowDays) * 24 * time.Hour
|
|
if age >= window {
|
|
return 0
|
|
}
|
|
return float32(1 - float64(age)/float64(window))
|
|
}
|