mirror of
https://github.com/Finesssee/ProxyPilot.git
synced 2026-11-18 18:04:20 +00:00
test(memory): add unit tests for file store, pruning, and semantic search
Covers event serialization, file operations, token-based pruning, recency preservation, and semantic similarity scoring.
This commit is contained in:
@@ -0,0 +1,641 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestFileStore_Append_CreatesFile(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
session string
|
||||
events []Event
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "creates file with single event",
|
||||
session: "test-session-1",
|
||||
events: []Event{
|
||||
{
|
||||
TS: time.Now(),
|
||||
Kind: "message",
|
||||
Role: "user",
|
||||
Text: "Hello, world!",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "creates file with event that has zero timestamp",
|
||||
session: "test-session-2",
|
||||
events: []Event{
|
||||
{
|
||||
Kind: "message",
|
||||
Role: "user",
|
||||
Text: "This event has no timestamp set",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "empty session returns nil without creating file",
|
||||
session: "",
|
||||
events: []Event{
|
||||
{Kind: "message", Text: "test"},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "empty events returns nil without creating file",
|
||||
session: "test-session-3",
|
||||
events: []Event{},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
|
||||
err := store.Append(tt.session, tt.events)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Append() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
|
||||
if tt.session == "" || len(tt.events) == 0 {
|
||||
// File should not be created
|
||||
eventsPath := filepath.Join(tmpDir, "sessions", tt.session, "events.jsonl")
|
||||
if _, err := os.Stat(eventsPath); !os.IsNotExist(err) {
|
||||
t.Errorf("File should not exist for empty session or events")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Verify file was created
|
||||
eventsPath := filepath.Join(tmpDir, "sessions", tt.session, "events.jsonl")
|
||||
if _, err := os.Stat(eventsPath); os.IsNotExist(err) {
|
||||
t.Errorf("events.jsonl was not created")
|
||||
return
|
||||
}
|
||||
|
||||
// Verify content is valid JSONL
|
||||
data, err := os.ReadFile(eventsPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read events file: %v", err)
|
||||
}
|
||||
|
||||
lines := strings.Split(strings.TrimSpace(string(data)), "\n")
|
||||
if len(lines) != len(tt.events) {
|
||||
t.Errorf("Expected %d lines, got %d", len(tt.events), len(lines))
|
||||
}
|
||||
|
||||
for i, line := range lines {
|
||||
var e Event
|
||||
if err := json.Unmarshal([]byte(line), &e); err != nil {
|
||||
t.Errorf("Line %d is not valid JSON: %v", i, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileStore_Append_MultipleEvents(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
session string
|
||||
eventSets [][]Event
|
||||
wantTotal int
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "append multiple events in single call",
|
||||
session: "multi-event-session",
|
||||
eventSets: [][]Event{
|
||||
{
|
||||
{Kind: "message", Role: "user", Text: "First message"},
|
||||
{Kind: "message", Role: "assistant", Text: "First response"},
|
||||
{Kind: "message", Role: "user", Text: "Second message"},
|
||||
},
|
||||
},
|
||||
wantTotal: 3,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "append events across multiple calls",
|
||||
session: "multiple-calls-session",
|
||||
eventSets: [][]Event{
|
||||
{{Kind: "message", Role: "user", Text: "Message 1"}},
|
||||
{{Kind: "message", Role: "assistant", Text: "Response 1"}},
|
||||
{{Kind: "message", Role: "user", Text: "Message 2"}},
|
||||
{{Kind: "message", Role: "assistant", Text: "Response 2"}},
|
||||
},
|
||||
wantTotal: 4,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "append mixed events with metadata",
|
||||
session: "mixed-events-session",
|
||||
eventSets: [][]Event{
|
||||
{
|
||||
{Kind: "system", Text: "Session started"},
|
||||
{Kind: "message", Role: "user", Text: "Hello", Meta: map[string]string{"lang": "en"}},
|
||||
{Kind: "tool_use", Role: "assistant", Type: "bash", Text: "ls"},
|
||||
{Kind: "tool_result", Role: "user", Type: "bash", Text: "file.txt"},
|
||||
},
|
||||
},
|
||||
wantTotal: 4,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "append preserves order",
|
||||
session: "order-session",
|
||||
eventSets: [][]Event{
|
||||
{
|
||||
{Kind: "message", Role: "user", Text: "First"},
|
||||
{Kind: "message", Role: "user", Text: "Second"},
|
||||
{Kind: "message", Role: "user", Text: "Third"},
|
||||
},
|
||||
},
|
||||
wantTotal: 3,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
|
||||
for _, events := range tt.eventSets {
|
||||
err := store.Append(tt.session, events)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Append() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Read and verify the file
|
||||
eventsPath := filepath.Join(tmpDir, "sessions", tt.session, "events.jsonl")
|
||||
data, err := os.ReadFile(eventsPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read events file: %v", err)
|
||||
}
|
||||
|
||||
lines := strings.Split(strings.TrimSpace(string(data)), "\n")
|
||||
if len(lines) != tt.wantTotal {
|
||||
t.Errorf("Expected %d events, got %d", tt.wantTotal, len(lines))
|
||||
}
|
||||
|
||||
// Verify each line is valid JSON and check order for order test
|
||||
if tt.name == "append preserves order" {
|
||||
expectedTexts := []string{"First", "Second", "Third"}
|
||||
for i, line := range lines {
|
||||
var e Event
|
||||
if err := json.Unmarshal([]byte(line), &e); err != nil {
|
||||
t.Errorf("Line %d is not valid JSON: %v", i, err)
|
||||
continue
|
||||
}
|
||||
if e.Text != expectedTexts[i] {
|
||||
t.Errorf("Event %d: expected text %q, got %q", i, expectedTexts[i], e.Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileStore_Append_RedactsSecrets(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
inputText string
|
||||
wantRedact bool
|
||||
}{
|
||||
{
|
||||
name: "redacts sk- API keys",
|
||||
inputText: "My key is sk-1234567890abcdef",
|
||||
wantRedact: true,
|
||||
},
|
||||
{
|
||||
name: "redacts bearer tokens",
|
||||
inputText: "Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9",
|
||||
wantRedact: true,
|
||||
},
|
||||
{
|
||||
name: "redacts AIza keys",
|
||||
inputText: "API key: AIzaSyB1234567890abcdefghij",
|
||||
wantRedact: true,
|
||||
},
|
||||
{
|
||||
name: "does not modify normal text",
|
||||
inputText: "This is a normal message without secrets",
|
||||
wantRedact: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
session := "redact-test"
|
||||
|
||||
err := store.Append(session, []Event{
|
||||
{Kind: "message", Text: tt.inputText},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Append() error = %v", err)
|
||||
}
|
||||
|
||||
// Read the stored event
|
||||
eventsPath := filepath.Join(tmpDir, "sessions", session, "events.jsonl")
|
||||
data, err := os.ReadFile(eventsPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read events file: %v", err)
|
||||
}
|
||||
|
||||
var e Event
|
||||
if err := json.Unmarshal(data, &e); err != nil {
|
||||
t.Fatalf("Failed to parse event: %v", err)
|
||||
}
|
||||
|
||||
hasRedacted := strings.Contains(e.Text, "[REDACTED]")
|
||||
if tt.wantRedact && !hasRedacted {
|
||||
t.Errorf("Expected text to be redacted, got: %s", e.Text)
|
||||
}
|
||||
if !tt.wantRedact && hasRedacted {
|
||||
t.Errorf("Text should not be redacted, got: %s", e.Text)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileStore_Search_ExactMatch(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
events []Event
|
||||
query string
|
||||
wantCount int
|
||||
wantMatch string
|
||||
}{
|
||||
{
|
||||
name: "finds exact keyword match",
|
||||
events: []Event{
|
||||
{Kind: "message", Role: "user", Text: "How do I configure the database?"},
|
||||
{Kind: "message", Role: "assistant", Text: "You can configure the database in config.yaml"},
|
||||
{Kind: "message", Role: "user", Text: "Thanks, what about logging?"},
|
||||
},
|
||||
query: "database",
|
||||
wantCount: 2,
|
||||
wantMatch: "database",
|
||||
},
|
||||
{
|
||||
name: "case insensitive search",
|
||||
events: []Event{
|
||||
{Kind: "message", Text: "The DATABASE connection failed"},
|
||||
{Kind: "message", Text: "Database retry succeeded"},
|
||||
{Kind: "message", Text: "No matches here"},
|
||||
},
|
||||
query: "database",
|
||||
wantCount: 2,
|
||||
wantMatch: "",
|
||||
},
|
||||
{
|
||||
name: "no matches returns empty",
|
||||
events: []Event{
|
||||
{Kind: "message", Text: "Hello world"},
|
||||
{Kind: "message", Text: "Goodbye world"},
|
||||
},
|
||||
query: "nonexistent",
|
||||
wantCount: 0,
|
||||
wantMatch: "",
|
||||
},
|
||||
{
|
||||
name: "multiple keywords - matches any keyword",
|
||||
events: []Event{
|
||||
{Kind: "message", Text: "Error in authentication module"},
|
||||
{Kind: "message", Text: "Authentication failed for user"},
|
||||
{Kind: "message", Text: "Module loaded successfully"},
|
||||
},
|
||||
query: "authentication module",
|
||||
wantCount: 3, // all three contain at least one keyword
|
||||
wantMatch: "",
|
||||
},
|
||||
{
|
||||
name: "empty query returns nil",
|
||||
events: []Event{
|
||||
{Kind: "message", Text: "Some text"},
|
||||
},
|
||||
query: "",
|
||||
wantCount: 0,
|
||||
wantMatch: "",
|
||||
},
|
||||
{
|
||||
name: "whitespace query returns nil",
|
||||
events: []Event{
|
||||
{Kind: "message", Text: "Some text"},
|
||||
},
|
||||
query: " ",
|
||||
wantCount: 0,
|
||||
wantMatch: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
session := "search-test"
|
||||
|
||||
// Append test events
|
||||
if err := store.Append(session, tt.events); err != nil {
|
||||
t.Fatalf("Append() error = %v", err)
|
||||
}
|
||||
|
||||
// Search
|
||||
results, err := store.Search(session, tt.query, 6000, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("Search() error = %v", err)
|
||||
}
|
||||
|
||||
if len(results) != tt.wantCount {
|
||||
t.Errorf("Search() returned %d results, want %d", len(results), tt.wantCount)
|
||||
}
|
||||
|
||||
if tt.wantMatch != "" {
|
||||
found := false
|
||||
for _, r := range results {
|
||||
if strings.Contains(strings.ToLower(r), tt.wantMatch) {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("Search results should contain %q", tt.wantMatch)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileStore_Search_MaxSnippets(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
eventCount int
|
||||
maxSnippets int
|
||||
wantMax int
|
||||
}{
|
||||
{
|
||||
name: "limits to maxSnippets",
|
||||
eventCount: 20,
|
||||
maxSnippets: 5,
|
||||
wantMax: 5,
|
||||
},
|
||||
{
|
||||
name: "returns all if under limit",
|
||||
eventCount: 3,
|
||||
maxSnippets: 10,
|
||||
wantMax: 3,
|
||||
},
|
||||
{
|
||||
name: "default maxSnippets when zero",
|
||||
eventCount: 20,
|
||||
maxSnippets: 0,
|
||||
wantMax: 8, // default is 8
|
||||
},
|
||||
{
|
||||
name: "maxSnippets of 1",
|
||||
eventCount: 10,
|
||||
maxSnippets: 1,
|
||||
wantMax: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
session := "maxsnippets-test"
|
||||
|
||||
// Create events with searchable text
|
||||
events := make([]Event, tt.eventCount)
|
||||
for i := 0; i < tt.eventCount; i++ {
|
||||
events[i] = Event{
|
||||
Kind: "message",
|
||||
Role: "user",
|
||||
Text: "This is a searchable test message number " + string(rune('A'+i)),
|
||||
}
|
||||
}
|
||||
|
||||
if err := store.Append(session, events); err != nil {
|
||||
t.Fatalf("Append() error = %v", err)
|
||||
}
|
||||
|
||||
results, err := store.Search(session, "searchable test message", 100000, tt.maxSnippets)
|
||||
if err != nil {
|
||||
t.Fatalf("Search() error = %v", err)
|
||||
}
|
||||
|
||||
if len(results) > tt.wantMax {
|
||||
t.Errorf("Search() returned %d results, want at most %d", len(results), tt.wantMax)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileStore_Search_MaxChars(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
events []Event
|
||||
maxChars int
|
||||
wantUnder bool
|
||||
}{
|
||||
{
|
||||
name: "respects maxChars limit",
|
||||
events: []Event{
|
||||
{Kind: "message", Text: strings.Repeat("searchable text ", 100)},
|
||||
{Kind: "message", Text: strings.Repeat("more searchable text ", 100)},
|
||||
},
|
||||
maxChars: 500,
|
||||
wantUnder: true,
|
||||
},
|
||||
{
|
||||
name: "default maxChars when zero",
|
||||
events: []Event{
|
||||
{Kind: "message", Text: "searchable text"},
|
||||
},
|
||||
maxChars: 0,
|
||||
wantUnder: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
session := "maxchars-test"
|
||||
|
||||
if err := store.Append(session, tt.events); err != nil {
|
||||
t.Fatalf("Append() error = %v", err)
|
||||
}
|
||||
|
||||
results, err := store.Search(session, "searchable text", tt.maxChars, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("Search() error = %v", err)
|
||||
}
|
||||
|
||||
if tt.wantUnder {
|
||||
totalChars := 0
|
||||
for _, r := range results {
|
||||
totalChars += len(r)
|
||||
}
|
||||
|
||||
limit := tt.maxChars
|
||||
if limit <= 0 {
|
||||
limit = 6000 // default
|
||||
}
|
||||
|
||||
if totalChars > limit {
|
||||
t.Errorf("Total chars %d exceeds maxChars %d", totalChars, limit)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileStore_Search_NonExistentSession(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
|
||||
results, err := store.Search("nonexistent-session", "query", 6000, 10)
|
||||
if err != nil {
|
||||
t.Errorf("Search() should not error for nonexistent session, got: %v", err)
|
||||
}
|
||||
if len(results) != 0 {
|
||||
t.Errorf("Search() should return empty for nonexistent session, got %d results", len(results))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileStore_NilStore(t *testing.T) {
|
||||
var store *FileStore
|
||||
|
||||
t.Run("Append on nil store", func(t *testing.T) {
|
||||
err := store.Append("session", []Event{{Kind: "message", Text: "test"}})
|
||||
if err == nil {
|
||||
t.Error("Expected error for nil store, got nil")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Search on nil store", func(t *testing.T) {
|
||||
_, err := store.Search("session", "query", 100, 10)
|
||||
if err == nil {
|
||||
t.Error("Expected error for nil store, got nil")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFileStore_EmptyBaseDir(t *testing.T) {
|
||||
store := NewFileStore("")
|
||||
|
||||
t.Run("Append with empty BaseDir", func(t *testing.T) {
|
||||
err := store.Append("session", []Event{{Kind: "message", Text: "test"}})
|
||||
if err == nil {
|
||||
t.Error("Expected error for empty BaseDir, got nil")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Search with empty BaseDir", func(t *testing.T) {
|
||||
_, err := store.Search("session", "query", 100, 10)
|
||||
if err == nil {
|
||||
t.Error("Expected error for empty BaseDir, got nil")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFileStore_SearchCache(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
session := "cache-test"
|
||||
|
||||
events := []Event{
|
||||
{Kind: "message", Role: "user", Text: "This is a cacheable message"},
|
||||
}
|
||||
|
||||
if err := store.Append(session, events); err != nil {
|
||||
t.Fatalf("Append() error = %v", err)
|
||||
}
|
||||
|
||||
// First search populates cache
|
||||
results1, err := store.Search(session, "cacheable", 6000, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("Search() error = %v", err)
|
||||
}
|
||||
|
||||
// Second search should use cache (same query)
|
||||
results2, err := store.Search(session, "cacheable", 6000, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("Search() error = %v", err)
|
||||
}
|
||||
|
||||
if len(results1) != len(results2) {
|
||||
t.Errorf("Cached result count mismatch: %d vs %d", len(results1), len(results2))
|
||||
}
|
||||
|
||||
// Append invalidates cache
|
||||
if err := store.Append(session, []Event{{Kind: "message", Text: "New cacheable event"}}); err != nil {
|
||||
t.Fatalf("Append() error = %v", err)
|
||||
}
|
||||
|
||||
// Search after append should get fresh results
|
||||
results3, err := store.Search(session, "cacheable", 6000, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("Search() error = %v", err)
|
||||
}
|
||||
|
||||
if len(results3) <= len(results1) {
|
||||
t.Logf("Results after append: %d (was %d)", len(results3), len(results1))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileStore_SessionDir(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
session string
|
||||
wantDir bool
|
||||
}{
|
||||
{
|
||||
name: "normal session name",
|
||||
session: "my-session-123",
|
||||
wantDir: true,
|
||||
},
|
||||
{
|
||||
name: "session with special characters",
|
||||
session: "session/with\\special:chars*",
|
||||
wantDir: true,
|
||||
},
|
||||
{
|
||||
name: "empty session",
|
||||
session: "",
|
||||
wantDir: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
|
||||
dir := store.SessionDir(tt.session)
|
||||
if tt.wantDir && dir == "" {
|
||||
t.Error("SessionDir() returned empty string")
|
||||
}
|
||||
if tt.wantDir && !strings.HasPrefix(dir, tmpDir) {
|
||||
t.Errorf("SessionDir() = %q, should be under %q", dir, tmpDir)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,610 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPrune_ByTokenCount(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
sessionCount int
|
||||
eventsPerSession int
|
||||
maxBytesPerSession int64
|
||||
wantSessionsTrimmed int
|
||||
}{
|
||||
{
|
||||
name: "trims large sessions",
|
||||
sessionCount: 2,
|
||||
eventsPerSession: 100,
|
||||
maxBytesPerSession: 500,
|
||||
wantSessionsTrimmed: 2,
|
||||
},
|
||||
{
|
||||
name: "no trimming needed for small sessions",
|
||||
sessionCount: 2,
|
||||
eventsPerSession: 2,
|
||||
maxBytesPerSession: 100000,
|
||||
wantSessionsTrimmed: 0,
|
||||
},
|
||||
{
|
||||
name: "zero maxBytes skips trimming",
|
||||
sessionCount: 2,
|
||||
eventsPerSession: 100,
|
||||
maxBytesPerSession: 0,
|
||||
wantSessionsTrimmed: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
|
||||
// Create sessions with events
|
||||
for i := 0; i < tt.sessionCount; i++ {
|
||||
session := "session-" + string(rune('A'+i))
|
||||
events := make([]Event, tt.eventsPerSession)
|
||||
for j := 0; j < tt.eventsPerSession; j++ {
|
||||
events[j] = Event{
|
||||
TS: time.Now(),
|
||||
Kind: "message",
|
||||
Role: "user",
|
||||
Text: strings.Repeat("This is a test message with some content. ", 10),
|
||||
}
|
||||
}
|
||||
if err := store.Append(session, events); err != nil {
|
||||
t.Fatalf("Append() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Prune sessions
|
||||
result, err := store.PruneSessions(0, 0, tt.maxBytesPerSession)
|
||||
if err != nil {
|
||||
t.Fatalf("PruneSessions() error = %v", err)
|
||||
}
|
||||
|
||||
if result.SessionsTrimmed != tt.wantSessionsTrimmed {
|
||||
t.Errorf("SessionsTrimmed = %d, want %d", result.SessionsTrimmed, tt.wantSessionsTrimmed)
|
||||
}
|
||||
|
||||
if result.SessionsTrimmed > 0 && result.BytesFreed <= 0 {
|
||||
t.Error("Expected BytesFreed > 0 when sessions are trimmed")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrune_PreservesRecent(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
sessionAges []int // days ago
|
||||
maxAgeDays int
|
||||
maxSessions int
|
||||
wantRemoved int
|
||||
wantKept []string
|
||||
}{
|
||||
{
|
||||
name: "preserves recent sessions by age",
|
||||
sessionAges: []int{1, 5, 10, 20, 40}, // days ago
|
||||
maxAgeDays: 30,
|
||||
maxSessions: 0,
|
||||
wantRemoved: 1, // only 40 days old is removed
|
||||
wantKept: []string{"session-0", "session-1", "session-2", "session-3"},
|
||||
},
|
||||
{
|
||||
name: "preserves by max sessions count",
|
||||
sessionAges: []int{1, 2, 3, 4, 5},
|
||||
maxAgeDays: 0,
|
||||
maxSessions: 3,
|
||||
wantRemoved: 2, // keeps 3 most recent
|
||||
wantKept: []string{"session-0", "session-1", "session-2"},
|
||||
},
|
||||
{
|
||||
name: "combines age and count limits",
|
||||
sessionAges: []int{1, 5, 35, 40},
|
||||
maxAgeDays: 30,
|
||||
maxSessions: 2,
|
||||
wantRemoved: 2, // 35 and 40 days old are removed
|
||||
wantKept: []string{"session-0", "session-1"},
|
||||
},
|
||||
{
|
||||
name: "no pruning when all within limits",
|
||||
sessionAges: []int{1, 2, 3},
|
||||
maxAgeDays: 30,
|
||||
maxSessions: 10,
|
||||
wantRemoved: 0,
|
||||
wantKept: []string{"session-0", "session-1", "session-2"},
|
||||
},
|
||||
{
|
||||
name: "removes all old sessions",
|
||||
sessionAges: []int{100, 200, 300},
|
||||
maxAgeDays: 30,
|
||||
maxSessions: 0,
|
||||
wantRemoved: 3,
|
||||
wantKept: []string{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
|
||||
// Create sessions with different ages
|
||||
for i, daysAgo := range tt.sessionAges {
|
||||
session := "session-" + string(rune('0'+i))
|
||||
sessionDir := filepath.Join(tmpDir, "sessions", session)
|
||||
if err := os.MkdirAll(sessionDir, 0o755); err != nil {
|
||||
t.Fatalf("Failed to create session dir: %v", err)
|
||||
}
|
||||
|
||||
// Create events file
|
||||
eventsPath := filepath.Join(sessionDir, "events.jsonl")
|
||||
if err := os.WriteFile(eventsPath, []byte(`{"kind":"message","text":"test"}`+"\n"), 0o644); err != nil {
|
||||
t.Fatalf("Failed to create events file: %v", err)
|
||||
}
|
||||
|
||||
// Set modification time
|
||||
modTime := time.Now().Add(-time.Duration(daysAgo) * 24 * time.Hour)
|
||||
if err := os.Chtimes(eventsPath, modTime, modTime); err != nil {
|
||||
t.Fatalf("Failed to set file times: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Prune sessions
|
||||
result, err := store.PruneSessions(tt.maxAgeDays, tt.maxSessions, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("PruneSessions() error = %v", err)
|
||||
}
|
||||
|
||||
if result.SessionsRemoved != tt.wantRemoved {
|
||||
t.Errorf("SessionsRemoved = %d, want %d", result.SessionsRemoved, tt.wantRemoved)
|
||||
}
|
||||
|
||||
// Verify kept sessions exist
|
||||
for _, session := range tt.wantKept {
|
||||
sessionDir := filepath.Join(tmpDir, "sessions", session)
|
||||
if _, err := os.Stat(sessionDir); os.IsNotExist(err) {
|
||||
t.Errorf("Session %q should have been preserved but was removed", session)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify removed sessions don't exist
|
||||
sessions, _ := store.ListSessions(0)
|
||||
if len(sessions) != len(tt.wantKept) {
|
||||
t.Errorf("Expected %d sessions remaining, got %d", len(tt.wantKept), len(sessions))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrune_SemanticNamespaces(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
namespaceCount int
|
||||
recordsPerNs int
|
||||
maxAgeDays int
|
||||
maxNamespaces int
|
||||
maxBytesPerNs int64
|
||||
wantRemoved int
|
||||
wantTrimmed int
|
||||
}{
|
||||
{
|
||||
name: "removes old namespaces",
|
||||
namespaceCount: 3,
|
||||
recordsPerNs: 5,
|
||||
maxAgeDays: 30,
|
||||
maxNamespaces: 0,
|
||||
maxBytesPerNs: 0,
|
||||
wantRemoved: 0, // all are recent
|
||||
wantTrimmed: 0,
|
||||
},
|
||||
{
|
||||
name: "limits namespace count",
|
||||
namespaceCount: 5,
|
||||
recordsPerNs: 5,
|
||||
maxAgeDays: 0,
|
||||
maxNamespaces: 3,
|
||||
maxBytesPerNs: 0,
|
||||
wantRemoved: 2,
|
||||
wantTrimmed: 0,
|
||||
},
|
||||
{
|
||||
name: "trims large namespaces",
|
||||
namespaceCount: 2,
|
||||
recordsPerNs: 500,
|
||||
maxAgeDays: 0,
|
||||
maxNamespaces: 0,
|
||||
maxBytesPerNs: 2000,
|
||||
wantRemoved: 0,
|
||||
wantTrimmed: 2,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
|
||||
// Create namespaces with records
|
||||
for i := 0; i < tt.namespaceCount; i++ {
|
||||
namespace := "namespace-" + string(rune('A'+i))
|
||||
records := make([]SemanticRecord, tt.recordsPerNs)
|
||||
for j := 0; j < tt.recordsPerNs; j++ {
|
||||
// Make each text unique to avoid deduplication
|
||||
records[j] = SemanticRecord{
|
||||
TS: time.Now(),
|
||||
Text: strings.Repeat("Test record content ", 20) + " " + string(rune('A'+i)) + string(rune('0'+j%10)),
|
||||
Vec: []float32{float32(i), float32(j), 0.5},
|
||||
}
|
||||
}
|
||||
if err := store.AppendSemantic(namespace, records); err != nil {
|
||||
t.Fatalf("AppendSemantic() error = %v", err)
|
||||
}
|
||||
// Add small delay to ensure different modification times
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
// Prune semantic namespaces
|
||||
result, err := store.PruneSemantic(tt.maxAgeDays, tt.maxNamespaces, tt.maxBytesPerNs)
|
||||
if err != nil {
|
||||
t.Fatalf("PruneSemantic() error = %v", err)
|
||||
}
|
||||
|
||||
if result.SemanticNamespacesRemoved != tt.wantRemoved {
|
||||
t.Errorf("SemanticNamespacesRemoved = %d, want %d", result.SemanticNamespacesRemoved, tt.wantRemoved)
|
||||
}
|
||||
|
||||
if result.SemanticNamespacesTrimmed != tt.wantTrimmed {
|
||||
t.Errorf("SemanticNamespacesTrimmed = %d, want %d", result.SemanticNamespacesTrimmed, tt.wantTrimmed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimJSONLFile(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
content string
|
||||
maxBytes int64
|
||||
wantTrim bool
|
||||
wantFreed bool
|
||||
}{
|
||||
{
|
||||
name: "trims file exceeding maxBytes",
|
||||
content: `{"a":1}` + "\n" + `{"b":2}` + "\n" + `{"c":3}` + "\n" + `{"d":4}` + "\n",
|
||||
maxBytes: 20,
|
||||
wantTrim: true,
|
||||
wantFreed: true,
|
||||
},
|
||||
{
|
||||
name: "no trim needed for small file",
|
||||
content: `{"a":1}` + "\n",
|
||||
maxBytes: 1000,
|
||||
wantTrim: false,
|
||||
wantFreed: false,
|
||||
},
|
||||
{
|
||||
name: "zero maxBytes skips trimming",
|
||||
content: `{"a":1}` + "\n" + `{"b":2}` + "\n",
|
||||
maxBytes: 0,
|
||||
wantTrim: false,
|
||||
wantFreed: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
path := filepath.Join(tmpDir, "test.jsonl")
|
||||
|
||||
if err := os.WriteFile(path, []byte(tt.content), 0o644); err != nil {
|
||||
t.Fatalf("Failed to write test file: %v", err)
|
||||
}
|
||||
|
||||
originalSize := int64(len(tt.content))
|
||||
trimmed, freed := trimJSONLFile(path, tt.maxBytes)
|
||||
|
||||
if trimmed != tt.wantTrim {
|
||||
t.Errorf("trimJSONLFile() trimmed = %v, want %v", trimmed, tt.wantTrim)
|
||||
}
|
||||
|
||||
if tt.wantFreed && freed <= 0 {
|
||||
t.Errorf("Expected freed > 0, got %d", freed)
|
||||
}
|
||||
|
||||
if !tt.wantFreed && freed != 0 {
|
||||
t.Errorf("Expected freed = 0, got %d", freed)
|
||||
}
|
||||
|
||||
// Verify file was actually trimmed
|
||||
if tt.wantTrim {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read trimmed file: %v", err)
|
||||
}
|
||||
if int64(len(data)) >= originalSize {
|
||||
t.Errorf("File was not actually trimmed: %d >= %d", len(data), originalSize)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneResult_Fields(t *testing.T) {
|
||||
result := PruneResult{
|
||||
SessionsRemoved: 5,
|
||||
SessionsTrimmed: 3,
|
||||
SemanticNamespacesRemoved: 2,
|
||||
SemanticNamespacesTrimmed: 1,
|
||||
BytesFreed: 1024 * 1024,
|
||||
}
|
||||
|
||||
if result.SessionsRemoved != 5 {
|
||||
t.Errorf("SessionsRemoved = %d, want 5", result.SessionsRemoved)
|
||||
}
|
||||
if result.SessionsTrimmed != 3 {
|
||||
t.Errorf("SessionsTrimmed = %d, want 3", result.SessionsTrimmed)
|
||||
}
|
||||
if result.SemanticNamespacesRemoved != 2 {
|
||||
t.Errorf("SemanticNamespacesRemoved = %d, want 2", result.SemanticNamespacesRemoved)
|
||||
}
|
||||
if result.SemanticNamespacesTrimmed != 1 {
|
||||
t.Errorf("SemanticNamespacesTrimmed = %d, want 1", result.SemanticNamespacesTrimmed)
|
||||
}
|
||||
if result.BytesFreed != 1024*1024 {
|
||||
t.Errorf("BytesFreed = %d, want %d", result.BytesFreed, 1024*1024)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionInfo_Fields(t *testing.T) {
|
||||
now := time.Now()
|
||||
info := SessionInfo{
|
||||
Key: "test-session",
|
||||
Path: "/path/to/session",
|
||||
UpdatedAt: now,
|
||||
SizeBytes: 2048,
|
||||
EventsBytes: 1024,
|
||||
HasSummary: true,
|
||||
HasTodo: true,
|
||||
HasPinned: false,
|
||||
HasAnchorPending: true,
|
||||
SemanticDisabled: false,
|
||||
}
|
||||
|
||||
if info.Key != "test-session" {
|
||||
t.Errorf("Key = %q, want %q", info.Key, "test-session")
|
||||
}
|
||||
if info.Path != "/path/to/session" {
|
||||
t.Errorf("Path = %q, want %q", info.Path, "/path/to/session")
|
||||
}
|
||||
if !info.UpdatedAt.Equal(now) {
|
||||
t.Errorf("UpdatedAt mismatch")
|
||||
}
|
||||
if info.SizeBytes != 2048 {
|
||||
t.Errorf("SizeBytes = %d, want 2048", info.SizeBytes)
|
||||
}
|
||||
if info.EventsBytes != 1024 {
|
||||
t.Errorf("EventsBytes = %d, want 1024", info.EventsBytes)
|
||||
}
|
||||
if !info.HasSummary {
|
||||
t.Error("HasSummary should be true")
|
||||
}
|
||||
if !info.HasTodo {
|
||||
t.Error("HasTodo should be true")
|
||||
}
|
||||
if info.HasPinned {
|
||||
t.Error("HasPinned should be false")
|
||||
}
|
||||
if !info.HasAnchorPending {
|
||||
t.Error("HasAnchorPending should be true")
|
||||
}
|
||||
if info.SemanticDisabled {
|
||||
t.Error("SemanticDisabled should be false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestListSessions(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
|
||||
// Create some sessions
|
||||
sessions := []string{"alpha", "beta", "gamma"}
|
||||
for _, session := range sessions {
|
||||
events := []Event{{Kind: "message", Text: "test"}}
|
||||
if err := store.Append(session, events); err != nil {
|
||||
t.Fatalf("Append() error = %v", err)
|
||||
}
|
||||
// Small delay to ensure different mod times
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
// List all sessions
|
||||
list, err := store.ListSessions(0)
|
||||
if err != nil {
|
||||
t.Fatalf("ListSessions() error = %v", err)
|
||||
}
|
||||
|
||||
if len(list) != len(sessions) {
|
||||
t.Errorf("ListSessions() returned %d, want %d", len(list), len(sessions))
|
||||
}
|
||||
|
||||
// List with limit
|
||||
limited, err := store.ListSessions(2)
|
||||
if err != nil {
|
||||
t.Fatalf("ListSessions() error = %v", err)
|
||||
}
|
||||
|
||||
if len(limited) != 2 {
|
||||
t.Errorf("ListSessions(2) returned %d, want 2", len(limited))
|
||||
}
|
||||
|
||||
// Verify most recent is first (gamma was created last)
|
||||
if len(limited) > 0 && limited[0].Key != "gamma" {
|
||||
t.Errorf("Most recent session should be 'gamma', got %q", limited[0].Key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetSessionInfo(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
session := "info-test"
|
||||
|
||||
// Create session with various files
|
||||
events := []Event{{Kind: "message", Text: "test event"}}
|
||||
if err := store.Append(session, events); err != nil {
|
||||
t.Fatalf("Append() error = %v", err)
|
||||
}
|
||||
|
||||
if err := store.WriteSummary(session, "Test summary", 0); err != nil {
|
||||
t.Fatalf("WriteSummary() error = %v", err)
|
||||
}
|
||||
|
||||
if err := store.WriteTodo(session, "Test todo", 0); err != nil {
|
||||
t.Fatalf("WriteTodo() error = %v", err)
|
||||
}
|
||||
|
||||
// Get session info
|
||||
info, err := store.GetSessionInfo(session)
|
||||
if err != nil {
|
||||
t.Fatalf("GetSessionInfo() error = %v", err)
|
||||
}
|
||||
|
||||
if info.Key != session {
|
||||
t.Errorf("Key = %q, want %q", info.Key, session)
|
||||
}
|
||||
if !info.HasSummary {
|
||||
t.Error("HasSummary should be true")
|
||||
}
|
||||
if !info.HasTodo {
|
||||
t.Error("HasTodo should be true")
|
||||
}
|
||||
if info.SizeBytes <= 0 {
|
||||
t.Error("SizeBytes should be > 0")
|
||||
}
|
||||
if info.EventsBytes <= 0 {
|
||||
t.Error("EventsBytes should be > 0")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadEventTail(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
session := "tail-test"
|
||||
|
||||
// Create many events
|
||||
events := make([]Event, 20)
|
||||
for i := 0; i < 20; i++ {
|
||||
events[i] = Event{
|
||||
Kind: "message",
|
||||
Role: "user",
|
||||
Text: "Message " + string(rune('A'+i)),
|
||||
}
|
||||
}
|
||||
|
||||
if err := store.Append(session, events); err != nil {
|
||||
t.Fatalf("Append() error = %v", err)
|
||||
}
|
||||
|
||||
// Read tail with limit
|
||||
tail, err := store.ReadEventTail(session, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadEventTail() error = %v", err)
|
||||
}
|
||||
|
||||
if len(tail) != 5 {
|
||||
t.Errorf("ReadEventTail() returned %d events, want 5", len(tail))
|
||||
}
|
||||
|
||||
// Verify chronological order (oldest first in result)
|
||||
if len(tail) > 0 {
|
||||
// Should be last 5 events: P, Q, R, S, T
|
||||
expectedFirst := "Message P"
|
||||
if tail[0].Text != expectedFirst {
|
||||
t.Errorf("First event text = %q, want %q", tail[0].Text, expectedFirst)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportSessionZip(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
session := "export-test"
|
||||
|
||||
// Create session with content
|
||||
events := []Event{{Kind: "message", Text: "test event"}}
|
||||
if err := store.Append(session, events); err != nil {
|
||||
t.Fatalf("Append() error = %v", err)
|
||||
}
|
||||
|
||||
if err := store.WriteSummary(session, "Test summary", 0); err != nil {
|
||||
t.Fatalf("WriteSummary() error = %v", err)
|
||||
}
|
||||
|
||||
// Export as zip
|
||||
sessionDir := store.SessionDir(session)
|
||||
zipData, err := ExportSessionZip(sessionDir)
|
||||
if err != nil {
|
||||
t.Fatalf("ExportSessionZip() error = %v", err)
|
||||
}
|
||||
|
||||
if len(zipData) == 0 {
|
||||
t.Error("Expected non-empty zip data")
|
||||
}
|
||||
|
||||
// Verify it's a valid zip (starts with PK)
|
||||
if len(zipData) < 2 || zipData[0] != 'P' || zipData[1] != 'K' {
|
||||
t.Error("Expected valid zip file header")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportSessionZip_EmptyDir(t *testing.T) {
|
||||
_, err := ExportSessionZip("")
|
||||
if err == nil {
|
||||
t.Error("Expected error for empty directory")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportAllZip(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
|
||||
// Create multiple sessions
|
||||
for _, session := range []string{"session-1", "session-2"} {
|
||||
events := []Event{{Kind: "message", Text: "test in " + session}}
|
||||
if err := store.Append(session, events); err != nil {
|
||||
t.Fatalf("Append() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Export all
|
||||
zipData, err := ExportAllZip(tmpDir, 10*1024*1024)
|
||||
if err != nil {
|
||||
t.Fatalf("ExportAllZip() error = %v", err)
|
||||
}
|
||||
|
||||
if len(zipData) == 0 {
|
||||
t.Error("Expected non-empty zip data")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportAllZip_SizeLimit(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
|
||||
// Create session with large content
|
||||
events := []Event{{Kind: "message", Text: strings.Repeat("x", 10000)}}
|
||||
if err := store.Append("large-session", events); err != nil {
|
||||
t.Fatalf("Append() error = %v", err)
|
||||
}
|
||||
|
||||
// Try to export with very small limit
|
||||
_, err := ExportAllZip(tmpDir, 100)
|
||||
if err == nil {
|
||||
t.Error("Expected error when size exceeds limit")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,826 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSemanticStore_Append(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
namespace string
|
||||
records []SemanticRecord
|
||||
wantErr bool
|
||||
wantCount int
|
||||
}{
|
||||
{
|
||||
name: "appends single record",
|
||||
namespace: "test-namespace",
|
||||
records: []SemanticRecord{
|
||||
{
|
||||
TS: time.Now(),
|
||||
Role: "user",
|
||||
Text: "Hello, world!",
|
||||
Vec: []float32{0.1, 0.2, 0.3},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantCount: 1,
|
||||
},
|
||||
{
|
||||
name: "appends multiple records",
|
||||
namespace: "multi-record-ns",
|
||||
records: []SemanticRecord{
|
||||
{Text: "First message", Vec: []float32{0.1, 0.2, 0.3}},
|
||||
{Text: "Second message", Vec: []float32{0.4, 0.5, 0.6}},
|
||||
{Text: "Third message", Vec: []float32{0.7, 0.8, 0.9}},
|
||||
},
|
||||
wantErr: false,
|
||||
wantCount: 3,
|
||||
},
|
||||
{
|
||||
name: "skips records with empty text",
|
||||
namespace: "skip-empty-ns",
|
||||
records: []SemanticRecord{
|
||||
{Text: "Valid text", Vec: []float32{0.1, 0.2, 0.3}},
|
||||
{Text: "", Vec: []float32{0.4, 0.5, 0.6}},
|
||||
{Text: " ", Vec: []float32{0.7, 0.8, 0.9}},
|
||||
},
|
||||
wantErr: false,
|
||||
wantCount: 1,
|
||||
},
|
||||
{
|
||||
name: "skips records with empty vector",
|
||||
namespace: "skip-empty-vec-ns",
|
||||
records: []SemanticRecord{
|
||||
{Text: "Valid", Vec: []float32{0.1, 0.2, 0.3}},
|
||||
{Text: "No vector", Vec: []float32{}},
|
||||
{Text: "Nil vector", Vec: nil},
|
||||
},
|
||||
wantErr: false,
|
||||
wantCount: 1,
|
||||
},
|
||||
{
|
||||
name: "skips duplicate texts",
|
||||
namespace: "skip-dupe-ns",
|
||||
records: []SemanticRecord{
|
||||
{Text: "Same text", Vec: []float32{0.1, 0.2, 0.3}},
|
||||
{Text: "Same text", Vec: []float32{0.4, 0.5, 0.6}},
|
||||
{Text: "Different text", Vec: []float32{0.7, 0.8, 0.9}},
|
||||
},
|
||||
wantErr: false,
|
||||
wantCount: 2,
|
||||
},
|
||||
{
|
||||
name: "empty namespace returns nil",
|
||||
namespace: "",
|
||||
records: []SemanticRecord{
|
||||
{Text: "Test", Vec: []float32{0.1, 0.2, 0.3}},
|
||||
},
|
||||
wantErr: false,
|
||||
wantCount: 0,
|
||||
},
|
||||
{
|
||||
name: "empty records returns nil",
|
||||
namespace: "empty-records-ns",
|
||||
records: []SemanticRecord{},
|
||||
wantErr: false,
|
||||
wantCount: 0,
|
||||
},
|
||||
{
|
||||
name: "computes norm if not provided",
|
||||
namespace: "compute-norm-ns",
|
||||
records: []SemanticRecord{
|
||||
{Text: "Norm will be computed", Vec: []float32{3, 4}, Norm: 0},
|
||||
},
|
||||
wantErr: false,
|
||||
wantCount: 1,
|
||||
},
|
||||
{
|
||||
name: "uses provided norm",
|
||||
namespace: "provided-norm-ns",
|
||||
records: []SemanticRecord{
|
||||
{Text: "Norm is provided", Vec: []float32{3, 4}, Norm: 5.0},
|
||||
},
|
||||
wantErr: false,
|
||||
wantCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
|
||||
err := store.AppendSemantic(tt.namespace, tt.records)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("AppendSemantic() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
|
||||
if tt.namespace == "" || len(tt.records) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// Verify file content
|
||||
dir := filepath.Join(tmpDir, "semantic")
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
if tt.wantCount == 0 {
|
||||
return // No file expected
|
||||
}
|
||||
t.Fatalf("Failed to read semantic dir: %v", err)
|
||||
}
|
||||
|
||||
if len(entries) == 0 && tt.wantCount > 0 {
|
||||
t.Errorf("Expected semantic namespace directory to be created")
|
||||
return
|
||||
}
|
||||
|
||||
// Find the items.jsonl file
|
||||
var itemsPath string
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
itemsPath = filepath.Join(dir, e.Name(), "items.jsonl")
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if itemsPath == "" {
|
||||
if tt.wantCount > 0 {
|
||||
t.Error("items.jsonl not found")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(itemsPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read items.jsonl: %v", err)
|
||||
}
|
||||
|
||||
lines := strings.Split(strings.TrimSpace(string(data)), "\n")
|
||||
actualCount := 0
|
||||
for _, line := range lines {
|
||||
if strings.TrimSpace(line) != "" {
|
||||
actualCount++
|
||||
}
|
||||
}
|
||||
|
||||
if actualCount != tt.wantCount {
|
||||
t.Errorf("Expected %d records, got %d", tt.wantCount, actualCount)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSemanticStore_Search(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
records []SemanticRecord
|
||||
queryVec []float32
|
||||
maxChars int
|
||||
maxSnippets int
|
||||
wantCount int
|
||||
wantFirst string
|
||||
}{
|
||||
{
|
||||
name: "finds similar vectors",
|
||||
records: []SemanticRecord{
|
||||
{Text: "Apple is a fruit", Vec: []float32{0.9, 0.1, 0.0}},
|
||||
{Text: "Car is a vehicle", Vec: []float32{0.1, 0.9, 0.0}},
|
||||
{Text: "Orange is a fruit", Vec: []float32{0.85, 0.15, 0.0}},
|
||||
},
|
||||
queryVec: []float32{0.9, 0.1, 0.0},
|
||||
maxChars: 6000,
|
||||
maxSnippets: 10,
|
||||
wantCount: 3,
|
||||
wantFirst: "Apple is a fruit",
|
||||
},
|
||||
{
|
||||
name: "returns empty for no matches",
|
||||
records: []SemanticRecord{
|
||||
{Text: "Text 1", Vec: []float32{1, 0, 0}},
|
||||
{Text: "Text 2", Vec: []float32{0, 1, 0}},
|
||||
},
|
||||
queryVec: []float32{0, 0, 1},
|
||||
maxChars: 6000,
|
||||
maxSnippets: 10,
|
||||
wantCount: 0,
|
||||
wantFirst: "",
|
||||
},
|
||||
{
|
||||
name: "respects maxSnippets",
|
||||
records: []SemanticRecord{
|
||||
{Text: "Text 1", Vec: []float32{0.9, 0.1, 0.0}},
|
||||
{Text: "Text 2", Vec: []float32{0.85, 0.15, 0.0}},
|
||||
{Text: "Text 3", Vec: []float32{0.8, 0.2, 0.0}},
|
||||
{Text: "Text 4", Vec: []float32{0.75, 0.25, 0.0}},
|
||||
},
|
||||
queryVec: []float32{0.9, 0.1, 0.0},
|
||||
maxChars: 6000,
|
||||
maxSnippets: 2,
|
||||
wantCount: 2,
|
||||
wantFirst: "Text 1",
|
||||
},
|
||||
{
|
||||
name: "empty query vector returns nil",
|
||||
records: []SemanticRecord{
|
||||
{Text: "Some text", Vec: []float32{0.1, 0.2, 0.3}},
|
||||
},
|
||||
queryVec: []float32{},
|
||||
maxChars: 6000,
|
||||
maxSnippets: 10,
|
||||
wantCount: 0,
|
||||
wantFirst: "",
|
||||
},
|
||||
{
|
||||
name: "orders by similarity score",
|
||||
records: []SemanticRecord{
|
||||
{Text: "Low similarity", Vec: []float32{0.1, 0.9, 0.0}},
|
||||
{Text: "High similarity", Vec: []float32{0.95, 0.05, 0.0}},
|
||||
{Text: "Medium similarity", Vec: []float32{0.5, 0.5, 0.0}},
|
||||
},
|
||||
queryVec: []float32{1.0, 0.0, 0.0},
|
||||
maxChars: 6000,
|
||||
maxSnippets: 10,
|
||||
wantCount: 3,
|
||||
wantFirst: "High similarity",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
namespace := "search-test-ns"
|
||||
|
||||
// Append records
|
||||
if len(tt.records) > 0 {
|
||||
if err := store.AppendSemantic(namespace, tt.records); err != nil {
|
||||
t.Fatalf("AppendSemantic() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Search
|
||||
results, err := store.SearchSemantic(namespace, tt.queryVec, tt.maxChars, tt.maxSnippets)
|
||||
if err != nil {
|
||||
t.Fatalf("SearchSemantic() error = %v", err)
|
||||
}
|
||||
|
||||
if len(results) != tt.wantCount {
|
||||
t.Errorf("SearchSemantic() returned %d results, want %d", len(results), tt.wantCount)
|
||||
}
|
||||
|
||||
if tt.wantFirst != "" && len(results) > 0 {
|
||||
if results[0] != tt.wantFirst {
|
||||
t.Errorf("First result = %q, want %q", results[0], tt.wantFirst)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSemanticStore_SearchWithText(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
namespace := "text-search-ns"
|
||||
|
||||
records := []SemanticRecord{
|
||||
{Text: "Python programming tutorial", Vec: []float32{0.9, 0.1, 0.0}},
|
||||
{Text: "Java development guide", Vec: []float32{0.85, 0.15, 0.0}},
|
||||
{Text: "Python data science course", Vec: []float32{0.88, 0.12, 0.0}},
|
||||
}
|
||||
|
||||
if err := store.AppendSemantic(namespace, records); err != nil {
|
||||
t.Fatalf("AppendSemantic() error = %v", err)
|
||||
}
|
||||
|
||||
// Search with both vector and text (keyword boost)
|
||||
results, err := store.SearchSemanticWithText(namespace, []float32{0.9, 0.1, 0.0}, "python", 6000, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("SearchSemanticWithText() error = %v", err)
|
||||
}
|
||||
|
||||
if len(results) == 0 {
|
||||
t.Error("Expected results from SearchSemanticWithText")
|
||||
}
|
||||
|
||||
// Python results should be boosted
|
||||
pythonCount := 0
|
||||
for _, r := range results {
|
||||
if strings.Contains(strings.ToLower(r), "python") {
|
||||
pythonCount++
|
||||
}
|
||||
}
|
||||
|
||||
if pythonCount == 0 {
|
||||
t.Error("Expected Python-related results to be present")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbedding_Normalize(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
vec []float32
|
||||
wantNorm float32
|
||||
wantZero bool
|
||||
}{
|
||||
{
|
||||
name: "unit vector",
|
||||
vec: []float32{1, 0, 0},
|
||||
wantNorm: 1.0,
|
||||
wantZero: false,
|
||||
},
|
||||
{
|
||||
name: "3-4-5 right triangle",
|
||||
vec: []float32{3, 4},
|
||||
wantNorm: 5.0,
|
||||
wantZero: false,
|
||||
},
|
||||
{
|
||||
name: "all zeros",
|
||||
vec: []float32{0, 0, 0},
|
||||
wantNorm: 0,
|
||||
wantZero: true,
|
||||
},
|
||||
{
|
||||
name: "empty vector",
|
||||
vec: []float32{},
|
||||
wantNorm: 0,
|
||||
wantZero: true,
|
||||
},
|
||||
{
|
||||
name: "negative values",
|
||||
vec: []float32{-3, 4},
|
||||
wantNorm: 5.0,
|
||||
wantZero: false,
|
||||
},
|
||||
{
|
||||
name: "high dimensional",
|
||||
vec: []float32{1, 1, 1, 1},
|
||||
wantNorm: 2.0, // sqrt(4) = 2
|
||||
wantZero: false,
|
||||
},
|
||||
{
|
||||
name: "small values",
|
||||
vec: []float32{0.001, 0.001, 0.001},
|
||||
wantNorm: float32(math.Sqrt(0.000003)),
|
||||
wantZero: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
norm := vectorNorm(tt.vec)
|
||||
|
||||
if tt.wantZero && norm != 0 {
|
||||
t.Errorf("vectorNorm() = %v, want 0", norm)
|
||||
return
|
||||
}
|
||||
|
||||
if !tt.wantZero {
|
||||
tolerance := float32(0.0001)
|
||||
if diff := math.Abs(float64(norm - tt.wantNorm)); diff > float64(tolerance) {
|
||||
t.Errorf("vectorNorm() = %v, want %v (diff: %v)", norm, tt.wantNorm, diff)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbedding_CosineSimilarity(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
vecA []float32
|
||||
normA float32
|
||||
vecB []float32
|
||||
normB float32
|
||||
wantSim float32
|
||||
tolerance float32
|
||||
}{
|
||||
{
|
||||
name: "identical vectors",
|
||||
vecA: []float32{1, 0, 0},
|
||||
normA: 1.0,
|
||||
vecB: []float32{1, 0, 0},
|
||||
normB: 1.0,
|
||||
wantSim: 1.0,
|
||||
tolerance: 0.0001,
|
||||
},
|
||||
{
|
||||
name: "orthogonal vectors",
|
||||
vecA: []float32{1, 0, 0},
|
||||
normA: 1.0,
|
||||
vecB: []float32{0, 1, 0},
|
||||
normB: 1.0,
|
||||
wantSim: 0.0,
|
||||
tolerance: 0.0001,
|
||||
},
|
||||
{
|
||||
name: "opposite vectors",
|
||||
vecA: []float32{1, 0, 0},
|
||||
normA: 1.0,
|
||||
vecB: []float32{-1, 0, 0},
|
||||
normB: 1.0,
|
||||
wantSim: -1.0,
|
||||
tolerance: 0.0001,
|
||||
},
|
||||
{
|
||||
name: "45 degree angle",
|
||||
vecA: []float32{1, 0},
|
||||
normA: 1.0,
|
||||
vecB: []float32{1, 1},
|
||||
normB: float32(math.Sqrt(2)),
|
||||
wantSim: float32(1.0 / math.Sqrt(2)), // cos(45) = 1/sqrt(2)
|
||||
tolerance: 0.0001,
|
||||
},
|
||||
{
|
||||
name: "non-unit vectors",
|
||||
vecA: []float32{3, 4},
|
||||
normA: 5.0,
|
||||
vecB: []float32{6, 8},
|
||||
normB: 10.0,
|
||||
wantSim: 1.0, // Same direction
|
||||
tolerance: 0.0001,
|
||||
},
|
||||
{
|
||||
name: "zero norm A returns 0",
|
||||
vecA: []float32{0, 0, 0},
|
||||
normA: 0,
|
||||
vecB: []float32{1, 0, 0},
|
||||
normB: 1.0,
|
||||
wantSim: 0,
|
||||
tolerance: 0,
|
||||
},
|
||||
{
|
||||
name: "zero norm B returns 0",
|
||||
vecA: []float32{1, 0, 0},
|
||||
normA: 1.0,
|
||||
vecB: []float32{0, 0, 0},
|
||||
normB: 0,
|
||||
wantSim: 0,
|
||||
tolerance: 0,
|
||||
},
|
||||
{
|
||||
name: "different length vectors",
|
||||
vecA: []float32{1, 2, 3, 4, 5},
|
||||
normA: vectorNorm([]float32{1, 2, 3, 4, 5}),
|
||||
vecB: []float32{1, 2, 3},
|
||||
normB: vectorNorm([]float32{1, 2, 3}),
|
||||
wantSim: float32(14.0) / (vectorNorm([]float32{1, 2, 3, 4, 5}) * vectorNorm([]float32{1, 2, 3})),
|
||||
tolerance: 0.0001,
|
||||
},
|
||||
{
|
||||
name: "high dimensional similar",
|
||||
vecA: []float32{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8},
|
||||
normA: vectorNorm([]float32{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8}),
|
||||
vecB: []float32{0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85},
|
||||
normB: vectorNorm([]float32{0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85}),
|
||||
wantSim: 0.999, // Very similar
|
||||
tolerance: 0.01,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
sim := cosineSim(tt.vecA, tt.normA, tt.vecB, tt.normB)
|
||||
|
||||
diff := math.Abs(float64(sim - tt.wantSim))
|
||||
if diff > float64(tt.tolerance) {
|
||||
t.Errorf("cosineSim() = %v, want %v (diff: %v, tolerance: %v)", sim, tt.wantSim, diff, tt.tolerance)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSemanticRecord_Serialization(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
record SemanticRecord
|
||||
}{
|
||||
{
|
||||
name: "full record",
|
||||
record: SemanticRecord{
|
||||
TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC),
|
||||
Role: "user",
|
||||
Text: "Test message",
|
||||
Vec: []float32{0.1, 0.2, 0.3},
|
||||
Norm: 0.374,
|
||||
Source: "chat",
|
||||
Session: "session-123",
|
||||
Repo: "org/repo",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "minimal record",
|
||||
record: SemanticRecord{
|
||||
Text: "Minimal",
|
||||
Vec: []float32{1.0},
|
||||
Norm: 1.0,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
data, err := json.Marshal(tt.record)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal() error = %v", err)
|
||||
}
|
||||
|
||||
var decoded SemanticRecord
|
||||
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
|
||||
if decoded.Text != tt.record.Text {
|
||||
t.Errorf("Text mismatch: got %q, want %q", decoded.Text, tt.record.Text)
|
||||
}
|
||||
if len(decoded.Vec) != len(tt.record.Vec) {
|
||||
t.Errorf("Vec length mismatch: got %d, want %d", len(decoded.Vec), len(tt.record.Vec))
|
||||
}
|
||||
if decoded.Norm != tt.record.Norm {
|
||||
t.Errorf("Norm mismatch: got %v, want %v", decoded.Norm, tt.record.Norm)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSemanticStore_ReadSemanticTail(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
namespace := "tail-test-ns"
|
||||
|
||||
// Append multiple records
|
||||
records := make([]SemanticRecord, 10)
|
||||
for i := 0; i < 10; i++ {
|
||||
records[i] = SemanticRecord{
|
||||
TS: time.Now().Add(time.Duration(i) * time.Minute),
|
||||
Text: "Message " + string(rune('A'+i)),
|
||||
Vec: []float32{float32(i), float32(i + 1), float32(i + 2)},
|
||||
}
|
||||
}
|
||||
|
||||
if err := store.AppendSemantic(namespace, records); err != nil {
|
||||
t.Fatalf("AppendSemantic() error = %v", err)
|
||||
}
|
||||
|
||||
// Read tail with limit
|
||||
tail, err := store.ReadSemanticTail(namespace, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadSemanticTail() error = %v", err)
|
||||
}
|
||||
|
||||
if len(tail) != 5 {
|
||||
t.Errorf("ReadSemanticTail() returned %d records, want 5", len(tail))
|
||||
}
|
||||
|
||||
// Verify order (should be chronological, last 5 records)
|
||||
if len(tail) > 0 {
|
||||
// First in tail should be 6th record (index 5)
|
||||
if !strings.Contains(tail[0].Text, "Message F") {
|
||||
t.Errorf("First record = %q, expected Message F", tail[0].Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSemanticStore_ListNamespaces(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
store := NewFileStore(tmpDir)
|
||||
|
||||
namespaces := []string{"ns-alpha", "ns-beta", "ns-gamma"}
|
||||
for _, ns := range namespaces {
|
||||
records := []SemanticRecord{
|
||||
{Text: "Record in " + ns, Vec: []float32{0.1, 0.2, 0.3}},
|
||||
}
|
||||
if err := store.AppendSemantic(ns, records); err != nil {
|
||||
t.Fatalf("AppendSemantic() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// List all namespaces
|
||||
list, err := store.ListSemanticNamespaces(0)
|
||||
if err != nil {
|
||||
t.Fatalf("ListSemanticNamespaces() error = %v", err)
|
||||
}
|
||||
|
||||
if len(list) != len(namespaces) {
|
||||
t.Errorf("ListSemanticNamespaces() returned %d, want %d", len(list), len(namespaces))
|
||||
}
|
||||
|
||||
// List with limit
|
||||
limited, err := store.ListSemanticNamespaces(2)
|
||||
if err != nil {
|
||||
t.Fatalf("ListSemanticNamespaces() error = %v", err)
|
||||
}
|
||||
|
||||
if len(limited) != 2 {
|
||||
t.Errorf("ListSemanticNamespaces(2) returned %d, want 2", len(limited))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenizeSemanticQuery(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
max int
|
||||
want []string
|
||||
wantN int
|
||||
}{
|
||||
{
|
||||
name: "basic tokenization",
|
||||
query: "Hello World",
|
||||
max: 10,
|
||||
want: []string{"hello", "world"},
|
||||
wantN: 2,
|
||||
},
|
||||
{
|
||||
name: "removes stop words",
|
||||
query: "the quick brown fox and the lazy dog",
|
||||
max: 10,
|
||||
want: []string{"quick", "brown", "fox", "lazy", "dog"},
|
||||
wantN: 5,
|
||||
},
|
||||
{
|
||||
name: "respects max limit",
|
||||
query: "one two three four five six seven",
|
||||
max: 3,
|
||||
want: []string{"one", "two", "three"},
|
||||
wantN: 3,
|
||||
},
|
||||
{
|
||||
name: "removes short words",
|
||||
query: "I am a test is it ok",
|
||||
max: 10,
|
||||
want: []string{"test"},
|
||||
wantN: 1,
|
||||
},
|
||||
{
|
||||
name: "handles special characters",
|
||||
query: "hello-world test_case foo.bar",
|
||||
max: 10,
|
||||
want: []string{"hello-world", "test_case", "foo", "bar"},
|
||||
wantN: 4,
|
||||
},
|
||||
{
|
||||
name: "empty query",
|
||||
query: "",
|
||||
max: 10,
|
||||
want: nil,
|
||||
wantN: 0,
|
||||
},
|
||||
{
|
||||
name: "only stop words",
|
||||
query: "the and for with",
|
||||
max: 10,
|
||||
want: nil,
|
||||
wantN: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tokens := tokenizeSemanticQuery(tt.query, tt.max)
|
||||
|
||||
if len(tokens) != tt.wantN {
|
||||
t.Errorf("tokenizeSemanticQuery() returned %d tokens, want %d", len(tokens), tt.wantN)
|
||||
}
|
||||
|
||||
if tt.want != nil {
|
||||
for i, want := range tt.want {
|
||||
if i >= len(tokens) {
|
||||
t.Errorf("Missing token at index %d: want %q", i, want)
|
||||
continue
|
||||
}
|
||||
if tokens[i] != want {
|
||||
t.Errorf("Token %d = %q, want %q", i, tokens[i], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSemanticTokenOverlap(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
tokens []string
|
||||
text string
|
||||
want int
|
||||
}{
|
||||
{
|
||||
name: "all tokens match",
|
||||
tokens: []string{"hello", "world"},
|
||||
text: "Hello World Test",
|
||||
want: 2,
|
||||
},
|
||||
{
|
||||
name: "some tokens match",
|
||||
tokens: []string{"hello", "world", "foo"},
|
||||
text: "Hello bar baz",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "no tokens match",
|
||||
tokens: []string{"foo", "bar"},
|
||||
text: "Hello World",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "empty tokens",
|
||||
tokens: []string{},
|
||||
text: "Hello World",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "empty text",
|
||||
tokens: []string{"hello"},
|
||||
text: "",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "case insensitive",
|
||||
tokens: []string{"hello"},
|
||||
text: "HELLO World",
|
||||
want: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
overlap := semanticTokenOverlap(tt.tokens, tt.text)
|
||||
if overlap != tt.want {
|
||||
t.Errorf("semanticTokenOverlap() = %d, want %d", overlap, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSemanticRecencyScore(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
ts time.Time
|
||||
windowDays int
|
||||
wantMin float32
|
||||
wantMax float32
|
||||
}{
|
||||
{
|
||||
name: "current time",
|
||||
ts: now,
|
||||
windowDays: 30,
|
||||
wantMin: 0.99,
|
||||
wantMax: 1.0,
|
||||
},
|
||||
{
|
||||
name: "half window age",
|
||||
ts: now.Add(-15 * 24 * time.Hour),
|
||||
windowDays: 30,
|
||||
wantMin: 0.4,
|
||||
wantMax: 0.6,
|
||||
},
|
||||
{
|
||||
name: "at window edge",
|
||||
ts: now.Add(-30 * 24 * time.Hour),
|
||||
windowDays: 30,
|
||||
wantMin: 0.0,
|
||||
wantMax: 0.01,
|
||||
},
|
||||
{
|
||||
name: "beyond window",
|
||||
ts: now.Add(-60 * 24 * time.Hour),
|
||||
windowDays: 30,
|
||||
wantMin: 0.0,
|
||||
wantMax: 0.0,
|
||||
},
|
||||
{
|
||||
name: "zero time",
|
||||
ts: time.Time{},
|
||||
windowDays: 30,
|
||||
wantMin: 0.0,
|
||||
wantMax: 0.0,
|
||||
},
|
||||
{
|
||||
name: "zero window",
|
||||
ts: now,
|
||||
windowDays: 0,
|
||||
wantMin: 0.0,
|
||||
wantMax: 0.0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
score := semanticRecencyScore(now, tt.ts, tt.windowDays)
|
||||
if score < tt.wantMin || score > tt.wantMax {
|
||||
t.Errorf("semanticRecencyScore() = %v, want between %v and %v", score, tt.wantMin, tt.wantMax)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,365 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestEvent_JSON_Serialization(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
event Event
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "full event with all fields",
|
||||
event: Event{
|
||||
TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC),
|
||||
Kind: "message",
|
||||
Role: "user",
|
||||
Type: "text",
|
||||
Text: "Hello, world!",
|
||||
Meta: map[string]string{"key": "value", "model": "gpt-4"},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "minimal event with only required fields",
|
||||
event: Event{
|
||||
TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC),
|
||||
Kind: "system",
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "event with empty optional fields",
|
||||
event: Event{
|
||||
TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC),
|
||||
Kind: "assistant",
|
||||
Role: "",
|
||||
Type: "",
|
||||
Text: "",
|
||||
Meta: nil,
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "event with zero timestamp",
|
||||
event: Event{
|
||||
Kind: "message",
|
||||
Text: "test",
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "event with unicode text",
|
||||
event: Event{
|
||||
TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC),
|
||||
Kind: "message",
|
||||
Text: "Hello \u4e16\u754c! \U0001F600",
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "event with multiline text",
|
||||
event: Event{
|
||||
TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC),
|
||||
Kind: "message",
|
||||
Text: "Line 1\nLine 2\nLine 3",
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Marshal to JSON
|
||||
data, err := json.Marshal(tt.event)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("json.Marshal() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if tt.wantErr {
|
||||
return
|
||||
}
|
||||
|
||||
// Unmarshal back to Event
|
||||
var decoded Event
|
||||
err = json.Unmarshal(data, &decoded)
|
||||
if err != nil {
|
||||
t.Errorf("json.Unmarshal() error = %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Compare fields
|
||||
if !tt.event.TS.Equal(decoded.TS) {
|
||||
t.Errorf("TS mismatch: got %v, want %v", decoded.TS, tt.event.TS)
|
||||
}
|
||||
if decoded.Kind != tt.event.Kind {
|
||||
t.Errorf("Kind mismatch: got %q, want %q", decoded.Kind, tt.event.Kind)
|
||||
}
|
||||
if decoded.Role != tt.event.Role {
|
||||
t.Errorf("Role mismatch: got %q, want %q", decoded.Role, tt.event.Role)
|
||||
}
|
||||
if decoded.Type != tt.event.Type {
|
||||
t.Errorf("Type mismatch: got %q, want %q", decoded.Type, tt.event.Type)
|
||||
}
|
||||
if decoded.Text != tt.event.Text {
|
||||
t.Errorf("Text mismatch: got %q, want %q", decoded.Text, tt.event.Text)
|
||||
}
|
||||
if len(tt.event.Meta) != len(decoded.Meta) {
|
||||
t.Errorf("Meta length mismatch: got %d, want %d", len(decoded.Meta), len(tt.event.Meta))
|
||||
}
|
||||
for k, v := range tt.event.Meta {
|
||||
if decoded.Meta[k] != v {
|
||||
t.Errorf("Meta[%q] mismatch: got %q, want %q", k, decoded.Meta[k], v)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvent_JSON_OmitEmpty(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
event Event
|
||||
shouldContain []string
|
||||
shouldOmit []string
|
||||
}{
|
||||
{
|
||||
name: "omits empty optional fields",
|
||||
event: Event{
|
||||
TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC),
|
||||
Kind: "message",
|
||||
},
|
||||
shouldContain: []string{"ts", "kind"},
|
||||
shouldOmit: []string{"role", "type", "text", "meta"},
|
||||
},
|
||||
{
|
||||
name: "includes non-empty optional fields",
|
||||
event: Event{
|
||||
TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC),
|
||||
Kind: "message",
|
||||
Role: "user",
|
||||
Text: "hello",
|
||||
},
|
||||
shouldContain: []string{"ts", "kind", "role", "text"},
|
||||
shouldOmit: []string{"type", "meta"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
data, err := json.Marshal(tt.event)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal() error = %v", err)
|
||||
}
|
||||
jsonStr := string(data)
|
||||
|
||||
for _, field := range tt.shouldContain {
|
||||
if !containsField(jsonStr, field) {
|
||||
t.Errorf("JSON should contain field %q, got: %s", field, jsonStr)
|
||||
}
|
||||
}
|
||||
for _, field := range tt.shouldOmit {
|
||||
if containsField(jsonStr, field) {
|
||||
t.Errorf("JSON should omit field %q, got: %s", field, jsonStr)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvent_Fields(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
event Event
|
||||
wantKind string
|
||||
wantRole string
|
||||
wantType string
|
||||
wantText string
|
||||
}{
|
||||
{
|
||||
name: "user message event",
|
||||
event: Event{
|
||||
Kind: "message",
|
||||
Role: "user",
|
||||
Type: "text",
|
||||
Text: "What is the weather?",
|
||||
},
|
||||
wantKind: "message",
|
||||
wantRole: "user",
|
||||
wantType: "text",
|
||||
wantText: "What is the weather?",
|
||||
},
|
||||
{
|
||||
name: "assistant response event",
|
||||
event: Event{
|
||||
Kind: "message",
|
||||
Role: "assistant",
|
||||
Type: "text",
|
||||
Text: "The weather is sunny.",
|
||||
},
|
||||
wantKind: "message",
|
||||
wantRole: "assistant",
|
||||
wantType: "text",
|
||||
wantText: "The weather is sunny.",
|
||||
},
|
||||
{
|
||||
name: "system event",
|
||||
event: Event{
|
||||
Kind: "system",
|
||||
Type: "config",
|
||||
Text: "Session started",
|
||||
},
|
||||
wantKind: "system",
|
||||
wantRole: "",
|
||||
wantType: "config",
|
||||
wantText: "Session started",
|
||||
},
|
||||
{
|
||||
name: "tool use event",
|
||||
event: Event{
|
||||
Kind: "tool_use",
|
||||
Role: "assistant",
|
||||
Type: "bash",
|
||||
Text: "ls -la",
|
||||
},
|
||||
wantKind: "tool_use",
|
||||
wantRole: "assistant",
|
||||
wantType: "bash",
|
||||
wantText: "ls -la",
|
||||
},
|
||||
{
|
||||
name: "tool result event",
|
||||
event: Event{
|
||||
Kind: "tool_result",
|
||||
Role: "user",
|
||||
Type: "bash",
|
||||
Text: "file1.txt\nfile2.txt",
|
||||
},
|
||||
wantKind: "tool_result",
|
||||
wantRole: "user",
|
||||
wantType: "bash",
|
||||
wantText: "file1.txt\nfile2.txt",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if tt.event.Kind != tt.wantKind {
|
||||
t.Errorf("Kind = %q, want %q", tt.event.Kind, tt.wantKind)
|
||||
}
|
||||
if tt.event.Role != tt.wantRole {
|
||||
t.Errorf("Role = %q, want %q", tt.event.Role, tt.wantRole)
|
||||
}
|
||||
if tt.event.Type != tt.wantType {
|
||||
t.Errorf("Type = %q, want %q", tt.event.Type, tt.wantType)
|
||||
}
|
||||
if tt.event.Text != tt.wantText {
|
||||
t.Errorf("Text = %q, want %q", tt.event.Text, tt.wantText)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvent_MetaField(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
meta map[string]string
|
||||
key string
|
||||
wantVal string
|
||||
wantOk bool
|
||||
}{
|
||||
{
|
||||
name: "existing key",
|
||||
meta: map[string]string{"model": "gpt-4", "tokens": "100"},
|
||||
key: "model",
|
||||
wantVal: "gpt-4",
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "missing key",
|
||||
meta: map[string]string{"model": "gpt-4"},
|
||||
key: "tokens",
|
||||
wantVal: "",
|
||||
wantOk: false,
|
||||
},
|
||||
{
|
||||
name: "nil meta",
|
||||
meta: nil,
|
||||
key: "any",
|
||||
wantVal: "",
|
||||
wantOk: false,
|
||||
},
|
||||
{
|
||||
name: "empty meta",
|
||||
meta: map[string]string{},
|
||||
key: "any",
|
||||
wantVal: "",
|
||||
wantOk: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
event := Event{Meta: tt.meta}
|
||||
val, ok := event.Meta[tt.key]
|
||||
if ok != tt.wantOk {
|
||||
t.Errorf("Meta[%q] ok = %v, want %v", tt.key, ok, tt.wantOk)
|
||||
}
|
||||
if val != tt.wantVal {
|
||||
t.Errorf("Meta[%q] = %q, want %q", tt.key, val, tt.wantVal)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvent_Timestamp(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
ts time.Time
|
||||
isZero bool
|
||||
}{
|
||||
{
|
||||
name: "valid timestamp",
|
||||
ts: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC),
|
||||
isZero: false,
|
||||
},
|
||||
{
|
||||
name: "zero timestamp",
|
||||
ts: time.Time{},
|
||||
isZero: true,
|
||||
},
|
||||
{
|
||||
name: "unix epoch",
|
||||
ts: time.Unix(0, 0).UTC(),
|
||||
isZero: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
event := Event{TS: tt.ts}
|
||||
if event.TS.IsZero() != tt.isZero {
|
||||
t.Errorf("TS.IsZero() = %v, want %v", event.TS.IsZero(), tt.isZero)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// containsField checks if a JSON string contains a specific field key
|
||||
func containsField(jsonStr, field string) bool {
|
||||
return json.Valid([]byte(jsonStr)) && (len(jsonStr) > 0 && (jsonStr[0] == '{' || jsonStr[0] == '[')) &&
|
||||
(len(field) > 0 && (jsonFieldPresent(jsonStr, field)))
|
||||
}
|
||||
|
||||
func jsonFieldPresent(jsonStr, field string) bool {
|
||||
var m map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(jsonStr), &m); err != nil {
|
||||
return false
|
||||
}
|
||||
_, ok := m[field]
|
||||
return ok
|
||||
}
|
||||
Reference in New Issue
Block a user