From 009fbb2a3791e065cf4e0f4eda3adeb915cedbd7 Mon Sep 17 00:00:00 2001 From: Finessse Date: Sun, 28 Dec 2025 10:46:21 +0700 Subject: [PATCH] test(memory): add unit tests for file store, pruning, and semantic search Covers event serialization, file operations, token-based pruning, recency preservation, and semantic similarity scoring. --- internal/memory/file_store_test.go | 641 ++++++++++++++++++++++ internal/memory/prune_test.go | 610 +++++++++++++++++++++ internal/memory/semantic_test.go | 826 +++++++++++++++++++++++++++++ internal/memory/store_test.go | 365 +++++++++++++ 4 files changed, 2442 insertions(+) create mode 100644 internal/memory/file_store_test.go create mode 100644 internal/memory/prune_test.go create mode 100644 internal/memory/semantic_test.go create mode 100644 internal/memory/store_test.go diff --git a/internal/memory/file_store_test.go b/internal/memory/file_store_test.go new file mode 100644 index 00000000..0da51d52 --- /dev/null +++ b/internal/memory/file_store_test.go @@ -0,0 +1,641 @@ +package memory + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestFileStore_Append_CreatesFile(t *testing.T) { + tests := []struct { + name string + session string + events []Event + wantErr bool + }{ + { + name: "creates file with single event", + session: "test-session-1", + events: []Event{ + { + TS: time.Now(), + Kind: "message", + Role: "user", + Text: "Hello, world!", + }, + }, + wantErr: false, + }, + { + name: "creates file with event that has zero timestamp", + session: "test-session-2", + events: []Event{ + { + Kind: "message", + Role: "user", + Text: "This event has no timestamp set", + }, + }, + wantErr: false, + }, + { + name: "empty session returns nil without creating file", + session: "", + events: []Event{ + {Kind: "message", Text: "test"}, + }, + wantErr: false, + }, + { + name: "empty events returns nil without creating file", + session: "test-session-3", + events: []Event{}, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + + err := store.Append(tt.session, tt.events) + if (err != nil) != tt.wantErr { + t.Errorf("Append() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if tt.session == "" || len(tt.events) == 0 { + // File should not be created + eventsPath := filepath.Join(tmpDir, "sessions", tt.session, "events.jsonl") + if _, err := os.Stat(eventsPath); !os.IsNotExist(err) { + t.Errorf("File should not exist for empty session or events") + } + return + } + + // Verify file was created + eventsPath := filepath.Join(tmpDir, "sessions", tt.session, "events.jsonl") + if _, err := os.Stat(eventsPath); os.IsNotExist(err) { + t.Errorf("events.jsonl was not created") + return + } + + // Verify content is valid JSONL + data, err := os.ReadFile(eventsPath) + if err != nil { + t.Fatalf("Failed to read events file: %v", err) + } + + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + if len(lines) != len(tt.events) { + t.Errorf("Expected %d lines, got %d", len(tt.events), len(lines)) + } + + for i, line := range lines { + var e Event + if err := json.Unmarshal([]byte(line), &e); err != nil { + t.Errorf("Line %d is not valid JSON: %v", i, err) + } + } + }) + } +} + +func TestFileStore_Append_MultipleEvents(t *testing.T) { + tests := []struct { + name string + session string + eventSets [][]Event + wantTotal int + wantErr bool + }{ + { + name: "append multiple events in single call", + session: "multi-event-session", + eventSets: [][]Event{ + { + {Kind: "message", Role: "user", Text: "First message"}, + {Kind: "message", Role: "assistant", Text: "First response"}, + {Kind: "message", Role: "user", Text: "Second message"}, + }, + }, + wantTotal: 3, + wantErr: false, + }, + { + name: "append events across multiple calls", + session: "multiple-calls-session", + eventSets: [][]Event{ + {{Kind: "message", Role: "user", Text: "Message 1"}}, + {{Kind: "message", Role: "assistant", Text: "Response 1"}}, + {{Kind: "message", Role: "user", Text: "Message 2"}}, + {{Kind: "message", Role: "assistant", Text: "Response 2"}}, + }, + wantTotal: 4, + wantErr: false, + }, + { + name: "append mixed events with metadata", + session: "mixed-events-session", + eventSets: [][]Event{ + { + {Kind: "system", Text: "Session started"}, + {Kind: "message", Role: "user", Text: "Hello", Meta: map[string]string{"lang": "en"}}, + {Kind: "tool_use", Role: "assistant", Type: "bash", Text: "ls"}, + {Kind: "tool_result", Role: "user", Type: "bash", Text: "file.txt"}, + }, + }, + wantTotal: 4, + wantErr: false, + }, + { + name: "append preserves order", + session: "order-session", + eventSets: [][]Event{ + { + {Kind: "message", Role: "user", Text: "First"}, + {Kind: "message", Role: "user", Text: "Second"}, + {Kind: "message", Role: "user", Text: "Third"}, + }, + }, + wantTotal: 3, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + + for _, events := range tt.eventSets { + err := store.Append(tt.session, events) + if (err != nil) != tt.wantErr { + t.Errorf("Append() error = %v, wantErr %v", err, tt.wantErr) + return + } + } + + // Read and verify the file + eventsPath := filepath.Join(tmpDir, "sessions", tt.session, "events.jsonl") + data, err := os.ReadFile(eventsPath) + if err != nil { + t.Fatalf("Failed to read events file: %v", err) + } + + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + if len(lines) != tt.wantTotal { + t.Errorf("Expected %d events, got %d", tt.wantTotal, len(lines)) + } + + // Verify each line is valid JSON and check order for order test + if tt.name == "append preserves order" { + expectedTexts := []string{"First", "Second", "Third"} + for i, line := range lines { + var e Event + if err := json.Unmarshal([]byte(line), &e); err != nil { + t.Errorf("Line %d is not valid JSON: %v", i, err) + continue + } + if e.Text != expectedTexts[i] { + t.Errorf("Event %d: expected text %q, got %q", i, expectedTexts[i], e.Text) + } + } + } + }) + } +} + +func TestFileStore_Append_RedactsSecrets(t *testing.T) { + tests := []struct { + name string + inputText string + wantRedact bool + }{ + { + name: "redacts sk- API keys", + inputText: "My key is sk-1234567890abcdef", + wantRedact: true, + }, + { + name: "redacts bearer tokens", + inputText: "Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + wantRedact: true, + }, + { + name: "redacts AIza keys", + inputText: "API key: AIzaSyB1234567890abcdefghij", + wantRedact: true, + }, + { + name: "does not modify normal text", + inputText: "This is a normal message without secrets", + wantRedact: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + session := "redact-test" + + err := store.Append(session, []Event{ + {Kind: "message", Text: tt.inputText}, + }) + if err != nil { + t.Fatalf("Append() error = %v", err) + } + + // Read the stored event + eventsPath := filepath.Join(tmpDir, "sessions", session, "events.jsonl") + data, err := os.ReadFile(eventsPath) + if err != nil { + t.Fatalf("Failed to read events file: %v", err) + } + + var e Event + if err := json.Unmarshal(data, &e); err != nil { + t.Fatalf("Failed to parse event: %v", err) + } + + hasRedacted := strings.Contains(e.Text, "[REDACTED]") + if tt.wantRedact && !hasRedacted { + t.Errorf("Expected text to be redacted, got: %s", e.Text) + } + if !tt.wantRedact && hasRedacted { + t.Errorf("Text should not be redacted, got: %s", e.Text) + } + }) + } +} + +func TestFileStore_Search_ExactMatch(t *testing.T) { + tests := []struct { + name string + events []Event + query string + wantCount int + wantMatch string + }{ + { + name: "finds exact keyword match", + events: []Event{ + {Kind: "message", Role: "user", Text: "How do I configure the database?"}, + {Kind: "message", Role: "assistant", Text: "You can configure the database in config.yaml"}, + {Kind: "message", Role: "user", Text: "Thanks, what about logging?"}, + }, + query: "database", + wantCount: 2, + wantMatch: "database", + }, + { + name: "case insensitive search", + events: []Event{ + {Kind: "message", Text: "The DATABASE connection failed"}, + {Kind: "message", Text: "Database retry succeeded"}, + {Kind: "message", Text: "No matches here"}, + }, + query: "database", + wantCount: 2, + wantMatch: "", + }, + { + name: "no matches returns empty", + events: []Event{ + {Kind: "message", Text: "Hello world"}, + {Kind: "message", Text: "Goodbye world"}, + }, + query: "nonexistent", + wantCount: 0, + wantMatch: "", + }, + { + name: "multiple keywords - matches any keyword", + events: []Event{ + {Kind: "message", Text: "Error in authentication module"}, + {Kind: "message", Text: "Authentication failed for user"}, + {Kind: "message", Text: "Module loaded successfully"}, + }, + query: "authentication module", + wantCount: 3, // all three contain at least one keyword + wantMatch: "", + }, + { + name: "empty query returns nil", + events: []Event{ + {Kind: "message", Text: "Some text"}, + }, + query: "", + wantCount: 0, + wantMatch: "", + }, + { + name: "whitespace query returns nil", + events: []Event{ + {Kind: "message", Text: "Some text"}, + }, + query: " ", + wantCount: 0, + wantMatch: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + session := "search-test" + + // Append test events + if err := store.Append(session, tt.events); err != nil { + t.Fatalf("Append() error = %v", err) + } + + // Search + results, err := store.Search(session, tt.query, 6000, 10) + if err != nil { + t.Fatalf("Search() error = %v", err) + } + + if len(results) != tt.wantCount { + t.Errorf("Search() returned %d results, want %d", len(results), tt.wantCount) + } + + if tt.wantMatch != "" { + found := false + for _, r := range results { + if strings.Contains(strings.ToLower(r), tt.wantMatch) { + found = true + break + } + } + if !found { + t.Errorf("Search results should contain %q", tt.wantMatch) + } + } + }) + } +} + +func TestFileStore_Search_MaxSnippets(t *testing.T) { + tests := []struct { + name string + eventCount int + maxSnippets int + wantMax int + }{ + { + name: "limits to maxSnippets", + eventCount: 20, + maxSnippets: 5, + wantMax: 5, + }, + { + name: "returns all if under limit", + eventCount: 3, + maxSnippets: 10, + wantMax: 3, + }, + { + name: "default maxSnippets when zero", + eventCount: 20, + maxSnippets: 0, + wantMax: 8, // default is 8 + }, + { + name: "maxSnippets of 1", + eventCount: 10, + maxSnippets: 1, + wantMax: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + session := "maxsnippets-test" + + // Create events with searchable text + events := make([]Event, tt.eventCount) + for i := 0; i < tt.eventCount; i++ { + events[i] = Event{ + Kind: "message", + Role: "user", + Text: "This is a searchable test message number " + string(rune('A'+i)), + } + } + + if err := store.Append(session, events); err != nil { + t.Fatalf("Append() error = %v", err) + } + + results, err := store.Search(session, "searchable test message", 100000, tt.maxSnippets) + if err != nil { + t.Fatalf("Search() error = %v", err) + } + + if len(results) > tt.wantMax { + t.Errorf("Search() returned %d results, want at most %d", len(results), tt.wantMax) + } + }) + } +} + +func TestFileStore_Search_MaxChars(t *testing.T) { + tests := []struct { + name string + events []Event + maxChars int + wantUnder bool + }{ + { + name: "respects maxChars limit", + events: []Event{ + {Kind: "message", Text: strings.Repeat("searchable text ", 100)}, + {Kind: "message", Text: strings.Repeat("more searchable text ", 100)}, + }, + maxChars: 500, + wantUnder: true, + }, + { + name: "default maxChars when zero", + events: []Event{ + {Kind: "message", Text: "searchable text"}, + }, + maxChars: 0, + wantUnder: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + session := "maxchars-test" + + if err := store.Append(session, tt.events); err != nil { + t.Fatalf("Append() error = %v", err) + } + + results, err := store.Search(session, "searchable text", tt.maxChars, 100) + if err != nil { + t.Fatalf("Search() error = %v", err) + } + + if tt.wantUnder { + totalChars := 0 + for _, r := range results { + totalChars += len(r) + } + + limit := tt.maxChars + if limit <= 0 { + limit = 6000 // default + } + + if totalChars > limit { + t.Errorf("Total chars %d exceeds maxChars %d", totalChars, limit) + } + } + }) + } +} + +func TestFileStore_Search_NonExistentSession(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + + results, err := store.Search("nonexistent-session", "query", 6000, 10) + if err != nil { + t.Errorf("Search() should not error for nonexistent session, got: %v", err) + } + if len(results) != 0 { + t.Errorf("Search() should return empty for nonexistent session, got %d results", len(results)) + } +} + +func TestFileStore_NilStore(t *testing.T) { + var store *FileStore + + t.Run("Append on nil store", func(t *testing.T) { + err := store.Append("session", []Event{{Kind: "message", Text: "test"}}) + if err == nil { + t.Error("Expected error for nil store, got nil") + } + }) + + t.Run("Search on nil store", func(t *testing.T) { + _, err := store.Search("session", "query", 100, 10) + if err == nil { + t.Error("Expected error for nil store, got nil") + } + }) +} + +func TestFileStore_EmptyBaseDir(t *testing.T) { + store := NewFileStore("") + + t.Run("Append with empty BaseDir", func(t *testing.T) { + err := store.Append("session", []Event{{Kind: "message", Text: "test"}}) + if err == nil { + t.Error("Expected error for empty BaseDir, got nil") + } + }) + + t.Run("Search with empty BaseDir", func(t *testing.T) { + _, err := store.Search("session", "query", 100, 10) + if err == nil { + t.Error("Expected error for empty BaseDir, got nil") + } + }) +} + +func TestFileStore_SearchCache(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + session := "cache-test" + + events := []Event{ + {Kind: "message", Role: "user", Text: "This is a cacheable message"}, + } + + if err := store.Append(session, events); err != nil { + t.Fatalf("Append() error = %v", err) + } + + // First search populates cache + results1, err := store.Search(session, "cacheable", 6000, 10) + if err != nil { + t.Fatalf("Search() error = %v", err) + } + + // Second search should use cache (same query) + results2, err := store.Search(session, "cacheable", 6000, 10) + if err != nil { + t.Fatalf("Search() error = %v", err) + } + + if len(results1) != len(results2) { + t.Errorf("Cached result count mismatch: %d vs %d", len(results1), len(results2)) + } + + // Append invalidates cache + if err := store.Append(session, []Event{{Kind: "message", Text: "New cacheable event"}}); err != nil { + t.Fatalf("Append() error = %v", err) + } + + // Search after append should get fresh results + results3, err := store.Search(session, "cacheable", 6000, 10) + if err != nil { + t.Fatalf("Search() error = %v", err) + } + + if len(results3) <= len(results1) { + t.Logf("Results after append: %d (was %d)", len(results3), len(results1)) + } +} + +func TestFileStore_SessionDir(t *testing.T) { + tests := []struct { + name string + session string + wantDir bool + }{ + { + name: "normal session name", + session: "my-session-123", + wantDir: true, + }, + { + name: "session with special characters", + session: "session/with\\special:chars*", + wantDir: true, + }, + { + name: "empty session", + session: "", + wantDir: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + + dir := store.SessionDir(tt.session) + if tt.wantDir && dir == "" { + t.Error("SessionDir() returned empty string") + } + if tt.wantDir && !strings.HasPrefix(dir, tmpDir) { + t.Errorf("SessionDir() = %q, should be under %q", dir, tmpDir) + } + }) + } +} diff --git a/internal/memory/prune_test.go b/internal/memory/prune_test.go new file mode 100644 index 00000000..efdfe279 --- /dev/null +++ b/internal/memory/prune_test.go @@ -0,0 +1,610 @@ +package memory + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestPrune_ByTokenCount(t *testing.T) { + tests := []struct { + name string + sessionCount int + eventsPerSession int + maxBytesPerSession int64 + wantSessionsTrimmed int + }{ + { + name: "trims large sessions", + sessionCount: 2, + eventsPerSession: 100, + maxBytesPerSession: 500, + wantSessionsTrimmed: 2, + }, + { + name: "no trimming needed for small sessions", + sessionCount: 2, + eventsPerSession: 2, + maxBytesPerSession: 100000, + wantSessionsTrimmed: 0, + }, + { + name: "zero maxBytes skips trimming", + sessionCount: 2, + eventsPerSession: 100, + maxBytesPerSession: 0, + wantSessionsTrimmed: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + + // Create sessions with events + for i := 0; i < tt.sessionCount; i++ { + session := "session-" + string(rune('A'+i)) + events := make([]Event, tt.eventsPerSession) + for j := 0; j < tt.eventsPerSession; j++ { + events[j] = Event{ + TS: time.Now(), + Kind: "message", + Role: "user", + Text: strings.Repeat("This is a test message with some content. ", 10), + } + } + if err := store.Append(session, events); err != nil { + t.Fatalf("Append() error = %v", err) + } + } + + // Prune sessions + result, err := store.PruneSessions(0, 0, tt.maxBytesPerSession) + if err != nil { + t.Fatalf("PruneSessions() error = %v", err) + } + + if result.SessionsTrimmed != tt.wantSessionsTrimmed { + t.Errorf("SessionsTrimmed = %d, want %d", result.SessionsTrimmed, tt.wantSessionsTrimmed) + } + + if result.SessionsTrimmed > 0 && result.BytesFreed <= 0 { + t.Error("Expected BytesFreed > 0 when sessions are trimmed") + } + }) + } +} + +func TestPrune_PreservesRecent(t *testing.T) { + tests := []struct { + name string + sessionAges []int // days ago + maxAgeDays int + maxSessions int + wantRemoved int + wantKept []string + }{ + { + name: "preserves recent sessions by age", + sessionAges: []int{1, 5, 10, 20, 40}, // days ago + maxAgeDays: 30, + maxSessions: 0, + wantRemoved: 1, // only 40 days old is removed + wantKept: []string{"session-0", "session-1", "session-2", "session-3"}, + }, + { + name: "preserves by max sessions count", + sessionAges: []int{1, 2, 3, 4, 5}, + maxAgeDays: 0, + maxSessions: 3, + wantRemoved: 2, // keeps 3 most recent + wantKept: []string{"session-0", "session-1", "session-2"}, + }, + { + name: "combines age and count limits", + sessionAges: []int{1, 5, 35, 40}, + maxAgeDays: 30, + maxSessions: 2, + wantRemoved: 2, // 35 and 40 days old are removed + wantKept: []string{"session-0", "session-1"}, + }, + { + name: "no pruning when all within limits", + sessionAges: []int{1, 2, 3}, + maxAgeDays: 30, + maxSessions: 10, + wantRemoved: 0, + wantKept: []string{"session-0", "session-1", "session-2"}, + }, + { + name: "removes all old sessions", + sessionAges: []int{100, 200, 300}, + maxAgeDays: 30, + maxSessions: 0, + wantRemoved: 3, + wantKept: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + + // Create sessions with different ages + for i, daysAgo := range tt.sessionAges { + session := "session-" + string(rune('0'+i)) + sessionDir := filepath.Join(tmpDir, "sessions", session) + if err := os.MkdirAll(sessionDir, 0o755); err != nil { + t.Fatalf("Failed to create session dir: %v", err) + } + + // Create events file + eventsPath := filepath.Join(sessionDir, "events.jsonl") + if err := os.WriteFile(eventsPath, []byte(`{"kind":"message","text":"test"}`+"\n"), 0o644); err != nil { + t.Fatalf("Failed to create events file: %v", err) + } + + // Set modification time + modTime := time.Now().Add(-time.Duration(daysAgo) * 24 * time.Hour) + if err := os.Chtimes(eventsPath, modTime, modTime); err != nil { + t.Fatalf("Failed to set file times: %v", err) + } + } + + // Prune sessions + result, err := store.PruneSessions(tt.maxAgeDays, tt.maxSessions, 0) + if err != nil { + t.Fatalf("PruneSessions() error = %v", err) + } + + if result.SessionsRemoved != tt.wantRemoved { + t.Errorf("SessionsRemoved = %d, want %d", result.SessionsRemoved, tt.wantRemoved) + } + + // Verify kept sessions exist + for _, session := range tt.wantKept { + sessionDir := filepath.Join(tmpDir, "sessions", session) + if _, err := os.Stat(sessionDir); os.IsNotExist(err) { + t.Errorf("Session %q should have been preserved but was removed", session) + } + } + + // Verify removed sessions don't exist + sessions, _ := store.ListSessions(0) + if len(sessions) != len(tt.wantKept) { + t.Errorf("Expected %d sessions remaining, got %d", len(tt.wantKept), len(sessions)) + } + }) + } +} + +func TestPrune_SemanticNamespaces(t *testing.T) { + tests := []struct { + name string + namespaceCount int + recordsPerNs int + maxAgeDays int + maxNamespaces int + maxBytesPerNs int64 + wantRemoved int + wantTrimmed int + }{ + { + name: "removes old namespaces", + namespaceCount: 3, + recordsPerNs: 5, + maxAgeDays: 30, + maxNamespaces: 0, + maxBytesPerNs: 0, + wantRemoved: 0, // all are recent + wantTrimmed: 0, + }, + { + name: "limits namespace count", + namespaceCount: 5, + recordsPerNs: 5, + maxAgeDays: 0, + maxNamespaces: 3, + maxBytesPerNs: 0, + wantRemoved: 2, + wantTrimmed: 0, + }, + { + name: "trims large namespaces", + namespaceCount: 2, + recordsPerNs: 500, + maxAgeDays: 0, + maxNamespaces: 0, + maxBytesPerNs: 2000, + wantRemoved: 0, + wantTrimmed: 2, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + + // Create namespaces with records + for i := 0; i < tt.namespaceCount; i++ { + namespace := "namespace-" + string(rune('A'+i)) + records := make([]SemanticRecord, tt.recordsPerNs) + for j := 0; j < tt.recordsPerNs; j++ { + // Make each text unique to avoid deduplication + records[j] = SemanticRecord{ + TS: time.Now(), + Text: strings.Repeat("Test record content ", 20) + " " + string(rune('A'+i)) + string(rune('0'+j%10)), + Vec: []float32{float32(i), float32(j), 0.5}, + } + } + if err := store.AppendSemantic(namespace, records); err != nil { + t.Fatalf("AppendSemantic() error = %v", err) + } + // Add small delay to ensure different modification times + time.Sleep(10 * time.Millisecond) + } + + // Prune semantic namespaces + result, err := store.PruneSemantic(tt.maxAgeDays, tt.maxNamespaces, tt.maxBytesPerNs) + if err != nil { + t.Fatalf("PruneSemantic() error = %v", err) + } + + if result.SemanticNamespacesRemoved != tt.wantRemoved { + t.Errorf("SemanticNamespacesRemoved = %d, want %d", result.SemanticNamespacesRemoved, tt.wantRemoved) + } + + if result.SemanticNamespacesTrimmed != tt.wantTrimmed { + t.Errorf("SemanticNamespacesTrimmed = %d, want %d", result.SemanticNamespacesTrimmed, tt.wantTrimmed) + } + }) + } +} + +func TestTrimJSONLFile(t *testing.T) { + tests := []struct { + name string + content string + maxBytes int64 + wantTrim bool + wantFreed bool + }{ + { + name: "trims file exceeding maxBytes", + content: `{"a":1}` + "\n" + `{"b":2}` + "\n" + `{"c":3}` + "\n" + `{"d":4}` + "\n", + maxBytes: 20, + wantTrim: true, + wantFreed: true, + }, + { + name: "no trim needed for small file", + content: `{"a":1}` + "\n", + maxBytes: 1000, + wantTrim: false, + wantFreed: false, + }, + { + name: "zero maxBytes skips trimming", + content: `{"a":1}` + "\n" + `{"b":2}` + "\n", + maxBytes: 0, + wantTrim: false, + wantFreed: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "test.jsonl") + + if err := os.WriteFile(path, []byte(tt.content), 0o644); err != nil { + t.Fatalf("Failed to write test file: %v", err) + } + + originalSize := int64(len(tt.content)) + trimmed, freed := trimJSONLFile(path, tt.maxBytes) + + if trimmed != tt.wantTrim { + t.Errorf("trimJSONLFile() trimmed = %v, want %v", trimmed, tt.wantTrim) + } + + if tt.wantFreed && freed <= 0 { + t.Errorf("Expected freed > 0, got %d", freed) + } + + if !tt.wantFreed && freed != 0 { + t.Errorf("Expected freed = 0, got %d", freed) + } + + // Verify file was actually trimmed + if tt.wantTrim { + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("Failed to read trimmed file: %v", err) + } + if int64(len(data)) >= originalSize { + t.Errorf("File was not actually trimmed: %d >= %d", len(data), originalSize) + } + } + }) + } +} + +func TestPruneResult_Fields(t *testing.T) { + result := PruneResult{ + SessionsRemoved: 5, + SessionsTrimmed: 3, + SemanticNamespacesRemoved: 2, + SemanticNamespacesTrimmed: 1, + BytesFreed: 1024 * 1024, + } + + if result.SessionsRemoved != 5 { + t.Errorf("SessionsRemoved = %d, want 5", result.SessionsRemoved) + } + if result.SessionsTrimmed != 3 { + t.Errorf("SessionsTrimmed = %d, want 3", result.SessionsTrimmed) + } + if result.SemanticNamespacesRemoved != 2 { + t.Errorf("SemanticNamespacesRemoved = %d, want 2", result.SemanticNamespacesRemoved) + } + if result.SemanticNamespacesTrimmed != 1 { + t.Errorf("SemanticNamespacesTrimmed = %d, want 1", result.SemanticNamespacesTrimmed) + } + if result.BytesFreed != 1024*1024 { + t.Errorf("BytesFreed = %d, want %d", result.BytesFreed, 1024*1024) + } +} + +func TestSessionInfo_Fields(t *testing.T) { + now := time.Now() + info := SessionInfo{ + Key: "test-session", + Path: "/path/to/session", + UpdatedAt: now, + SizeBytes: 2048, + EventsBytes: 1024, + HasSummary: true, + HasTodo: true, + HasPinned: false, + HasAnchorPending: true, + SemanticDisabled: false, + } + + if info.Key != "test-session" { + t.Errorf("Key = %q, want %q", info.Key, "test-session") + } + if info.Path != "/path/to/session" { + t.Errorf("Path = %q, want %q", info.Path, "/path/to/session") + } + if !info.UpdatedAt.Equal(now) { + t.Errorf("UpdatedAt mismatch") + } + if info.SizeBytes != 2048 { + t.Errorf("SizeBytes = %d, want 2048", info.SizeBytes) + } + if info.EventsBytes != 1024 { + t.Errorf("EventsBytes = %d, want 1024", info.EventsBytes) + } + if !info.HasSummary { + t.Error("HasSummary should be true") + } + if !info.HasTodo { + t.Error("HasTodo should be true") + } + if info.HasPinned { + t.Error("HasPinned should be false") + } + if !info.HasAnchorPending { + t.Error("HasAnchorPending should be true") + } + if info.SemanticDisabled { + t.Error("SemanticDisabled should be false") + } +} + +func TestListSessions(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + + // Create some sessions + sessions := []string{"alpha", "beta", "gamma"} + for _, session := range sessions { + events := []Event{{Kind: "message", Text: "test"}} + if err := store.Append(session, events); err != nil { + t.Fatalf("Append() error = %v", err) + } + // Small delay to ensure different mod times + time.Sleep(10 * time.Millisecond) + } + + // List all sessions + list, err := store.ListSessions(0) + if err != nil { + t.Fatalf("ListSessions() error = %v", err) + } + + if len(list) != len(sessions) { + t.Errorf("ListSessions() returned %d, want %d", len(list), len(sessions)) + } + + // List with limit + limited, err := store.ListSessions(2) + if err != nil { + t.Fatalf("ListSessions() error = %v", err) + } + + if len(limited) != 2 { + t.Errorf("ListSessions(2) returned %d, want 2", len(limited)) + } + + // Verify most recent is first (gamma was created last) + if len(limited) > 0 && limited[0].Key != "gamma" { + t.Errorf("Most recent session should be 'gamma', got %q", limited[0].Key) + } +} + +func TestGetSessionInfo(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + session := "info-test" + + // Create session with various files + events := []Event{{Kind: "message", Text: "test event"}} + if err := store.Append(session, events); err != nil { + t.Fatalf("Append() error = %v", err) + } + + if err := store.WriteSummary(session, "Test summary", 0); err != nil { + t.Fatalf("WriteSummary() error = %v", err) + } + + if err := store.WriteTodo(session, "Test todo", 0); err != nil { + t.Fatalf("WriteTodo() error = %v", err) + } + + // Get session info + info, err := store.GetSessionInfo(session) + if err != nil { + t.Fatalf("GetSessionInfo() error = %v", err) + } + + if info.Key != session { + t.Errorf("Key = %q, want %q", info.Key, session) + } + if !info.HasSummary { + t.Error("HasSummary should be true") + } + if !info.HasTodo { + t.Error("HasTodo should be true") + } + if info.SizeBytes <= 0 { + t.Error("SizeBytes should be > 0") + } + if info.EventsBytes <= 0 { + t.Error("EventsBytes should be > 0") + } +} + +func TestReadEventTail(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + session := "tail-test" + + // Create many events + events := make([]Event, 20) + for i := 0; i < 20; i++ { + events[i] = Event{ + Kind: "message", + Role: "user", + Text: "Message " + string(rune('A'+i)), + } + } + + if err := store.Append(session, events); err != nil { + t.Fatalf("Append() error = %v", err) + } + + // Read tail with limit + tail, err := store.ReadEventTail(session, 5) + if err != nil { + t.Fatalf("ReadEventTail() error = %v", err) + } + + if len(tail) != 5 { + t.Errorf("ReadEventTail() returned %d events, want 5", len(tail)) + } + + // Verify chronological order (oldest first in result) + if len(tail) > 0 { + // Should be last 5 events: P, Q, R, S, T + expectedFirst := "Message P" + if tail[0].Text != expectedFirst { + t.Errorf("First event text = %q, want %q", tail[0].Text, expectedFirst) + } + } +} + +func TestExportSessionZip(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + session := "export-test" + + // Create session with content + events := []Event{{Kind: "message", Text: "test event"}} + if err := store.Append(session, events); err != nil { + t.Fatalf("Append() error = %v", err) + } + + if err := store.WriteSummary(session, "Test summary", 0); err != nil { + t.Fatalf("WriteSummary() error = %v", err) + } + + // Export as zip + sessionDir := store.SessionDir(session) + zipData, err := ExportSessionZip(sessionDir) + if err != nil { + t.Fatalf("ExportSessionZip() error = %v", err) + } + + if len(zipData) == 0 { + t.Error("Expected non-empty zip data") + } + + // Verify it's a valid zip (starts with PK) + if len(zipData) < 2 || zipData[0] != 'P' || zipData[1] != 'K' { + t.Error("Expected valid zip file header") + } +} + +func TestExportSessionZip_EmptyDir(t *testing.T) { + _, err := ExportSessionZip("") + if err == nil { + t.Error("Expected error for empty directory") + } +} + +func TestExportAllZip(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + + // Create multiple sessions + for _, session := range []string{"session-1", "session-2"} { + events := []Event{{Kind: "message", Text: "test in " + session}} + if err := store.Append(session, events); err != nil { + t.Fatalf("Append() error = %v", err) + } + } + + // Export all + zipData, err := ExportAllZip(tmpDir, 10*1024*1024) + if err != nil { + t.Fatalf("ExportAllZip() error = %v", err) + } + + if len(zipData) == 0 { + t.Error("Expected non-empty zip data") + } +} + +func TestExportAllZip_SizeLimit(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + + // Create session with large content + events := []Event{{Kind: "message", Text: strings.Repeat("x", 10000)}} + if err := store.Append("large-session", events); err != nil { + t.Fatalf("Append() error = %v", err) + } + + // Try to export with very small limit + _, err := ExportAllZip(tmpDir, 100) + if err == nil { + t.Error("Expected error when size exceeds limit") + } +} diff --git a/internal/memory/semantic_test.go b/internal/memory/semantic_test.go new file mode 100644 index 00000000..f1905531 --- /dev/null +++ b/internal/memory/semantic_test.go @@ -0,0 +1,826 @@ +package memory + +import ( + "encoding/json" + "math" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestSemanticStore_Append(t *testing.T) { + tests := []struct { + name string + namespace string + records []SemanticRecord + wantErr bool + wantCount int + }{ + { + name: "appends single record", + namespace: "test-namespace", + records: []SemanticRecord{ + { + TS: time.Now(), + Role: "user", + Text: "Hello, world!", + Vec: []float32{0.1, 0.2, 0.3}, + }, + }, + wantErr: false, + wantCount: 1, + }, + { + name: "appends multiple records", + namespace: "multi-record-ns", + records: []SemanticRecord{ + {Text: "First message", Vec: []float32{0.1, 0.2, 0.3}}, + {Text: "Second message", Vec: []float32{0.4, 0.5, 0.6}}, + {Text: "Third message", Vec: []float32{0.7, 0.8, 0.9}}, + }, + wantErr: false, + wantCount: 3, + }, + { + name: "skips records with empty text", + namespace: "skip-empty-ns", + records: []SemanticRecord{ + {Text: "Valid text", Vec: []float32{0.1, 0.2, 0.3}}, + {Text: "", Vec: []float32{0.4, 0.5, 0.6}}, + {Text: " ", Vec: []float32{0.7, 0.8, 0.9}}, + }, + wantErr: false, + wantCount: 1, + }, + { + name: "skips records with empty vector", + namespace: "skip-empty-vec-ns", + records: []SemanticRecord{ + {Text: "Valid", Vec: []float32{0.1, 0.2, 0.3}}, + {Text: "No vector", Vec: []float32{}}, + {Text: "Nil vector", Vec: nil}, + }, + wantErr: false, + wantCount: 1, + }, + { + name: "skips duplicate texts", + namespace: "skip-dupe-ns", + records: []SemanticRecord{ + {Text: "Same text", Vec: []float32{0.1, 0.2, 0.3}}, + {Text: "Same text", Vec: []float32{0.4, 0.5, 0.6}}, + {Text: "Different text", Vec: []float32{0.7, 0.8, 0.9}}, + }, + wantErr: false, + wantCount: 2, + }, + { + name: "empty namespace returns nil", + namespace: "", + records: []SemanticRecord{ + {Text: "Test", Vec: []float32{0.1, 0.2, 0.3}}, + }, + wantErr: false, + wantCount: 0, + }, + { + name: "empty records returns nil", + namespace: "empty-records-ns", + records: []SemanticRecord{}, + wantErr: false, + wantCount: 0, + }, + { + name: "computes norm if not provided", + namespace: "compute-norm-ns", + records: []SemanticRecord{ + {Text: "Norm will be computed", Vec: []float32{3, 4}, Norm: 0}, + }, + wantErr: false, + wantCount: 1, + }, + { + name: "uses provided norm", + namespace: "provided-norm-ns", + records: []SemanticRecord{ + {Text: "Norm is provided", Vec: []float32{3, 4}, Norm: 5.0}, + }, + wantErr: false, + wantCount: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + + err := store.AppendSemantic(tt.namespace, tt.records) + if (err != nil) != tt.wantErr { + t.Errorf("AppendSemantic() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if tt.namespace == "" || len(tt.records) == 0 { + return + } + + // Verify file content + dir := filepath.Join(tmpDir, "semantic") + entries, err := os.ReadDir(dir) + if err != nil { + if tt.wantCount == 0 { + return // No file expected + } + t.Fatalf("Failed to read semantic dir: %v", err) + } + + if len(entries) == 0 && tt.wantCount > 0 { + t.Errorf("Expected semantic namespace directory to be created") + return + } + + // Find the items.jsonl file + var itemsPath string + for _, e := range entries { + if e.IsDir() { + itemsPath = filepath.Join(dir, e.Name(), "items.jsonl") + break + } + } + + if itemsPath == "" { + if tt.wantCount > 0 { + t.Error("items.jsonl not found") + } + return + } + + data, err := os.ReadFile(itemsPath) + if err != nil { + t.Fatalf("Failed to read items.jsonl: %v", err) + } + + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + actualCount := 0 + for _, line := range lines { + if strings.TrimSpace(line) != "" { + actualCount++ + } + } + + if actualCount != tt.wantCount { + t.Errorf("Expected %d records, got %d", tt.wantCount, actualCount) + } + }) + } +} + +func TestSemanticStore_Search(t *testing.T) { + tests := []struct { + name string + records []SemanticRecord + queryVec []float32 + maxChars int + maxSnippets int + wantCount int + wantFirst string + }{ + { + name: "finds similar vectors", + records: []SemanticRecord{ + {Text: "Apple is a fruit", Vec: []float32{0.9, 0.1, 0.0}}, + {Text: "Car is a vehicle", Vec: []float32{0.1, 0.9, 0.0}}, + {Text: "Orange is a fruit", Vec: []float32{0.85, 0.15, 0.0}}, + }, + queryVec: []float32{0.9, 0.1, 0.0}, + maxChars: 6000, + maxSnippets: 10, + wantCount: 3, + wantFirst: "Apple is a fruit", + }, + { + name: "returns empty for no matches", + records: []SemanticRecord{ + {Text: "Text 1", Vec: []float32{1, 0, 0}}, + {Text: "Text 2", Vec: []float32{0, 1, 0}}, + }, + queryVec: []float32{0, 0, 1}, + maxChars: 6000, + maxSnippets: 10, + wantCount: 0, + wantFirst: "", + }, + { + name: "respects maxSnippets", + records: []SemanticRecord{ + {Text: "Text 1", Vec: []float32{0.9, 0.1, 0.0}}, + {Text: "Text 2", Vec: []float32{0.85, 0.15, 0.0}}, + {Text: "Text 3", Vec: []float32{0.8, 0.2, 0.0}}, + {Text: "Text 4", Vec: []float32{0.75, 0.25, 0.0}}, + }, + queryVec: []float32{0.9, 0.1, 0.0}, + maxChars: 6000, + maxSnippets: 2, + wantCount: 2, + wantFirst: "Text 1", + }, + { + name: "empty query vector returns nil", + records: []SemanticRecord{ + {Text: "Some text", Vec: []float32{0.1, 0.2, 0.3}}, + }, + queryVec: []float32{}, + maxChars: 6000, + maxSnippets: 10, + wantCount: 0, + wantFirst: "", + }, + { + name: "orders by similarity score", + records: []SemanticRecord{ + {Text: "Low similarity", Vec: []float32{0.1, 0.9, 0.0}}, + {Text: "High similarity", Vec: []float32{0.95, 0.05, 0.0}}, + {Text: "Medium similarity", Vec: []float32{0.5, 0.5, 0.0}}, + }, + queryVec: []float32{1.0, 0.0, 0.0}, + maxChars: 6000, + maxSnippets: 10, + wantCount: 3, + wantFirst: "High similarity", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + namespace := "search-test-ns" + + // Append records + if len(tt.records) > 0 { + if err := store.AppendSemantic(namespace, tt.records); err != nil { + t.Fatalf("AppendSemantic() error = %v", err) + } + } + + // Search + results, err := store.SearchSemantic(namespace, tt.queryVec, tt.maxChars, tt.maxSnippets) + if err != nil { + t.Fatalf("SearchSemantic() error = %v", err) + } + + if len(results) != tt.wantCount { + t.Errorf("SearchSemantic() returned %d results, want %d", len(results), tt.wantCount) + } + + if tt.wantFirst != "" && len(results) > 0 { + if results[0] != tt.wantFirst { + t.Errorf("First result = %q, want %q", results[0], tt.wantFirst) + } + } + }) + } +} + +func TestSemanticStore_SearchWithText(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + namespace := "text-search-ns" + + records := []SemanticRecord{ + {Text: "Python programming tutorial", Vec: []float32{0.9, 0.1, 0.0}}, + {Text: "Java development guide", Vec: []float32{0.85, 0.15, 0.0}}, + {Text: "Python data science course", Vec: []float32{0.88, 0.12, 0.0}}, + } + + if err := store.AppendSemantic(namespace, records); err != nil { + t.Fatalf("AppendSemantic() error = %v", err) + } + + // Search with both vector and text (keyword boost) + results, err := store.SearchSemanticWithText(namespace, []float32{0.9, 0.1, 0.0}, "python", 6000, 10) + if err != nil { + t.Fatalf("SearchSemanticWithText() error = %v", err) + } + + if len(results) == 0 { + t.Error("Expected results from SearchSemanticWithText") + } + + // Python results should be boosted + pythonCount := 0 + for _, r := range results { + if strings.Contains(strings.ToLower(r), "python") { + pythonCount++ + } + } + + if pythonCount == 0 { + t.Error("Expected Python-related results to be present") + } +} + +func TestEmbedding_Normalize(t *testing.T) { + tests := []struct { + name string + vec []float32 + wantNorm float32 + wantZero bool + }{ + { + name: "unit vector", + vec: []float32{1, 0, 0}, + wantNorm: 1.0, + wantZero: false, + }, + { + name: "3-4-5 right triangle", + vec: []float32{3, 4}, + wantNorm: 5.0, + wantZero: false, + }, + { + name: "all zeros", + vec: []float32{0, 0, 0}, + wantNorm: 0, + wantZero: true, + }, + { + name: "empty vector", + vec: []float32{}, + wantNorm: 0, + wantZero: true, + }, + { + name: "negative values", + vec: []float32{-3, 4}, + wantNorm: 5.0, + wantZero: false, + }, + { + name: "high dimensional", + vec: []float32{1, 1, 1, 1}, + wantNorm: 2.0, // sqrt(4) = 2 + wantZero: false, + }, + { + name: "small values", + vec: []float32{0.001, 0.001, 0.001}, + wantNorm: float32(math.Sqrt(0.000003)), + wantZero: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + norm := vectorNorm(tt.vec) + + if tt.wantZero && norm != 0 { + t.Errorf("vectorNorm() = %v, want 0", norm) + return + } + + if !tt.wantZero { + tolerance := float32(0.0001) + if diff := math.Abs(float64(norm - tt.wantNorm)); diff > float64(tolerance) { + t.Errorf("vectorNorm() = %v, want %v (diff: %v)", norm, tt.wantNorm, diff) + } + } + }) + } +} + +func TestEmbedding_CosineSimilarity(t *testing.T) { + tests := []struct { + name string + vecA []float32 + normA float32 + vecB []float32 + normB float32 + wantSim float32 + tolerance float32 + }{ + { + name: "identical vectors", + vecA: []float32{1, 0, 0}, + normA: 1.0, + vecB: []float32{1, 0, 0}, + normB: 1.0, + wantSim: 1.0, + tolerance: 0.0001, + }, + { + name: "orthogonal vectors", + vecA: []float32{1, 0, 0}, + normA: 1.0, + vecB: []float32{0, 1, 0}, + normB: 1.0, + wantSim: 0.0, + tolerance: 0.0001, + }, + { + name: "opposite vectors", + vecA: []float32{1, 0, 0}, + normA: 1.0, + vecB: []float32{-1, 0, 0}, + normB: 1.0, + wantSim: -1.0, + tolerance: 0.0001, + }, + { + name: "45 degree angle", + vecA: []float32{1, 0}, + normA: 1.0, + vecB: []float32{1, 1}, + normB: float32(math.Sqrt(2)), + wantSim: float32(1.0 / math.Sqrt(2)), // cos(45) = 1/sqrt(2) + tolerance: 0.0001, + }, + { + name: "non-unit vectors", + vecA: []float32{3, 4}, + normA: 5.0, + vecB: []float32{6, 8}, + normB: 10.0, + wantSim: 1.0, // Same direction + tolerance: 0.0001, + }, + { + name: "zero norm A returns 0", + vecA: []float32{0, 0, 0}, + normA: 0, + vecB: []float32{1, 0, 0}, + normB: 1.0, + wantSim: 0, + tolerance: 0, + }, + { + name: "zero norm B returns 0", + vecA: []float32{1, 0, 0}, + normA: 1.0, + vecB: []float32{0, 0, 0}, + normB: 0, + wantSim: 0, + tolerance: 0, + }, + { + name: "different length vectors", + vecA: []float32{1, 2, 3, 4, 5}, + normA: vectorNorm([]float32{1, 2, 3, 4, 5}), + vecB: []float32{1, 2, 3}, + normB: vectorNorm([]float32{1, 2, 3}), + wantSim: float32(14.0) / (vectorNorm([]float32{1, 2, 3, 4, 5}) * vectorNorm([]float32{1, 2, 3})), + tolerance: 0.0001, + }, + { + name: "high dimensional similar", + vecA: []float32{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8}, + normA: vectorNorm([]float32{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8}), + vecB: []float32{0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85}, + normB: vectorNorm([]float32{0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85}), + wantSim: 0.999, // Very similar + tolerance: 0.01, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sim := cosineSim(tt.vecA, tt.normA, tt.vecB, tt.normB) + + diff := math.Abs(float64(sim - tt.wantSim)) + if diff > float64(tt.tolerance) { + t.Errorf("cosineSim() = %v, want %v (diff: %v, tolerance: %v)", sim, tt.wantSim, diff, tt.tolerance) + } + }) + } +} + +func TestSemanticRecord_Serialization(t *testing.T) { + tests := []struct { + name string + record SemanticRecord + }{ + { + name: "full record", + record: SemanticRecord{ + TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC), + Role: "user", + Text: "Test message", + Vec: []float32{0.1, 0.2, 0.3}, + Norm: 0.374, + Source: "chat", + Session: "session-123", + Repo: "org/repo", + }, + }, + { + name: "minimal record", + record: SemanticRecord{ + Text: "Minimal", + Vec: []float32{1.0}, + Norm: 1.0, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := json.Marshal(tt.record) + if err != nil { + t.Fatalf("json.Marshal() error = %v", err) + } + + var decoded SemanticRecord + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatalf("json.Unmarshal() error = %v", err) + } + + if decoded.Text != tt.record.Text { + t.Errorf("Text mismatch: got %q, want %q", decoded.Text, tt.record.Text) + } + if len(decoded.Vec) != len(tt.record.Vec) { + t.Errorf("Vec length mismatch: got %d, want %d", len(decoded.Vec), len(tt.record.Vec)) + } + if decoded.Norm != tt.record.Norm { + t.Errorf("Norm mismatch: got %v, want %v", decoded.Norm, tt.record.Norm) + } + }) + } +} + +func TestSemanticStore_ReadSemanticTail(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + namespace := "tail-test-ns" + + // Append multiple records + records := make([]SemanticRecord, 10) + for i := 0; i < 10; i++ { + records[i] = SemanticRecord{ + TS: time.Now().Add(time.Duration(i) * time.Minute), + Text: "Message " + string(rune('A'+i)), + Vec: []float32{float32(i), float32(i + 1), float32(i + 2)}, + } + } + + if err := store.AppendSemantic(namespace, records); err != nil { + t.Fatalf("AppendSemantic() error = %v", err) + } + + // Read tail with limit + tail, err := store.ReadSemanticTail(namespace, 5) + if err != nil { + t.Fatalf("ReadSemanticTail() error = %v", err) + } + + if len(tail) != 5 { + t.Errorf("ReadSemanticTail() returned %d records, want 5", len(tail)) + } + + // Verify order (should be chronological, last 5 records) + if len(tail) > 0 { + // First in tail should be 6th record (index 5) + if !strings.Contains(tail[0].Text, "Message F") { + t.Errorf("First record = %q, expected Message F", tail[0].Text) + } + } +} + +func TestSemanticStore_ListNamespaces(t *testing.T) { + tmpDir := t.TempDir() + store := NewFileStore(tmpDir) + + namespaces := []string{"ns-alpha", "ns-beta", "ns-gamma"} + for _, ns := range namespaces { + records := []SemanticRecord{ + {Text: "Record in " + ns, Vec: []float32{0.1, 0.2, 0.3}}, + } + if err := store.AppendSemantic(ns, records); err != nil { + t.Fatalf("AppendSemantic() error = %v", err) + } + } + + // List all namespaces + list, err := store.ListSemanticNamespaces(0) + if err != nil { + t.Fatalf("ListSemanticNamespaces() error = %v", err) + } + + if len(list) != len(namespaces) { + t.Errorf("ListSemanticNamespaces() returned %d, want %d", len(list), len(namespaces)) + } + + // List with limit + limited, err := store.ListSemanticNamespaces(2) + if err != nil { + t.Fatalf("ListSemanticNamespaces() error = %v", err) + } + + if len(limited) != 2 { + t.Errorf("ListSemanticNamespaces(2) returned %d, want 2", len(limited)) + } +} + +func TestTokenizeSemanticQuery(t *testing.T) { + tests := []struct { + name string + query string + max int + want []string + wantN int + }{ + { + name: "basic tokenization", + query: "Hello World", + max: 10, + want: []string{"hello", "world"}, + wantN: 2, + }, + { + name: "removes stop words", + query: "the quick brown fox and the lazy dog", + max: 10, + want: []string{"quick", "brown", "fox", "lazy", "dog"}, + wantN: 5, + }, + { + name: "respects max limit", + query: "one two three four five six seven", + max: 3, + want: []string{"one", "two", "three"}, + wantN: 3, + }, + { + name: "removes short words", + query: "I am a test is it ok", + max: 10, + want: []string{"test"}, + wantN: 1, + }, + { + name: "handles special characters", + query: "hello-world test_case foo.bar", + max: 10, + want: []string{"hello-world", "test_case", "foo", "bar"}, + wantN: 4, + }, + { + name: "empty query", + query: "", + max: 10, + want: nil, + wantN: 0, + }, + { + name: "only stop words", + query: "the and for with", + max: 10, + want: nil, + wantN: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tokens := tokenizeSemanticQuery(tt.query, tt.max) + + if len(tokens) != tt.wantN { + t.Errorf("tokenizeSemanticQuery() returned %d tokens, want %d", len(tokens), tt.wantN) + } + + if tt.want != nil { + for i, want := range tt.want { + if i >= len(tokens) { + t.Errorf("Missing token at index %d: want %q", i, want) + continue + } + if tokens[i] != want { + t.Errorf("Token %d = %q, want %q", i, tokens[i], want) + } + } + } + }) + } +} + +func TestSemanticTokenOverlap(t *testing.T) { + tests := []struct { + name string + tokens []string + text string + want int + }{ + { + name: "all tokens match", + tokens: []string{"hello", "world"}, + text: "Hello World Test", + want: 2, + }, + { + name: "some tokens match", + tokens: []string{"hello", "world", "foo"}, + text: "Hello bar baz", + want: 1, + }, + { + name: "no tokens match", + tokens: []string{"foo", "bar"}, + text: "Hello World", + want: 0, + }, + { + name: "empty tokens", + tokens: []string{}, + text: "Hello World", + want: 0, + }, + { + name: "empty text", + tokens: []string{"hello"}, + text: "", + want: 0, + }, + { + name: "case insensitive", + tokens: []string{"hello"}, + text: "HELLO World", + want: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + overlap := semanticTokenOverlap(tt.tokens, tt.text) + if overlap != tt.want { + t.Errorf("semanticTokenOverlap() = %d, want %d", overlap, tt.want) + } + }) + } +} + +func TestSemanticRecencyScore(t *testing.T) { + now := time.Now() + + tests := []struct { + name string + ts time.Time + windowDays int + wantMin float32 + wantMax float32 + }{ + { + name: "current time", + ts: now, + windowDays: 30, + wantMin: 0.99, + wantMax: 1.0, + }, + { + name: "half window age", + ts: now.Add(-15 * 24 * time.Hour), + windowDays: 30, + wantMin: 0.4, + wantMax: 0.6, + }, + { + name: "at window edge", + ts: now.Add(-30 * 24 * time.Hour), + windowDays: 30, + wantMin: 0.0, + wantMax: 0.01, + }, + { + name: "beyond window", + ts: now.Add(-60 * 24 * time.Hour), + windowDays: 30, + wantMin: 0.0, + wantMax: 0.0, + }, + { + name: "zero time", + ts: time.Time{}, + windowDays: 30, + wantMin: 0.0, + wantMax: 0.0, + }, + { + name: "zero window", + ts: now, + windowDays: 0, + wantMin: 0.0, + wantMax: 0.0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + score := semanticRecencyScore(now, tt.ts, tt.windowDays) + if score < tt.wantMin || score > tt.wantMax { + t.Errorf("semanticRecencyScore() = %v, want between %v and %v", score, tt.wantMin, tt.wantMax) + } + }) + } +} diff --git a/internal/memory/store_test.go b/internal/memory/store_test.go new file mode 100644 index 00000000..37c1b0b5 --- /dev/null +++ b/internal/memory/store_test.go @@ -0,0 +1,365 @@ +package memory + +import ( + "encoding/json" + "testing" + "time" +) + +func TestEvent_JSON_Serialization(t *testing.T) { + tests := []struct { + name string + event Event + wantErr bool + }{ + { + name: "full event with all fields", + event: Event{ + TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC), + Kind: "message", + Role: "user", + Type: "text", + Text: "Hello, world!", + Meta: map[string]string{"key": "value", "model": "gpt-4"}, + }, + wantErr: false, + }, + { + name: "minimal event with only required fields", + event: Event{ + TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC), + Kind: "system", + }, + wantErr: false, + }, + { + name: "event with empty optional fields", + event: Event{ + TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC), + Kind: "assistant", + Role: "", + Type: "", + Text: "", + Meta: nil, + }, + wantErr: false, + }, + { + name: "event with zero timestamp", + event: Event{ + Kind: "message", + Text: "test", + }, + wantErr: false, + }, + { + name: "event with unicode text", + event: Event{ + TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC), + Kind: "message", + Text: "Hello \u4e16\u754c! \U0001F600", + }, + wantErr: false, + }, + { + name: "event with multiline text", + event: Event{ + TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC), + Kind: "message", + Text: "Line 1\nLine 2\nLine 3", + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Marshal to JSON + data, err := json.Marshal(tt.event) + if (err != nil) != tt.wantErr { + t.Errorf("json.Marshal() error = %v, wantErr %v", err, tt.wantErr) + return + } + if tt.wantErr { + return + } + + // Unmarshal back to Event + var decoded Event + err = json.Unmarshal(data, &decoded) + if err != nil { + t.Errorf("json.Unmarshal() error = %v", err) + return + } + + // Compare fields + if !tt.event.TS.Equal(decoded.TS) { + t.Errorf("TS mismatch: got %v, want %v", decoded.TS, tt.event.TS) + } + if decoded.Kind != tt.event.Kind { + t.Errorf("Kind mismatch: got %q, want %q", decoded.Kind, tt.event.Kind) + } + if decoded.Role != tt.event.Role { + t.Errorf("Role mismatch: got %q, want %q", decoded.Role, tt.event.Role) + } + if decoded.Type != tt.event.Type { + t.Errorf("Type mismatch: got %q, want %q", decoded.Type, tt.event.Type) + } + if decoded.Text != tt.event.Text { + t.Errorf("Text mismatch: got %q, want %q", decoded.Text, tt.event.Text) + } + if len(tt.event.Meta) != len(decoded.Meta) { + t.Errorf("Meta length mismatch: got %d, want %d", len(decoded.Meta), len(tt.event.Meta)) + } + for k, v := range tt.event.Meta { + if decoded.Meta[k] != v { + t.Errorf("Meta[%q] mismatch: got %q, want %q", k, decoded.Meta[k], v) + } + } + }) + } +} + +func TestEvent_JSON_OmitEmpty(t *testing.T) { + tests := []struct { + name string + event Event + shouldContain []string + shouldOmit []string + }{ + { + name: "omits empty optional fields", + event: Event{ + TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC), + Kind: "message", + }, + shouldContain: []string{"ts", "kind"}, + shouldOmit: []string{"role", "type", "text", "meta"}, + }, + { + name: "includes non-empty optional fields", + event: Event{ + TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC), + Kind: "message", + Role: "user", + Text: "hello", + }, + shouldContain: []string{"ts", "kind", "role", "text"}, + shouldOmit: []string{"type", "meta"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := json.Marshal(tt.event) + if err != nil { + t.Fatalf("json.Marshal() error = %v", err) + } + jsonStr := string(data) + + for _, field := range tt.shouldContain { + if !containsField(jsonStr, field) { + t.Errorf("JSON should contain field %q, got: %s", field, jsonStr) + } + } + for _, field := range tt.shouldOmit { + if containsField(jsonStr, field) { + t.Errorf("JSON should omit field %q, got: %s", field, jsonStr) + } + } + }) + } +} + +func TestEvent_Fields(t *testing.T) { + tests := []struct { + name string + event Event + wantKind string + wantRole string + wantType string + wantText string + }{ + { + name: "user message event", + event: Event{ + Kind: "message", + Role: "user", + Type: "text", + Text: "What is the weather?", + }, + wantKind: "message", + wantRole: "user", + wantType: "text", + wantText: "What is the weather?", + }, + { + name: "assistant response event", + event: Event{ + Kind: "message", + Role: "assistant", + Type: "text", + Text: "The weather is sunny.", + }, + wantKind: "message", + wantRole: "assistant", + wantType: "text", + wantText: "The weather is sunny.", + }, + { + name: "system event", + event: Event{ + Kind: "system", + Type: "config", + Text: "Session started", + }, + wantKind: "system", + wantRole: "", + wantType: "config", + wantText: "Session started", + }, + { + name: "tool use event", + event: Event{ + Kind: "tool_use", + Role: "assistant", + Type: "bash", + Text: "ls -la", + }, + wantKind: "tool_use", + wantRole: "assistant", + wantType: "bash", + wantText: "ls -la", + }, + { + name: "tool result event", + event: Event{ + Kind: "tool_result", + Role: "user", + Type: "bash", + Text: "file1.txt\nfile2.txt", + }, + wantKind: "tool_result", + wantRole: "user", + wantType: "bash", + wantText: "file1.txt\nfile2.txt", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.event.Kind != tt.wantKind { + t.Errorf("Kind = %q, want %q", tt.event.Kind, tt.wantKind) + } + if tt.event.Role != tt.wantRole { + t.Errorf("Role = %q, want %q", tt.event.Role, tt.wantRole) + } + if tt.event.Type != tt.wantType { + t.Errorf("Type = %q, want %q", tt.event.Type, tt.wantType) + } + if tt.event.Text != tt.wantText { + t.Errorf("Text = %q, want %q", tt.event.Text, tt.wantText) + } + }) + } +} + +func TestEvent_MetaField(t *testing.T) { + tests := []struct { + name string + meta map[string]string + key string + wantVal string + wantOk bool + }{ + { + name: "existing key", + meta: map[string]string{"model": "gpt-4", "tokens": "100"}, + key: "model", + wantVal: "gpt-4", + wantOk: true, + }, + { + name: "missing key", + meta: map[string]string{"model": "gpt-4"}, + key: "tokens", + wantVal: "", + wantOk: false, + }, + { + name: "nil meta", + meta: nil, + key: "any", + wantVal: "", + wantOk: false, + }, + { + name: "empty meta", + meta: map[string]string{}, + key: "any", + wantVal: "", + wantOk: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + event := Event{Meta: tt.meta} + val, ok := event.Meta[tt.key] + if ok != tt.wantOk { + t.Errorf("Meta[%q] ok = %v, want %v", tt.key, ok, tt.wantOk) + } + if val != tt.wantVal { + t.Errorf("Meta[%q] = %q, want %q", tt.key, val, tt.wantVal) + } + }) + } +} + +func TestEvent_Timestamp(t *testing.T) { + tests := []struct { + name string + ts time.Time + isZero bool + }{ + { + name: "valid timestamp", + ts: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC), + isZero: false, + }, + { + name: "zero timestamp", + ts: time.Time{}, + isZero: true, + }, + { + name: "unix epoch", + ts: time.Unix(0, 0).UTC(), + isZero: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + event := Event{TS: tt.ts} + if event.TS.IsZero() != tt.isZero { + t.Errorf("TS.IsZero() = %v, want %v", event.TS.IsZero(), tt.isZero) + } + }) + } +} + +// containsField checks if a JSON string contains a specific field key +func containsField(jsonStr, field string) bool { + return json.Valid([]byte(jsonStr)) && (len(jsonStr) > 0 && (jsonStr[0] == '{' || jsonStr[0] == '[')) && + (len(field) > 0 && (jsonFieldPresent(jsonStr, field))) +} + +func jsonFieldPresent(jsonStr, field string) bool { + var m map[string]interface{} + if err := json.Unmarshal([]byte(jsonStr), &m); err != nil { + return false + } + _, ok := m[field] + return ok +}