test(memory): add unit tests for file store, pruning, and semantic search

Covers event serialization, file operations, token-based pruning,
recency preservation, and semantic similarity scoring.
This commit is contained in:
Finessse
2025-12-28 10:46:21 +07:00
parent 7d11a35a16
commit 009fbb2a37
4 changed files with 2442 additions and 0 deletions
+641
View File
@@ -0,0 +1,641 @@
package memory
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func TestFileStore_Append_CreatesFile(t *testing.T) {
tests := []struct {
name string
session string
events []Event
wantErr bool
}{
{
name: "creates file with single event",
session: "test-session-1",
events: []Event{
{
TS: time.Now(),
Kind: "message",
Role: "user",
Text: "Hello, world!",
},
},
wantErr: false,
},
{
name: "creates file with event that has zero timestamp",
session: "test-session-2",
events: []Event{
{
Kind: "message",
Role: "user",
Text: "This event has no timestamp set",
},
},
wantErr: false,
},
{
name: "empty session returns nil without creating file",
session: "",
events: []Event{
{Kind: "message", Text: "test"},
},
wantErr: false,
},
{
name: "empty events returns nil without creating file",
session: "test-session-3",
events: []Event{},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
err := store.Append(tt.session, tt.events)
if (err != nil) != tt.wantErr {
t.Errorf("Append() error = %v, wantErr %v", err, tt.wantErr)
return
}
if tt.session == "" || len(tt.events) == 0 {
// File should not be created
eventsPath := filepath.Join(tmpDir, "sessions", tt.session, "events.jsonl")
if _, err := os.Stat(eventsPath); !os.IsNotExist(err) {
t.Errorf("File should not exist for empty session or events")
}
return
}
// Verify file was created
eventsPath := filepath.Join(tmpDir, "sessions", tt.session, "events.jsonl")
if _, err := os.Stat(eventsPath); os.IsNotExist(err) {
t.Errorf("events.jsonl was not created")
return
}
// Verify content is valid JSONL
data, err := os.ReadFile(eventsPath)
if err != nil {
t.Fatalf("Failed to read events file: %v", err)
}
lines := strings.Split(strings.TrimSpace(string(data)), "\n")
if len(lines) != len(tt.events) {
t.Errorf("Expected %d lines, got %d", len(tt.events), len(lines))
}
for i, line := range lines {
var e Event
if err := json.Unmarshal([]byte(line), &e); err != nil {
t.Errorf("Line %d is not valid JSON: %v", i, err)
}
}
})
}
}
func TestFileStore_Append_MultipleEvents(t *testing.T) {
tests := []struct {
name string
session string
eventSets [][]Event
wantTotal int
wantErr bool
}{
{
name: "append multiple events in single call",
session: "multi-event-session",
eventSets: [][]Event{
{
{Kind: "message", Role: "user", Text: "First message"},
{Kind: "message", Role: "assistant", Text: "First response"},
{Kind: "message", Role: "user", Text: "Second message"},
},
},
wantTotal: 3,
wantErr: false,
},
{
name: "append events across multiple calls",
session: "multiple-calls-session",
eventSets: [][]Event{
{{Kind: "message", Role: "user", Text: "Message 1"}},
{{Kind: "message", Role: "assistant", Text: "Response 1"}},
{{Kind: "message", Role: "user", Text: "Message 2"}},
{{Kind: "message", Role: "assistant", Text: "Response 2"}},
},
wantTotal: 4,
wantErr: false,
},
{
name: "append mixed events with metadata",
session: "mixed-events-session",
eventSets: [][]Event{
{
{Kind: "system", Text: "Session started"},
{Kind: "message", Role: "user", Text: "Hello", Meta: map[string]string{"lang": "en"}},
{Kind: "tool_use", Role: "assistant", Type: "bash", Text: "ls"},
{Kind: "tool_result", Role: "user", Type: "bash", Text: "file.txt"},
},
},
wantTotal: 4,
wantErr: false,
},
{
name: "append preserves order",
session: "order-session",
eventSets: [][]Event{
{
{Kind: "message", Role: "user", Text: "First"},
{Kind: "message", Role: "user", Text: "Second"},
{Kind: "message", Role: "user", Text: "Third"},
},
},
wantTotal: 3,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
for _, events := range tt.eventSets {
err := store.Append(tt.session, events)
if (err != nil) != tt.wantErr {
t.Errorf("Append() error = %v, wantErr %v", err, tt.wantErr)
return
}
}
// Read and verify the file
eventsPath := filepath.Join(tmpDir, "sessions", tt.session, "events.jsonl")
data, err := os.ReadFile(eventsPath)
if err != nil {
t.Fatalf("Failed to read events file: %v", err)
}
lines := strings.Split(strings.TrimSpace(string(data)), "\n")
if len(lines) != tt.wantTotal {
t.Errorf("Expected %d events, got %d", tt.wantTotal, len(lines))
}
// Verify each line is valid JSON and check order for order test
if tt.name == "append preserves order" {
expectedTexts := []string{"First", "Second", "Third"}
for i, line := range lines {
var e Event
if err := json.Unmarshal([]byte(line), &e); err != nil {
t.Errorf("Line %d is not valid JSON: %v", i, err)
continue
}
if e.Text != expectedTexts[i] {
t.Errorf("Event %d: expected text %q, got %q", i, expectedTexts[i], e.Text)
}
}
}
})
}
}
func TestFileStore_Append_RedactsSecrets(t *testing.T) {
tests := []struct {
name string
inputText string
wantRedact bool
}{
{
name: "redacts sk- API keys",
inputText: "My key is sk-1234567890abcdef",
wantRedact: true,
},
{
name: "redacts bearer tokens",
inputText: "Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9",
wantRedact: true,
},
{
name: "redacts AIza keys",
inputText: "API key: AIzaSyB1234567890abcdefghij",
wantRedact: true,
},
{
name: "does not modify normal text",
inputText: "This is a normal message without secrets",
wantRedact: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
session := "redact-test"
err := store.Append(session, []Event{
{Kind: "message", Text: tt.inputText},
})
if err != nil {
t.Fatalf("Append() error = %v", err)
}
// Read the stored event
eventsPath := filepath.Join(tmpDir, "sessions", session, "events.jsonl")
data, err := os.ReadFile(eventsPath)
if err != nil {
t.Fatalf("Failed to read events file: %v", err)
}
var e Event
if err := json.Unmarshal(data, &e); err != nil {
t.Fatalf("Failed to parse event: %v", err)
}
hasRedacted := strings.Contains(e.Text, "[REDACTED]")
if tt.wantRedact && !hasRedacted {
t.Errorf("Expected text to be redacted, got: %s", e.Text)
}
if !tt.wantRedact && hasRedacted {
t.Errorf("Text should not be redacted, got: %s", e.Text)
}
})
}
}
func TestFileStore_Search_ExactMatch(t *testing.T) {
tests := []struct {
name string
events []Event
query string
wantCount int
wantMatch string
}{
{
name: "finds exact keyword match",
events: []Event{
{Kind: "message", Role: "user", Text: "How do I configure the database?"},
{Kind: "message", Role: "assistant", Text: "You can configure the database in config.yaml"},
{Kind: "message", Role: "user", Text: "Thanks, what about logging?"},
},
query: "database",
wantCount: 2,
wantMatch: "database",
},
{
name: "case insensitive search",
events: []Event{
{Kind: "message", Text: "The DATABASE connection failed"},
{Kind: "message", Text: "Database retry succeeded"},
{Kind: "message", Text: "No matches here"},
},
query: "database",
wantCount: 2,
wantMatch: "",
},
{
name: "no matches returns empty",
events: []Event{
{Kind: "message", Text: "Hello world"},
{Kind: "message", Text: "Goodbye world"},
},
query: "nonexistent",
wantCount: 0,
wantMatch: "",
},
{
name: "multiple keywords - matches any keyword",
events: []Event{
{Kind: "message", Text: "Error in authentication module"},
{Kind: "message", Text: "Authentication failed for user"},
{Kind: "message", Text: "Module loaded successfully"},
},
query: "authentication module",
wantCount: 3, // all three contain at least one keyword
wantMatch: "",
},
{
name: "empty query returns nil",
events: []Event{
{Kind: "message", Text: "Some text"},
},
query: "",
wantCount: 0,
wantMatch: "",
},
{
name: "whitespace query returns nil",
events: []Event{
{Kind: "message", Text: "Some text"},
},
query: " ",
wantCount: 0,
wantMatch: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
session := "search-test"
// Append test events
if err := store.Append(session, tt.events); err != nil {
t.Fatalf("Append() error = %v", err)
}
// Search
results, err := store.Search(session, tt.query, 6000, 10)
if err != nil {
t.Fatalf("Search() error = %v", err)
}
if len(results) != tt.wantCount {
t.Errorf("Search() returned %d results, want %d", len(results), tt.wantCount)
}
if tt.wantMatch != "" {
found := false
for _, r := range results {
if strings.Contains(strings.ToLower(r), tt.wantMatch) {
found = true
break
}
}
if !found {
t.Errorf("Search results should contain %q", tt.wantMatch)
}
}
})
}
}
func TestFileStore_Search_MaxSnippets(t *testing.T) {
tests := []struct {
name string
eventCount int
maxSnippets int
wantMax int
}{
{
name: "limits to maxSnippets",
eventCount: 20,
maxSnippets: 5,
wantMax: 5,
},
{
name: "returns all if under limit",
eventCount: 3,
maxSnippets: 10,
wantMax: 3,
},
{
name: "default maxSnippets when zero",
eventCount: 20,
maxSnippets: 0,
wantMax: 8, // default is 8
},
{
name: "maxSnippets of 1",
eventCount: 10,
maxSnippets: 1,
wantMax: 1,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
session := "maxsnippets-test"
// Create events with searchable text
events := make([]Event, tt.eventCount)
for i := 0; i < tt.eventCount; i++ {
events[i] = Event{
Kind: "message",
Role: "user",
Text: "This is a searchable test message number " + string(rune('A'+i)),
}
}
if err := store.Append(session, events); err != nil {
t.Fatalf("Append() error = %v", err)
}
results, err := store.Search(session, "searchable test message", 100000, tt.maxSnippets)
if err != nil {
t.Fatalf("Search() error = %v", err)
}
if len(results) > tt.wantMax {
t.Errorf("Search() returned %d results, want at most %d", len(results), tt.wantMax)
}
})
}
}
func TestFileStore_Search_MaxChars(t *testing.T) {
tests := []struct {
name string
events []Event
maxChars int
wantUnder bool
}{
{
name: "respects maxChars limit",
events: []Event{
{Kind: "message", Text: strings.Repeat("searchable text ", 100)},
{Kind: "message", Text: strings.Repeat("more searchable text ", 100)},
},
maxChars: 500,
wantUnder: true,
},
{
name: "default maxChars when zero",
events: []Event{
{Kind: "message", Text: "searchable text"},
},
maxChars: 0,
wantUnder: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
session := "maxchars-test"
if err := store.Append(session, tt.events); err != nil {
t.Fatalf("Append() error = %v", err)
}
results, err := store.Search(session, "searchable text", tt.maxChars, 100)
if err != nil {
t.Fatalf("Search() error = %v", err)
}
if tt.wantUnder {
totalChars := 0
for _, r := range results {
totalChars += len(r)
}
limit := tt.maxChars
if limit <= 0 {
limit = 6000 // default
}
if totalChars > limit {
t.Errorf("Total chars %d exceeds maxChars %d", totalChars, limit)
}
}
})
}
}
func TestFileStore_Search_NonExistentSession(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
results, err := store.Search("nonexistent-session", "query", 6000, 10)
if err != nil {
t.Errorf("Search() should not error for nonexistent session, got: %v", err)
}
if len(results) != 0 {
t.Errorf("Search() should return empty for nonexistent session, got %d results", len(results))
}
}
func TestFileStore_NilStore(t *testing.T) {
var store *FileStore
t.Run("Append on nil store", func(t *testing.T) {
err := store.Append("session", []Event{{Kind: "message", Text: "test"}})
if err == nil {
t.Error("Expected error for nil store, got nil")
}
})
t.Run("Search on nil store", func(t *testing.T) {
_, err := store.Search("session", "query", 100, 10)
if err == nil {
t.Error("Expected error for nil store, got nil")
}
})
}
func TestFileStore_EmptyBaseDir(t *testing.T) {
store := NewFileStore("")
t.Run("Append with empty BaseDir", func(t *testing.T) {
err := store.Append("session", []Event{{Kind: "message", Text: "test"}})
if err == nil {
t.Error("Expected error for empty BaseDir, got nil")
}
})
t.Run("Search with empty BaseDir", func(t *testing.T) {
_, err := store.Search("session", "query", 100, 10)
if err == nil {
t.Error("Expected error for empty BaseDir, got nil")
}
})
}
func TestFileStore_SearchCache(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
session := "cache-test"
events := []Event{
{Kind: "message", Role: "user", Text: "This is a cacheable message"},
}
if err := store.Append(session, events); err != nil {
t.Fatalf("Append() error = %v", err)
}
// First search populates cache
results1, err := store.Search(session, "cacheable", 6000, 10)
if err != nil {
t.Fatalf("Search() error = %v", err)
}
// Second search should use cache (same query)
results2, err := store.Search(session, "cacheable", 6000, 10)
if err != nil {
t.Fatalf("Search() error = %v", err)
}
if len(results1) != len(results2) {
t.Errorf("Cached result count mismatch: %d vs %d", len(results1), len(results2))
}
// Append invalidates cache
if err := store.Append(session, []Event{{Kind: "message", Text: "New cacheable event"}}); err != nil {
t.Fatalf("Append() error = %v", err)
}
// Search after append should get fresh results
results3, err := store.Search(session, "cacheable", 6000, 10)
if err != nil {
t.Fatalf("Search() error = %v", err)
}
if len(results3) <= len(results1) {
t.Logf("Results after append: %d (was %d)", len(results3), len(results1))
}
}
func TestFileStore_SessionDir(t *testing.T) {
tests := []struct {
name string
session string
wantDir bool
}{
{
name: "normal session name",
session: "my-session-123",
wantDir: true,
},
{
name: "session with special characters",
session: "session/with\\special:chars*",
wantDir: true,
},
{
name: "empty session",
session: "",
wantDir: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
dir := store.SessionDir(tt.session)
if tt.wantDir && dir == "" {
t.Error("SessionDir() returned empty string")
}
if tt.wantDir && !strings.HasPrefix(dir, tmpDir) {
t.Errorf("SessionDir() = %q, should be under %q", dir, tmpDir)
}
})
}
}
+610
View File
@@ -0,0 +1,610 @@
package memory
import (
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func TestPrune_ByTokenCount(t *testing.T) {
tests := []struct {
name string
sessionCount int
eventsPerSession int
maxBytesPerSession int64
wantSessionsTrimmed int
}{
{
name: "trims large sessions",
sessionCount: 2,
eventsPerSession: 100,
maxBytesPerSession: 500,
wantSessionsTrimmed: 2,
},
{
name: "no trimming needed for small sessions",
sessionCount: 2,
eventsPerSession: 2,
maxBytesPerSession: 100000,
wantSessionsTrimmed: 0,
},
{
name: "zero maxBytes skips trimming",
sessionCount: 2,
eventsPerSession: 100,
maxBytesPerSession: 0,
wantSessionsTrimmed: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
// Create sessions with events
for i := 0; i < tt.sessionCount; i++ {
session := "session-" + string(rune('A'+i))
events := make([]Event, tt.eventsPerSession)
for j := 0; j < tt.eventsPerSession; j++ {
events[j] = Event{
TS: time.Now(),
Kind: "message",
Role: "user",
Text: strings.Repeat("This is a test message with some content. ", 10),
}
}
if err := store.Append(session, events); err != nil {
t.Fatalf("Append() error = %v", err)
}
}
// Prune sessions
result, err := store.PruneSessions(0, 0, tt.maxBytesPerSession)
if err != nil {
t.Fatalf("PruneSessions() error = %v", err)
}
if result.SessionsTrimmed != tt.wantSessionsTrimmed {
t.Errorf("SessionsTrimmed = %d, want %d", result.SessionsTrimmed, tt.wantSessionsTrimmed)
}
if result.SessionsTrimmed > 0 && result.BytesFreed <= 0 {
t.Error("Expected BytesFreed > 0 when sessions are trimmed")
}
})
}
}
func TestPrune_PreservesRecent(t *testing.T) {
tests := []struct {
name string
sessionAges []int // days ago
maxAgeDays int
maxSessions int
wantRemoved int
wantKept []string
}{
{
name: "preserves recent sessions by age",
sessionAges: []int{1, 5, 10, 20, 40}, // days ago
maxAgeDays: 30,
maxSessions: 0,
wantRemoved: 1, // only 40 days old is removed
wantKept: []string{"session-0", "session-1", "session-2", "session-3"},
},
{
name: "preserves by max sessions count",
sessionAges: []int{1, 2, 3, 4, 5},
maxAgeDays: 0,
maxSessions: 3,
wantRemoved: 2, // keeps 3 most recent
wantKept: []string{"session-0", "session-1", "session-2"},
},
{
name: "combines age and count limits",
sessionAges: []int{1, 5, 35, 40},
maxAgeDays: 30,
maxSessions: 2,
wantRemoved: 2, // 35 and 40 days old are removed
wantKept: []string{"session-0", "session-1"},
},
{
name: "no pruning when all within limits",
sessionAges: []int{1, 2, 3},
maxAgeDays: 30,
maxSessions: 10,
wantRemoved: 0,
wantKept: []string{"session-0", "session-1", "session-2"},
},
{
name: "removes all old sessions",
sessionAges: []int{100, 200, 300},
maxAgeDays: 30,
maxSessions: 0,
wantRemoved: 3,
wantKept: []string{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
// Create sessions with different ages
for i, daysAgo := range tt.sessionAges {
session := "session-" + string(rune('0'+i))
sessionDir := filepath.Join(tmpDir, "sessions", session)
if err := os.MkdirAll(sessionDir, 0o755); err != nil {
t.Fatalf("Failed to create session dir: %v", err)
}
// Create events file
eventsPath := filepath.Join(sessionDir, "events.jsonl")
if err := os.WriteFile(eventsPath, []byte(`{"kind":"message","text":"test"}`+"\n"), 0o644); err != nil {
t.Fatalf("Failed to create events file: %v", err)
}
// Set modification time
modTime := time.Now().Add(-time.Duration(daysAgo) * 24 * time.Hour)
if err := os.Chtimes(eventsPath, modTime, modTime); err != nil {
t.Fatalf("Failed to set file times: %v", err)
}
}
// Prune sessions
result, err := store.PruneSessions(tt.maxAgeDays, tt.maxSessions, 0)
if err != nil {
t.Fatalf("PruneSessions() error = %v", err)
}
if result.SessionsRemoved != tt.wantRemoved {
t.Errorf("SessionsRemoved = %d, want %d", result.SessionsRemoved, tt.wantRemoved)
}
// Verify kept sessions exist
for _, session := range tt.wantKept {
sessionDir := filepath.Join(tmpDir, "sessions", session)
if _, err := os.Stat(sessionDir); os.IsNotExist(err) {
t.Errorf("Session %q should have been preserved but was removed", session)
}
}
// Verify removed sessions don't exist
sessions, _ := store.ListSessions(0)
if len(sessions) != len(tt.wantKept) {
t.Errorf("Expected %d sessions remaining, got %d", len(tt.wantKept), len(sessions))
}
})
}
}
func TestPrune_SemanticNamespaces(t *testing.T) {
tests := []struct {
name string
namespaceCount int
recordsPerNs int
maxAgeDays int
maxNamespaces int
maxBytesPerNs int64
wantRemoved int
wantTrimmed int
}{
{
name: "removes old namespaces",
namespaceCount: 3,
recordsPerNs: 5,
maxAgeDays: 30,
maxNamespaces: 0,
maxBytesPerNs: 0,
wantRemoved: 0, // all are recent
wantTrimmed: 0,
},
{
name: "limits namespace count",
namespaceCount: 5,
recordsPerNs: 5,
maxAgeDays: 0,
maxNamespaces: 3,
maxBytesPerNs: 0,
wantRemoved: 2,
wantTrimmed: 0,
},
{
name: "trims large namespaces",
namespaceCount: 2,
recordsPerNs: 500,
maxAgeDays: 0,
maxNamespaces: 0,
maxBytesPerNs: 2000,
wantRemoved: 0,
wantTrimmed: 2,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
// Create namespaces with records
for i := 0; i < tt.namespaceCount; i++ {
namespace := "namespace-" + string(rune('A'+i))
records := make([]SemanticRecord, tt.recordsPerNs)
for j := 0; j < tt.recordsPerNs; j++ {
// Make each text unique to avoid deduplication
records[j] = SemanticRecord{
TS: time.Now(),
Text: strings.Repeat("Test record content ", 20) + " " + string(rune('A'+i)) + string(rune('0'+j%10)),
Vec: []float32{float32(i), float32(j), 0.5},
}
}
if err := store.AppendSemantic(namespace, records); err != nil {
t.Fatalf("AppendSemantic() error = %v", err)
}
// Add small delay to ensure different modification times
time.Sleep(10 * time.Millisecond)
}
// Prune semantic namespaces
result, err := store.PruneSemantic(tt.maxAgeDays, tt.maxNamespaces, tt.maxBytesPerNs)
if err != nil {
t.Fatalf("PruneSemantic() error = %v", err)
}
if result.SemanticNamespacesRemoved != tt.wantRemoved {
t.Errorf("SemanticNamespacesRemoved = %d, want %d", result.SemanticNamespacesRemoved, tt.wantRemoved)
}
if result.SemanticNamespacesTrimmed != tt.wantTrimmed {
t.Errorf("SemanticNamespacesTrimmed = %d, want %d", result.SemanticNamespacesTrimmed, tt.wantTrimmed)
}
})
}
}
func TestTrimJSONLFile(t *testing.T) {
tests := []struct {
name string
content string
maxBytes int64
wantTrim bool
wantFreed bool
}{
{
name: "trims file exceeding maxBytes",
content: `{"a":1}` + "\n" + `{"b":2}` + "\n" + `{"c":3}` + "\n" + `{"d":4}` + "\n",
maxBytes: 20,
wantTrim: true,
wantFreed: true,
},
{
name: "no trim needed for small file",
content: `{"a":1}` + "\n",
maxBytes: 1000,
wantTrim: false,
wantFreed: false,
},
{
name: "zero maxBytes skips trimming",
content: `{"a":1}` + "\n" + `{"b":2}` + "\n",
maxBytes: 0,
wantTrim: false,
wantFreed: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpDir := t.TempDir()
path := filepath.Join(tmpDir, "test.jsonl")
if err := os.WriteFile(path, []byte(tt.content), 0o644); err != nil {
t.Fatalf("Failed to write test file: %v", err)
}
originalSize := int64(len(tt.content))
trimmed, freed := trimJSONLFile(path, tt.maxBytes)
if trimmed != tt.wantTrim {
t.Errorf("trimJSONLFile() trimmed = %v, want %v", trimmed, tt.wantTrim)
}
if tt.wantFreed && freed <= 0 {
t.Errorf("Expected freed > 0, got %d", freed)
}
if !tt.wantFreed && freed != 0 {
t.Errorf("Expected freed = 0, got %d", freed)
}
// Verify file was actually trimmed
if tt.wantTrim {
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read trimmed file: %v", err)
}
if int64(len(data)) >= originalSize {
t.Errorf("File was not actually trimmed: %d >= %d", len(data), originalSize)
}
}
})
}
}
func TestPruneResult_Fields(t *testing.T) {
result := PruneResult{
SessionsRemoved: 5,
SessionsTrimmed: 3,
SemanticNamespacesRemoved: 2,
SemanticNamespacesTrimmed: 1,
BytesFreed: 1024 * 1024,
}
if result.SessionsRemoved != 5 {
t.Errorf("SessionsRemoved = %d, want 5", result.SessionsRemoved)
}
if result.SessionsTrimmed != 3 {
t.Errorf("SessionsTrimmed = %d, want 3", result.SessionsTrimmed)
}
if result.SemanticNamespacesRemoved != 2 {
t.Errorf("SemanticNamespacesRemoved = %d, want 2", result.SemanticNamespacesRemoved)
}
if result.SemanticNamespacesTrimmed != 1 {
t.Errorf("SemanticNamespacesTrimmed = %d, want 1", result.SemanticNamespacesTrimmed)
}
if result.BytesFreed != 1024*1024 {
t.Errorf("BytesFreed = %d, want %d", result.BytesFreed, 1024*1024)
}
}
func TestSessionInfo_Fields(t *testing.T) {
now := time.Now()
info := SessionInfo{
Key: "test-session",
Path: "/path/to/session",
UpdatedAt: now,
SizeBytes: 2048,
EventsBytes: 1024,
HasSummary: true,
HasTodo: true,
HasPinned: false,
HasAnchorPending: true,
SemanticDisabled: false,
}
if info.Key != "test-session" {
t.Errorf("Key = %q, want %q", info.Key, "test-session")
}
if info.Path != "/path/to/session" {
t.Errorf("Path = %q, want %q", info.Path, "/path/to/session")
}
if !info.UpdatedAt.Equal(now) {
t.Errorf("UpdatedAt mismatch")
}
if info.SizeBytes != 2048 {
t.Errorf("SizeBytes = %d, want 2048", info.SizeBytes)
}
if info.EventsBytes != 1024 {
t.Errorf("EventsBytes = %d, want 1024", info.EventsBytes)
}
if !info.HasSummary {
t.Error("HasSummary should be true")
}
if !info.HasTodo {
t.Error("HasTodo should be true")
}
if info.HasPinned {
t.Error("HasPinned should be false")
}
if !info.HasAnchorPending {
t.Error("HasAnchorPending should be true")
}
if info.SemanticDisabled {
t.Error("SemanticDisabled should be false")
}
}
func TestListSessions(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
// Create some sessions
sessions := []string{"alpha", "beta", "gamma"}
for _, session := range sessions {
events := []Event{{Kind: "message", Text: "test"}}
if err := store.Append(session, events); err != nil {
t.Fatalf("Append() error = %v", err)
}
// Small delay to ensure different mod times
time.Sleep(10 * time.Millisecond)
}
// List all sessions
list, err := store.ListSessions(0)
if err != nil {
t.Fatalf("ListSessions() error = %v", err)
}
if len(list) != len(sessions) {
t.Errorf("ListSessions() returned %d, want %d", len(list), len(sessions))
}
// List with limit
limited, err := store.ListSessions(2)
if err != nil {
t.Fatalf("ListSessions() error = %v", err)
}
if len(limited) != 2 {
t.Errorf("ListSessions(2) returned %d, want 2", len(limited))
}
// Verify most recent is first (gamma was created last)
if len(limited) > 0 && limited[0].Key != "gamma" {
t.Errorf("Most recent session should be 'gamma', got %q", limited[0].Key)
}
}
func TestGetSessionInfo(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
session := "info-test"
// Create session with various files
events := []Event{{Kind: "message", Text: "test event"}}
if err := store.Append(session, events); err != nil {
t.Fatalf("Append() error = %v", err)
}
if err := store.WriteSummary(session, "Test summary", 0); err != nil {
t.Fatalf("WriteSummary() error = %v", err)
}
if err := store.WriteTodo(session, "Test todo", 0); err != nil {
t.Fatalf("WriteTodo() error = %v", err)
}
// Get session info
info, err := store.GetSessionInfo(session)
if err != nil {
t.Fatalf("GetSessionInfo() error = %v", err)
}
if info.Key != session {
t.Errorf("Key = %q, want %q", info.Key, session)
}
if !info.HasSummary {
t.Error("HasSummary should be true")
}
if !info.HasTodo {
t.Error("HasTodo should be true")
}
if info.SizeBytes <= 0 {
t.Error("SizeBytes should be > 0")
}
if info.EventsBytes <= 0 {
t.Error("EventsBytes should be > 0")
}
}
func TestReadEventTail(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
session := "tail-test"
// Create many events
events := make([]Event, 20)
for i := 0; i < 20; i++ {
events[i] = Event{
Kind: "message",
Role: "user",
Text: "Message " + string(rune('A'+i)),
}
}
if err := store.Append(session, events); err != nil {
t.Fatalf("Append() error = %v", err)
}
// Read tail with limit
tail, err := store.ReadEventTail(session, 5)
if err != nil {
t.Fatalf("ReadEventTail() error = %v", err)
}
if len(tail) != 5 {
t.Errorf("ReadEventTail() returned %d events, want 5", len(tail))
}
// Verify chronological order (oldest first in result)
if len(tail) > 0 {
// Should be last 5 events: P, Q, R, S, T
expectedFirst := "Message P"
if tail[0].Text != expectedFirst {
t.Errorf("First event text = %q, want %q", tail[0].Text, expectedFirst)
}
}
}
func TestExportSessionZip(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
session := "export-test"
// Create session with content
events := []Event{{Kind: "message", Text: "test event"}}
if err := store.Append(session, events); err != nil {
t.Fatalf("Append() error = %v", err)
}
if err := store.WriteSummary(session, "Test summary", 0); err != nil {
t.Fatalf("WriteSummary() error = %v", err)
}
// Export as zip
sessionDir := store.SessionDir(session)
zipData, err := ExportSessionZip(sessionDir)
if err != nil {
t.Fatalf("ExportSessionZip() error = %v", err)
}
if len(zipData) == 0 {
t.Error("Expected non-empty zip data")
}
// Verify it's a valid zip (starts with PK)
if len(zipData) < 2 || zipData[0] != 'P' || zipData[1] != 'K' {
t.Error("Expected valid zip file header")
}
}
func TestExportSessionZip_EmptyDir(t *testing.T) {
_, err := ExportSessionZip("")
if err == nil {
t.Error("Expected error for empty directory")
}
}
func TestExportAllZip(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
// Create multiple sessions
for _, session := range []string{"session-1", "session-2"} {
events := []Event{{Kind: "message", Text: "test in " + session}}
if err := store.Append(session, events); err != nil {
t.Fatalf("Append() error = %v", err)
}
}
// Export all
zipData, err := ExportAllZip(tmpDir, 10*1024*1024)
if err != nil {
t.Fatalf("ExportAllZip() error = %v", err)
}
if len(zipData) == 0 {
t.Error("Expected non-empty zip data")
}
}
func TestExportAllZip_SizeLimit(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
// Create session with large content
events := []Event{{Kind: "message", Text: strings.Repeat("x", 10000)}}
if err := store.Append("large-session", events); err != nil {
t.Fatalf("Append() error = %v", err)
}
// Try to export with very small limit
_, err := ExportAllZip(tmpDir, 100)
if err == nil {
t.Error("Expected error when size exceeds limit")
}
}
+826
View File
@@ -0,0 +1,826 @@
package memory
import (
"encoding/json"
"math"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func TestSemanticStore_Append(t *testing.T) {
tests := []struct {
name string
namespace string
records []SemanticRecord
wantErr bool
wantCount int
}{
{
name: "appends single record",
namespace: "test-namespace",
records: []SemanticRecord{
{
TS: time.Now(),
Role: "user",
Text: "Hello, world!",
Vec: []float32{0.1, 0.2, 0.3},
},
},
wantErr: false,
wantCount: 1,
},
{
name: "appends multiple records",
namespace: "multi-record-ns",
records: []SemanticRecord{
{Text: "First message", Vec: []float32{0.1, 0.2, 0.3}},
{Text: "Second message", Vec: []float32{0.4, 0.5, 0.6}},
{Text: "Third message", Vec: []float32{0.7, 0.8, 0.9}},
},
wantErr: false,
wantCount: 3,
},
{
name: "skips records with empty text",
namespace: "skip-empty-ns",
records: []SemanticRecord{
{Text: "Valid text", Vec: []float32{0.1, 0.2, 0.3}},
{Text: "", Vec: []float32{0.4, 0.5, 0.6}},
{Text: " ", Vec: []float32{0.7, 0.8, 0.9}},
},
wantErr: false,
wantCount: 1,
},
{
name: "skips records with empty vector",
namespace: "skip-empty-vec-ns",
records: []SemanticRecord{
{Text: "Valid", Vec: []float32{0.1, 0.2, 0.3}},
{Text: "No vector", Vec: []float32{}},
{Text: "Nil vector", Vec: nil},
},
wantErr: false,
wantCount: 1,
},
{
name: "skips duplicate texts",
namespace: "skip-dupe-ns",
records: []SemanticRecord{
{Text: "Same text", Vec: []float32{0.1, 0.2, 0.3}},
{Text: "Same text", Vec: []float32{0.4, 0.5, 0.6}},
{Text: "Different text", Vec: []float32{0.7, 0.8, 0.9}},
},
wantErr: false,
wantCount: 2,
},
{
name: "empty namespace returns nil",
namespace: "",
records: []SemanticRecord{
{Text: "Test", Vec: []float32{0.1, 0.2, 0.3}},
},
wantErr: false,
wantCount: 0,
},
{
name: "empty records returns nil",
namespace: "empty-records-ns",
records: []SemanticRecord{},
wantErr: false,
wantCount: 0,
},
{
name: "computes norm if not provided",
namespace: "compute-norm-ns",
records: []SemanticRecord{
{Text: "Norm will be computed", Vec: []float32{3, 4}, Norm: 0},
},
wantErr: false,
wantCount: 1,
},
{
name: "uses provided norm",
namespace: "provided-norm-ns",
records: []SemanticRecord{
{Text: "Norm is provided", Vec: []float32{3, 4}, Norm: 5.0},
},
wantErr: false,
wantCount: 1,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
err := store.AppendSemantic(tt.namespace, tt.records)
if (err != nil) != tt.wantErr {
t.Errorf("AppendSemantic() error = %v, wantErr %v", err, tt.wantErr)
return
}
if tt.namespace == "" || len(tt.records) == 0 {
return
}
// Verify file content
dir := filepath.Join(tmpDir, "semantic")
entries, err := os.ReadDir(dir)
if err != nil {
if tt.wantCount == 0 {
return // No file expected
}
t.Fatalf("Failed to read semantic dir: %v", err)
}
if len(entries) == 0 && tt.wantCount > 0 {
t.Errorf("Expected semantic namespace directory to be created")
return
}
// Find the items.jsonl file
var itemsPath string
for _, e := range entries {
if e.IsDir() {
itemsPath = filepath.Join(dir, e.Name(), "items.jsonl")
break
}
}
if itemsPath == "" {
if tt.wantCount > 0 {
t.Error("items.jsonl not found")
}
return
}
data, err := os.ReadFile(itemsPath)
if err != nil {
t.Fatalf("Failed to read items.jsonl: %v", err)
}
lines := strings.Split(strings.TrimSpace(string(data)), "\n")
actualCount := 0
for _, line := range lines {
if strings.TrimSpace(line) != "" {
actualCount++
}
}
if actualCount != tt.wantCount {
t.Errorf("Expected %d records, got %d", tt.wantCount, actualCount)
}
})
}
}
func TestSemanticStore_Search(t *testing.T) {
tests := []struct {
name string
records []SemanticRecord
queryVec []float32
maxChars int
maxSnippets int
wantCount int
wantFirst string
}{
{
name: "finds similar vectors",
records: []SemanticRecord{
{Text: "Apple is a fruit", Vec: []float32{0.9, 0.1, 0.0}},
{Text: "Car is a vehicle", Vec: []float32{0.1, 0.9, 0.0}},
{Text: "Orange is a fruit", Vec: []float32{0.85, 0.15, 0.0}},
},
queryVec: []float32{0.9, 0.1, 0.0},
maxChars: 6000,
maxSnippets: 10,
wantCount: 3,
wantFirst: "Apple is a fruit",
},
{
name: "returns empty for no matches",
records: []SemanticRecord{
{Text: "Text 1", Vec: []float32{1, 0, 0}},
{Text: "Text 2", Vec: []float32{0, 1, 0}},
},
queryVec: []float32{0, 0, 1},
maxChars: 6000,
maxSnippets: 10,
wantCount: 0,
wantFirst: "",
},
{
name: "respects maxSnippets",
records: []SemanticRecord{
{Text: "Text 1", Vec: []float32{0.9, 0.1, 0.0}},
{Text: "Text 2", Vec: []float32{0.85, 0.15, 0.0}},
{Text: "Text 3", Vec: []float32{0.8, 0.2, 0.0}},
{Text: "Text 4", Vec: []float32{0.75, 0.25, 0.0}},
},
queryVec: []float32{0.9, 0.1, 0.0},
maxChars: 6000,
maxSnippets: 2,
wantCount: 2,
wantFirst: "Text 1",
},
{
name: "empty query vector returns nil",
records: []SemanticRecord{
{Text: "Some text", Vec: []float32{0.1, 0.2, 0.3}},
},
queryVec: []float32{},
maxChars: 6000,
maxSnippets: 10,
wantCount: 0,
wantFirst: "",
},
{
name: "orders by similarity score",
records: []SemanticRecord{
{Text: "Low similarity", Vec: []float32{0.1, 0.9, 0.0}},
{Text: "High similarity", Vec: []float32{0.95, 0.05, 0.0}},
{Text: "Medium similarity", Vec: []float32{0.5, 0.5, 0.0}},
},
queryVec: []float32{1.0, 0.0, 0.0},
maxChars: 6000,
maxSnippets: 10,
wantCount: 3,
wantFirst: "High similarity",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
namespace := "search-test-ns"
// Append records
if len(tt.records) > 0 {
if err := store.AppendSemantic(namespace, tt.records); err != nil {
t.Fatalf("AppendSemantic() error = %v", err)
}
}
// Search
results, err := store.SearchSemantic(namespace, tt.queryVec, tt.maxChars, tt.maxSnippets)
if err != nil {
t.Fatalf("SearchSemantic() error = %v", err)
}
if len(results) != tt.wantCount {
t.Errorf("SearchSemantic() returned %d results, want %d", len(results), tt.wantCount)
}
if tt.wantFirst != "" && len(results) > 0 {
if results[0] != tt.wantFirst {
t.Errorf("First result = %q, want %q", results[0], tt.wantFirst)
}
}
})
}
}
func TestSemanticStore_SearchWithText(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
namespace := "text-search-ns"
records := []SemanticRecord{
{Text: "Python programming tutorial", Vec: []float32{0.9, 0.1, 0.0}},
{Text: "Java development guide", Vec: []float32{0.85, 0.15, 0.0}},
{Text: "Python data science course", Vec: []float32{0.88, 0.12, 0.0}},
}
if err := store.AppendSemantic(namespace, records); err != nil {
t.Fatalf("AppendSemantic() error = %v", err)
}
// Search with both vector and text (keyword boost)
results, err := store.SearchSemanticWithText(namespace, []float32{0.9, 0.1, 0.0}, "python", 6000, 10)
if err != nil {
t.Fatalf("SearchSemanticWithText() error = %v", err)
}
if len(results) == 0 {
t.Error("Expected results from SearchSemanticWithText")
}
// Python results should be boosted
pythonCount := 0
for _, r := range results {
if strings.Contains(strings.ToLower(r), "python") {
pythonCount++
}
}
if pythonCount == 0 {
t.Error("Expected Python-related results to be present")
}
}
func TestEmbedding_Normalize(t *testing.T) {
tests := []struct {
name string
vec []float32
wantNorm float32
wantZero bool
}{
{
name: "unit vector",
vec: []float32{1, 0, 0},
wantNorm: 1.0,
wantZero: false,
},
{
name: "3-4-5 right triangle",
vec: []float32{3, 4},
wantNorm: 5.0,
wantZero: false,
},
{
name: "all zeros",
vec: []float32{0, 0, 0},
wantNorm: 0,
wantZero: true,
},
{
name: "empty vector",
vec: []float32{},
wantNorm: 0,
wantZero: true,
},
{
name: "negative values",
vec: []float32{-3, 4},
wantNorm: 5.0,
wantZero: false,
},
{
name: "high dimensional",
vec: []float32{1, 1, 1, 1},
wantNorm: 2.0, // sqrt(4) = 2
wantZero: false,
},
{
name: "small values",
vec: []float32{0.001, 0.001, 0.001},
wantNorm: float32(math.Sqrt(0.000003)),
wantZero: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
norm := vectorNorm(tt.vec)
if tt.wantZero && norm != 0 {
t.Errorf("vectorNorm() = %v, want 0", norm)
return
}
if !tt.wantZero {
tolerance := float32(0.0001)
if diff := math.Abs(float64(norm - tt.wantNorm)); diff > float64(tolerance) {
t.Errorf("vectorNorm() = %v, want %v (diff: %v)", norm, tt.wantNorm, diff)
}
}
})
}
}
func TestEmbedding_CosineSimilarity(t *testing.T) {
tests := []struct {
name string
vecA []float32
normA float32
vecB []float32
normB float32
wantSim float32
tolerance float32
}{
{
name: "identical vectors",
vecA: []float32{1, 0, 0},
normA: 1.0,
vecB: []float32{1, 0, 0},
normB: 1.0,
wantSim: 1.0,
tolerance: 0.0001,
},
{
name: "orthogonal vectors",
vecA: []float32{1, 0, 0},
normA: 1.0,
vecB: []float32{0, 1, 0},
normB: 1.0,
wantSim: 0.0,
tolerance: 0.0001,
},
{
name: "opposite vectors",
vecA: []float32{1, 0, 0},
normA: 1.0,
vecB: []float32{-1, 0, 0},
normB: 1.0,
wantSim: -1.0,
tolerance: 0.0001,
},
{
name: "45 degree angle",
vecA: []float32{1, 0},
normA: 1.0,
vecB: []float32{1, 1},
normB: float32(math.Sqrt(2)),
wantSim: float32(1.0 / math.Sqrt(2)), // cos(45) = 1/sqrt(2)
tolerance: 0.0001,
},
{
name: "non-unit vectors",
vecA: []float32{3, 4},
normA: 5.0,
vecB: []float32{6, 8},
normB: 10.0,
wantSim: 1.0, // Same direction
tolerance: 0.0001,
},
{
name: "zero norm A returns 0",
vecA: []float32{0, 0, 0},
normA: 0,
vecB: []float32{1, 0, 0},
normB: 1.0,
wantSim: 0,
tolerance: 0,
},
{
name: "zero norm B returns 0",
vecA: []float32{1, 0, 0},
normA: 1.0,
vecB: []float32{0, 0, 0},
normB: 0,
wantSim: 0,
tolerance: 0,
},
{
name: "different length vectors",
vecA: []float32{1, 2, 3, 4, 5},
normA: vectorNorm([]float32{1, 2, 3, 4, 5}),
vecB: []float32{1, 2, 3},
normB: vectorNorm([]float32{1, 2, 3}),
wantSim: float32(14.0) / (vectorNorm([]float32{1, 2, 3, 4, 5}) * vectorNorm([]float32{1, 2, 3})),
tolerance: 0.0001,
},
{
name: "high dimensional similar",
vecA: []float32{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8},
normA: vectorNorm([]float32{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8}),
vecB: []float32{0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85},
normB: vectorNorm([]float32{0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85}),
wantSim: 0.999, // Very similar
tolerance: 0.01,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sim := cosineSim(tt.vecA, tt.normA, tt.vecB, tt.normB)
diff := math.Abs(float64(sim - tt.wantSim))
if diff > float64(tt.tolerance) {
t.Errorf("cosineSim() = %v, want %v (diff: %v, tolerance: %v)", sim, tt.wantSim, diff, tt.tolerance)
}
})
}
}
func TestSemanticRecord_Serialization(t *testing.T) {
tests := []struct {
name string
record SemanticRecord
}{
{
name: "full record",
record: SemanticRecord{
TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC),
Role: "user",
Text: "Test message",
Vec: []float32{0.1, 0.2, 0.3},
Norm: 0.374,
Source: "chat",
Session: "session-123",
Repo: "org/repo",
},
},
{
name: "minimal record",
record: SemanticRecord{
Text: "Minimal",
Vec: []float32{1.0},
Norm: 1.0,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
data, err := json.Marshal(tt.record)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
var decoded SemanticRecord
if err := json.Unmarshal(data, &decoded); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
if decoded.Text != tt.record.Text {
t.Errorf("Text mismatch: got %q, want %q", decoded.Text, tt.record.Text)
}
if len(decoded.Vec) != len(tt.record.Vec) {
t.Errorf("Vec length mismatch: got %d, want %d", len(decoded.Vec), len(tt.record.Vec))
}
if decoded.Norm != tt.record.Norm {
t.Errorf("Norm mismatch: got %v, want %v", decoded.Norm, tt.record.Norm)
}
})
}
}
func TestSemanticStore_ReadSemanticTail(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
namespace := "tail-test-ns"
// Append multiple records
records := make([]SemanticRecord, 10)
for i := 0; i < 10; i++ {
records[i] = SemanticRecord{
TS: time.Now().Add(time.Duration(i) * time.Minute),
Text: "Message " + string(rune('A'+i)),
Vec: []float32{float32(i), float32(i + 1), float32(i + 2)},
}
}
if err := store.AppendSemantic(namespace, records); err != nil {
t.Fatalf("AppendSemantic() error = %v", err)
}
// Read tail with limit
tail, err := store.ReadSemanticTail(namespace, 5)
if err != nil {
t.Fatalf("ReadSemanticTail() error = %v", err)
}
if len(tail) != 5 {
t.Errorf("ReadSemanticTail() returned %d records, want 5", len(tail))
}
// Verify order (should be chronological, last 5 records)
if len(tail) > 0 {
// First in tail should be 6th record (index 5)
if !strings.Contains(tail[0].Text, "Message F") {
t.Errorf("First record = %q, expected Message F", tail[0].Text)
}
}
}
func TestSemanticStore_ListNamespaces(t *testing.T) {
tmpDir := t.TempDir()
store := NewFileStore(tmpDir)
namespaces := []string{"ns-alpha", "ns-beta", "ns-gamma"}
for _, ns := range namespaces {
records := []SemanticRecord{
{Text: "Record in " + ns, Vec: []float32{0.1, 0.2, 0.3}},
}
if err := store.AppendSemantic(ns, records); err != nil {
t.Fatalf("AppendSemantic() error = %v", err)
}
}
// List all namespaces
list, err := store.ListSemanticNamespaces(0)
if err != nil {
t.Fatalf("ListSemanticNamespaces() error = %v", err)
}
if len(list) != len(namespaces) {
t.Errorf("ListSemanticNamespaces() returned %d, want %d", len(list), len(namespaces))
}
// List with limit
limited, err := store.ListSemanticNamespaces(2)
if err != nil {
t.Fatalf("ListSemanticNamespaces() error = %v", err)
}
if len(limited) != 2 {
t.Errorf("ListSemanticNamespaces(2) returned %d, want 2", len(limited))
}
}
func TestTokenizeSemanticQuery(t *testing.T) {
tests := []struct {
name string
query string
max int
want []string
wantN int
}{
{
name: "basic tokenization",
query: "Hello World",
max: 10,
want: []string{"hello", "world"},
wantN: 2,
},
{
name: "removes stop words",
query: "the quick brown fox and the lazy dog",
max: 10,
want: []string{"quick", "brown", "fox", "lazy", "dog"},
wantN: 5,
},
{
name: "respects max limit",
query: "one two three four five six seven",
max: 3,
want: []string{"one", "two", "three"},
wantN: 3,
},
{
name: "removes short words",
query: "I am a test is it ok",
max: 10,
want: []string{"test"},
wantN: 1,
},
{
name: "handles special characters",
query: "hello-world test_case foo.bar",
max: 10,
want: []string{"hello-world", "test_case", "foo", "bar"},
wantN: 4,
},
{
name: "empty query",
query: "",
max: 10,
want: nil,
wantN: 0,
},
{
name: "only stop words",
query: "the and for with",
max: 10,
want: nil,
wantN: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tokens := tokenizeSemanticQuery(tt.query, tt.max)
if len(tokens) != tt.wantN {
t.Errorf("tokenizeSemanticQuery() returned %d tokens, want %d", len(tokens), tt.wantN)
}
if tt.want != nil {
for i, want := range tt.want {
if i >= len(tokens) {
t.Errorf("Missing token at index %d: want %q", i, want)
continue
}
if tokens[i] != want {
t.Errorf("Token %d = %q, want %q", i, tokens[i], want)
}
}
}
})
}
}
func TestSemanticTokenOverlap(t *testing.T) {
tests := []struct {
name string
tokens []string
text string
want int
}{
{
name: "all tokens match",
tokens: []string{"hello", "world"},
text: "Hello World Test",
want: 2,
},
{
name: "some tokens match",
tokens: []string{"hello", "world", "foo"},
text: "Hello bar baz",
want: 1,
},
{
name: "no tokens match",
tokens: []string{"foo", "bar"},
text: "Hello World",
want: 0,
},
{
name: "empty tokens",
tokens: []string{},
text: "Hello World",
want: 0,
},
{
name: "empty text",
tokens: []string{"hello"},
text: "",
want: 0,
},
{
name: "case insensitive",
tokens: []string{"hello"},
text: "HELLO World",
want: 1,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
overlap := semanticTokenOverlap(tt.tokens, tt.text)
if overlap != tt.want {
t.Errorf("semanticTokenOverlap() = %d, want %d", overlap, tt.want)
}
})
}
}
func TestSemanticRecencyScore(t *testing.T) {
now := time.Now()
tests := []struct {
name string
ts time.Time
windowDays int
wantMin float32
wantMax float32
}{
{
name: "current time",
ts: now,
windowDays: 30,
wantMin: 0.99,
wantMax: 1.0,
},
{
name: "half window age",
ts: now.Add(-15 * 24 * time.Hour),
windowDays: 30,
wantMin: 0.4,
wantMax: 0.6,
},
{
name: "at window edge",
ts: now.Add(-30 * 24 * time.Hour),
windowDays: 30,
wantMin: 0.0,
wantMax: 0.01,
},
{
name: "beyond window",
ts: now.Add(-60 * 24 * time.Hour),
windowDays: 30,
wantMin: 0.0,
wantMax: 0.0,
},
{
name: "zero time",
ts: time.Time{},
windowDays: 30,
wantMin: 0.0,
wantMax: 0.0,
},
{
name: "zero window",
ts: now,
windowDays: 0,
wantMin: 0.0,
wantMax: 0.0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
score := semanticRecencyScore(now, tt.ts, tt.windowDays)
if score < tt.wantMin || score > tt.wantMax {
t.Errorf("semanticRecencyScore() = %v, want between %v and %v", score, tt.wantMin, tt.wantMax)
}
})
}
}
+365
View File
@@ -0,0 +1,365 @@
package memory
import (
"encoding/json"
"testing"
"time"
)
func TestEvent_JSON_Serialization(t *testing.T) {
tests := []struct {
name string
event Event
wantErr bool
}{
{
name: "full event with all fields",
event: Event{
TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC),
Kind: "message",
Role: "user",
Type: "text",
Text: "Hello, world!",
Meta: map[string]string{"key": "value", "model": "gpt-4"},
},
wantErr: false,
},
{
name: "minimal event with only required fields",
event: Event{
TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC),
Kind: "system",
},
wantErr: false,
},
{
name: "event with empty optional fields",
event: Event{
TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC),
Kind: "assistant",
Role: "",
Type: "",
Text: "",
Meta: nil,
},
wantErr: false,
},
{
name: "event with zero timestamp",
event: Event{
Kind: "message",
Text: "test",
},
wantErr: false,
},
{
name: "event with unicode text",
event: Event{
TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC),
Kind: "message",
Text: "Hello \u4e16\u754c! \U0001F600",
},
wantErr: false,
},
{
name: "event with multiline text",
event: Event{
TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC),
Kind: "message",
Text: "Line 1\nLine 2\nLine 3",
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Marshal to JSON
data, err := json.Marshal(tt.event)
if (err != nil) != tt.wantErr {
t.Errorf("json.Marshal() error = %v, wantErr %v", err, tt.wantErr)
return
}
if tt.wantErr {
return
}
// Unmarshal back to Event
var decoded Event
err = json.Unmarshal(data, &decoded)
if err != nil {
t.Errorf("json.Unmarshal() error = %v", err)
return
}
// Compare fields
if !tt.event.TS.Equal(decoded.TS) {
t.Errorf("TS mismatch: got %v, want %v", decoded.TS, tt.event.TS)
}
if decoded.Kind != tt.event.Kind {
t.Errorf("Kind mismatch: got %q, want %q", decoded.Kind, tt.event.Kind)
}
if decoded.Role != tt.event.Role {
t.Errorf("Role mismatch: got %q, want %q", decoded.Role, tt.event.Role)
}
if decoded.Type != tt.event.Type {
t.Errorf("Type mismatch: got %q, want %q", decoded.Type, tt.event.Type)
}
if decoded.Text != tt.event.Text {
t.Errorf("Text mismatch: got %q, want %q", decoded.Text, tt.event.Text)
}
if len(tt.event.Meta) != len(decoded.Meta) {
t.Errorf("Meta length mismatch: got %d, want %d", len(decoded.Meta), len(tt.event.Meta))
}
for k, v := range tt.event.Meta {
if decoded.Meta[k] != v {
t.Errorf("Meta[%q] mismatch: got %q, want %q", k, decoded.Meta[k], v)
}
}
})
}
}
func TestEvent_JSON_OmitEmpty(t *testing.T) {
tests := []struct {
name string
event Event
shouldContain []string
shouldOmit []string
}{
{
name: "omits empty optional fields",
event: Event{
TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC),
Kind: "message",
},
shouldContain: []string{"ts", "kind"},
shouldOmit: []string{"role", "type", "text", "meta"},
},
{
name: "includes non-empty optional fields",
event: Event{
TS: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC),
Kind: "message",
Role: "user",
Text: "hello",
},
shouldContain: []string{"ts", "kind", "role", "text"},
shouldOmit: []string{"type", "meta"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
data, err := json.Marshal(tt.event)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
jsonStr := string(data)
for _, field := range tt.shouldContain {
if !containsField(jsonStr, field) {
t.Errorf("JSON should contain field %q, got: %s", field, jsonStr)
}
}
for _, field := range tt.shouldOmit {
if containsField(jsonStr, field) {
t.Errorf("JSON should omit field %q, got: %s", field, jsonStr)
}
}
})
}
}
func TestEvent_Fields(t *testing.T) {
tests := []struct {
name string
event Event
wantKind string
wantRole string
wantType string
wantText string
}{
{
name: "user message event",
event: Event{
Kind: "message",
Role: "user",
Type: "text",
Text: "What is the weather?",
},
wantKind: "message",
wantRole: "user",
wantType: "text",
wantText: "What is the weather?",
},
{
name: "assistant response event",
event: Event{
Kind: "message",
Role: "assistant",
Type: "text",
Text: "The weather is sunny.",
},
wantKind: "message",
wantRole: "assistant",
wantType: "text",
wantText: "The weather is sunny.",
},
{
name: "system event",
event: Event{
Kind: "system",
Type: "config",
Text: "Session started",
},
wantKind: "system",
wantRole: "",
wantType: "config",
wantText: "Session started",
},
{
name: "tool use event",
event: Event{
Kind: "tool_use",
Role: "assistant",
Type: "bash",
Text: "ls -la",
},
wantKind: "tool_use",
wantRole: "assistant",
wantType: "bash",
wantText: "ls -la",
},
{
name: "tool result event",
event: Event{
Kind: "tool_result",
Role: "user",
Type: "bash",
Text: "file1.txt\nfile2.txt",
},
wantKind: "tool_result",
wantRole: "user",
wantType: "bash",
wantText: "file1.txt\nfile2.txt",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.event.Kind != tt.wantKind {
t.Errorf("Kind = %q, want %q", tt.event.Kind, tt.wantKind)
}
if tt.event.Role != tt.wantRole {
t.Errorf("Role = %q, want %q", tt.event.Role, tt.wantRole)
}
if tt.event.Type != tt.wantType {
t.Errorf("Type = %q, want %q", tt.event.Type, tt.wantType)
}
if tt.event.Text != tt.wantText {
t.Errorf("Text = %q, want %q", tt.event.Text, tt.wantText)
}
})
}
}
func TestEvent_MetaField(t *testing.T) {
tests := []struct {
name string
meta map[string]string
key string
wantVal string
wantOk bool
}{
{
name: "existing key",
meta: map[string]string{"model": "gpt-4", "tokens": "100"},
key: "model",
wantVal: "gpt-4",
wantOk: true,
},
{
name: "missing key",
meta: map[string]string{"model": "gpt-4"},
key: "tokens",
wantVal: "",
wantOk: false,
},
{
name: "nil meta",
meta: nil,
key: "any",
wantVal: "",
wantOk: false,
},
{
name: "empty meta",
meta: map[string]string{},
key: "any",
wantVal: "",
wantOk: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event := Event{Meta: tt.meta}
val, ok := event.Meta[tt.key]
if ok != tt.wantOk {
t.Errorf("Meta[%q] ok = %v, want %v", tt.key, ok, tt.wantOk)
}
if val != tt.wantVal {
t.Errorf("Meta[%q] = %q, want %q", tt.key, val, tt.wantVal)
}
})
}
}
func TestEvent_Timestamp(t *testing.T) {
tests := []struct {
name string
ts time.Time
isZero bool
}{
{
name: "valid timestamp",
ts: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC),
isZero: false,
},
{
name: "zero timestamp",
ts: time.Time{},
isZero: true,
},
{
name: "unix epoch",
ts: time.Unix(0, 0).UTC(),
isZero: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event := Event{TS: tt.ts}
if event.TS.IsZero() != tt.isZero {
t.Errorf("TS.IsZero() = %v, want %v", event.TS.IsZero(), tt.isZero)
}
})
}
}
// containsField checks if a JSON string contains a specific field key
func containsField(jsonStr, field string) bool {
return json.Valid([]byte(jsonStr)) && (len(jsonStr) > 0 && (jsonStr[0] == '{' || jsonStr[0] == '[')) &&
(len(field) > 0 && (jsonFieldPresent(jsonStr, field)))
}
func jsonFieldPresent(jsonStr, field string) bool {
var m map[string]interface{}
if err := json.Unmarshal([]byte(jsonStr), &m); err != nil {
return false
}
_, ok := m[field]
return ok
}