diff --git a/sdk/translator/benchmark.go b/sdk/translator/benchmark.go new file mode 100644 index 00000000..c346767b --- /dev/null +++ b/sdk/translator/benchmark.go @@ -0,0 +1,427 @@ +package translator + +import ( + "context" + "fmt" + "runtime" + "sort" + "time" +) + +// BenchmarkResult contains timing statistics from a benchmark run. +type BenchmarkResult struct { + Iterations int `json:"iterations"` + Min time.Duration `json:"min"` + Max time.Duration `json:"max"` + Avg time.Duration `json:"avg"` + P50 time.Duration `json:"p50"` + P95 time.Duration `json:"p95"` + P99 time.Duration `json:"p99"` + Total time.Duration `json:"total"` + OpsPerSec float64 `json:"ops_per_sec"` + Errors int `json:"errors"` +} + +// BenchmarkConfig configures benchmark behavior. +type BenchmarkConfig struct { + // WarmupIterations runs before measurement starts + WarmupIterations int + // ForceGC runs garbage collection between iterations + ForceGC bool + // DisableCache disables caching during benchmark + DisableCache bool + // Parallel runs iterations in parallel with this many goroutines + Parallel int +} + +// DefaultBenchmarkConfig returns sensible defaults. +func DefaultBenchmarkConfig() BenchmarkConfig { + return BenchmarkConfig{ + WarmupIterations: 10, + ForceGC: false, + DisableCache: true, + Parallel: 1, + } +} + +// BenchmarkTranslation measures translation performance. +func BenchmarkTranslation(from, to Format, model string, payload []byte, iterations int) BenchmarkResult { + return BenchmarkTranslationWithConfig(from, to, model, payload, iterations, DefaultBenchmarkConfig()) +} + +// BenchmarkTranslationWithConfig measures translation with custom configuration. +func BenchmarkTranslationWithConfig(from, to Format, model string, payload []byte, iterations int, config BenchmarkConfig) BenchmarkResult { + if iterations < 1 { + iterations = 1 + } + + // Save and restore cache state if disabling + if config.DisableCache { + wasEnabled := defaultCache.IsEnabled() + defaultCache.SetCacheEnabled(false) + defer defaultCache.SetCacheEnabled(wasEnabled) + } + + // Warmup + for i := 0; i < config.WarmupIterations; i++ { + _ = TranslateRequest(from, to, model, payload, false) + } + + if config.ForceGC { + runtime.GC() + } + + durations := make([]time.Duration, iterations) + errors := 0 + + if config.Parallel > 1 { + durations, errors = runParallelBenchmark(from, to, model, payload, iterations, config.Parallel) + } else { + for i := 0; i < iterations; i++ { + start := time.Now() + result := TranslateRequest(from, to, model, payload, false) + durations[i] = time.Since(start) + if result == nil { + errors++ + } + } + } + + return calculateBenchmarkResult(durations, errors) +} + +// runParallelBenchmark runs iterations across multiple goroutines. +func runParallelBenchmark(from, to Format, model string, payload []byte, iterations, parallel int) ([]time.Duration, int) { + durations := make([]time.Duration, iterations) + errCount := make(chan int, parallel) + + iterPerWorker := iterations / parallel + remainder := iterations % parallel + + done := make(chan struct{}) + idx := 0 + + for w := 0; w < parallel; w++ { + count := iterPerWorker + if w < remainder { + count++ + } + startIdx := idx + idx += count + + go func(start, cnt int) { + localErrors := 0 + for i := 0; i < cnt; i++ { + s := time.Now() + result := TranslateRequest(from, to, model, payload, false) + durations[start+i] = time.Since(s) + if result == nil { + localErrors++ + } + } + errCount <- localErrors + done <- struct{}{} + }(startIdx, count) + } + + totalErrors := 0 + for i := 0; i < parallel; i++ { + <-done + totalErrors += <-errCount + } + + return durations, totalErrors +} + +// calculateBenchmarkResult computes statistics from durations. +func calculateBenchmarkResult(durations []time.Duration, errors int) BenchmarkResult { + if len(durations) == 0 { + return BenchmarkResult{} + } + + // Sort for percentile calculation + sorted := make([]time.Duration, len(durations)) + copy(sorted, durations) + sort.Slice(sorted, func(i, j int) bool { + return sorted[i] < sorted[j] + }) + + var total time.Duration + for _, d := range durations { + total += d + } + + n := len(durations) + result := BenchmarkResult{ + Iterations: n, + Min: sorted[0], + Max: sorted[n-1], + Avg: total / time.Duration(n), + P50: sorted[n*50/100], + P95: sorted[n*95/100], + P99: sorted[n*99/100], + Total: total, + Errors: errors, + } + + if total > 0 { + result.OpsPerSec = float64(n) / total.Seconds() + } + + return result +} + +// PathKey represents a translation path for benchmarking. +type PathKey struct { + From Format + To Format +} + +// String returns a string representation of the path. +func (pk PathKey) String() string { + return fmt.Sprintf("%s->%s", pk.From, pk.To) +} + +// BenchmarkAllPaths benchmarks all registered translation paths. +func BenchmarkAllPaths(model string, payload []byte, iterations int) map[string]BenchmarkResult { + return BenchmarkAllPathsWithConfig(model, payload, iterations, DefaultBenchmarkConfig()) +} + +// BenchmarkAllPathsWithConfig benchmarks all paths with custom configuration. +func BenchmarkAllPathsWithConfig(model string, payload []byte, iterations int, config BenchmarkConfig) map[string]BenchmarkResult { + results := make(map[string]BenchmarkResult) + + defaultRegistry.mu.RLock() + paths := make([]PathKey, 0) + for from, targets := range defaultRegistry.requests { + for to := range targets { + paths = append(paths, PathKey{From: from, To: to}) + } + } + defaultRegistry.mu.RUnlock() + + for _, path := range paths { + key := path.String() + results[key] = BenchmarkTranslationWithConfig(path.From, path.To, model, payload, iterations, config) + } + + return results +} + +// BenchmarkRegistry benchmarks a specific registry instance. +func BenchmarkRegistry(r *Registry, from, to Format, model string, payload []byte, iterations int) BenchmarkResult { + if iterations < 1 { + iterations = 1 + } + + config := DefaultBenchmarkConfig() + + // Warmup + for i := 0; i < config.WarmupIterations; i++ { + _ = r.TranslateRequest(from, to, model, payload, false) + } + + durations := make([]time.Duration, iterations) + errors := 0 + + for i := 0; i < iterations; i++ { + start := time.Now() + result := r.TranslateRequest(from, to, model, payload, false) + durations[i] = time.Since(start) + if result == nil { + errors++ + } + } + + return calculateBenchmarkResult(durations, errors) +} + +// CompareBenchmarks compares two benchmark results. +type BenchmarkComparison struct { + Name1 string `json:"name1"` + Name2 string `json:"name2"` + AvgDiff time.Duration `json:"avg_diff"` + AvgDiffPct float64 `json:"avg_diff_pct"` + P95Diff time.Duration `json:"p95_diff"` + P95DiffPct float64 `json:"p95_diff_pct"` + Faster string `json:"faster"` + SpeedupRatio float64 `json:"speedup_ratio"` +} + +// CompareBenchmarks compares two benchmark results. +func CompareBenchmarks(name1 string, result1 BenchmarkResult, name2 string, result2 BenchmarkResult) BenchmarkComparison { + comp := BenchmarkComparison{ + Name1: name1, + Name2: name2, + AvgDiff: result1.Avg - result2.Avg, + P95Diff: result1.P95 - result2.P95, + } + + if result2.Avg > 0 { + comp.AvgDiffPct = float64(comp.AvgDiff) / float64(result2.Avg) * 100 + } + if result2.P95 > 0 { + comp.P95DiffPct = float64(comp.P95Diff) / float64(result2.P95) * 100 + } + + if result1.Avg < result2.Avg { + comp.Faster = name1 + if result1.Avg > 0 { + comp.SpeedupRatio = float64(result2.Avg) / float64(result1.Avg) + } + } else { + comp.Faster = name2 + if result2.Avg > 0 { + comp.SpeedupRatio = float64(result1.Avg) / float64(result2.Avg) + } + } + + return comp +} + +// BenchmarkSuite provides a comprehensive benchmarking API. +type BenchmarkSuite struct { + registry *Registry + config BenchmarkConfig + results map[string]BenchmarkResult +} + +// NewBenchmarkSuite creates a new benchmark suite. +func NewBenchmarkSuite(registry *Registry) *BenchmarkSuite { + if registry == nil { + registry = defaultRegistry + } + return &BenchmarkSuite{ + registry: registry, + config: DefaultBenchmarkConfig(), + results: make(map[string]BenchmarkResult), + } +} + +// WithConfig sets the benchmark configuration. +func (bs *BenchmarkSuite) WithConfig(config BenchmarkConfig) *BenchmarkSuite { + bs.config = config + return bs +} + +// Run benchmarks a single translation path. +func (bs *BenchmarkSuite) Run(name string, from, to Format, model string, payload []byte, iterations int) *BenchmarkSuite { + result := BenchmarkTranslationWithConfig(from, to, model, payload, iterations, bs.config) + bs.results[name] = result + return bs +} + +// RunAll benchmarks all registered paths. +func (bs *BenchmarkSuite) RunAll(model string, payload []byte, iterations int) *BenchmarkSuite { + allResults := BenchmarkAllPathsWithConfig(model, payload, iterations, bs.config) + for k, v := range allResults { + bs.results[k] = v + } + return bs +} + +// GetResults returns all benchmark results. +func (bs *BenchmarkSuite) GetResults() map[string]BenchmarkResult { + return bs.results +} + +// GetResult returns a specific benchmark result. +func (bs *BenchmarkSuite) GetResult(name string) (BenchmarkResult, bool) { + r, ok := bs.results[name] + return r, ok +} + +// Compare compares two named benchmarks in the suite. +func (bs *BenchmarkSuite) Compare(name1, name2 string) (BenchmarkComparison, bool) { + r1, ok1 := bs.results[name1] + r2, ok2 := bs.results[name2] + if !ok1 || !ok2 { + return BenchmarkComparison{}, false + } + return CompareBenchmarks(name1, r1, name2, r2), true +} + +// Report generates a summary report of all benchmarks. +func (bs *BenchmarkSuite) Report() []BenchmarkReportEntry { + entries := make([]BenchmarkReportEntry, 0, len(bs.results)) + for name, result := range bs.results { + entries = append(entries, BenchmarkReportEntry{ + Name: name, + Result: result, + }) + } + sort.Slice(entries, func(i, j int) bool { + return entries[i].Result.Avg < entries[j].Result.Avg + }) + return entries +} + +// BenchmarkReportEntry is a single entry in a benchmark report. +type BenchmarkReportEntry struct { + Name string `json:"name"` + Result BenchmarkResult `json:"result"` +} + +// BenchmarkWithContext benchmarks a translation with context timeout support. +func BenchmarkWithContext(ctx context.Context, from, to Format, model string, payload []byte, iterations int) (BenchmarkResult, error) { + _ = DefaultBenchmarkConfig() // Config available for future use + + if ctx == nil { + ctx = context.Background() + } + + durations := make([]time.Duration, 0, iterations) + errors := 0 + + for i := 0; i < iterations; i++ { + select { + case <-ctx.Done(): + // Return partial results + return calculateBenchmarkResult(durations, errors), ctx.Err() + default: + start := time.Now() + result := TranslateRequest(from, to, model, payload, false) + durations = append(durations, time.Since(start)) + if result == nil { + errors++ + } + } + } + + return calculateBenchmarkResult(durations, errors), nil +} + +// MemoryBenchmark measures memory allocation during translation. +type MemoryBenchmark struct { + AllocsBefore uint64 `json:"allocs_before"` + AllocsAfter uint64 `json:"allocs_after"` + AllocsDiff uint64 `json:"allocs_diff"` + BytesBefore uint64 `json:"bytes_before"` + BytesAfter uint64 `json:"bytes_after"` + BytesDiff uint64 `json:"bytes_diff"` + Duration time.Duration `json:"duration"` +} + +// BenchmarkMemory measures memory usage of a translation. +func BenchmarkMemory(from, to Format, model string, payload []byte) MemoryBenchmark { + var memBefore, memAfter runtime.MemStats + + runtime.GC() + runtime.ReadMemStats(&memBefore) + + start := time.Now() + _ = TranslateRequest(from, to, model, payload, false) + duration := time.Since(start) + + runtime.ReadMemStats(&memAfter) + + return MemoryBenchmark{ + AllocsBefore: memBefore.Mallocs, + AllocsAfter: memAfter.Mallocs, + AllocsDiff: memAfter.Mallocs - memBefore.Mallocs, + BytesBefore: memBefore.TotalAlloc, + BytesAfter: memAfter.TotalAlloc, + BytesDiff: memAfter.TotalAlloc - memBefore.TotalAlloc, + Duration: duration, + } +} diff --git a/sdk/translator/benchmark_test.go b/sdk/translator/benchmark_test.go new file mode 100644 index 00000000..924fb933 --- /dev/null +++ b/sdk/translator/benchmark_test.go @@ -0,0 +1,337 @@ +package translator + +import ( + "context" + "testing" + "time" +) + +func TestBenchmarkTranslation_Basic(t *testing.T) { + reg := NewRegistry() + + from := Format("test-from") + to := Format("test-to") + + reg.Register(from, to, func(model string, rawJSON []byte, stream bool) []byte { + // Simulate some work + time.Sleep(1 * time.Millisecond) + return rawJSON + }, ResponseTransform{}) + + // Use the default registry for the benchmark + defaultRegistry.Register(from, to, func(model string, rawJSON []byte, stream bool) []byte { + time.Sleep(1 * time.Millisecond) + return rawJSON + }, ResponseTransform{}) + + result := BenchmarkTranslation(from, to, "model", []byte("test payload"), 10) + + if result.Iterations != 10 { + t.Errorf("expected 10 iterations, got %d", result.Iterations) + } + if result.Min == 0 { + t.Error("expected non-zero min duration") + } + if result.Max == 0 { + t.Error("expected non-zero max duration") + } + if result.Avg == 0 { + t.Error("expected non-zero avg duration") + } + if result.Min > result.Avg || result.Avg > result.Max { + t.Error("invalid min/avg/max relationship") + } + if result.OpsPerSec == 0 { + t.Error("expected non-zero ops/sec") + } +} + +func TestBenchmarkTranslation_WithConfig(t *testing.T) { + from := Format("test-from") + to := Format("test-to") + + config := BenchmarkConfig{ + WarmupIterations: 2, + ForceGC: true, + DisableCache: true, + Parallel: 1, + } + + result := BenchmarkTranslationWithConfig(from, to, "model", []byte("test"), 5, config) + + if result.Iterations != 5 { + t.Errorf("expected 5 iterations, got %d", result.Iterations) + } +} + +func TestBenchmarkTranslation_Parallel(t *testing.T) { + from := Format("test-from") + to := Format("test-to") + + config := BenchmarkConfig{ + WarmupIterations: 1, + Parallel: 4, + } + + result := BenchmarkTranslationWithConfig(from, to, "model", []byte("test"), 20, config) + + if result.Iterations != 20 { + t.Errorf("expected 20 iterations, got %d", result.Iterations) + } +} + +func TestBenchmarkResult_Percentiles(t *testing.T) { + from := Format("test-from") + to := Format("test-to") + + // Need enough iterations for meaningful percentiles + result := BenchmarkTranslation(from, to, "model", []byte("test"), 100) + + if result.P50 == 0 { + t.Error("expected non-zero P50") + } + if result.P95 == 0 { + t.Error("expected non-zero P95") + } + if result.P99 == 0 { + t.Error("expected non-zero P99") + } + if result.P50 > result.P95 || result.P95 > result.P99 { + t.Error("percentiles should be in ascending order") + } +} + +func TestBenchmarkAllPaths(t *testing.T) { + // Register some test translators on default registry + from1 := Format("bench-from1") + to1 := Format("bench-to1") + from2 := Format("bench-from2") + to2 := Format("bench-to2") + + Register(from1, to1, func(model string, rawJSON []byte, stream bool) []byte { + return rawJSON + }, ResponseTransform{}) + + Register(from2, to2, func(model string, rawJSON []byte, stream bool) []byte { + return rawJSON + }, ResponseTransform{}) + + results := BenchmarkAllPaths("model", []byte("test"), 5) + + if len(results) < 2 { + t.Errorf("expected at least 2 paths benchmarked, got %d", len(results)) + } + + // Check specific paths exist + key1 := PathKey{From: from1, To: to1}.String() + if _, ok := results[key1]; !ok { + t.Errorf("expected results for %s", key1) + } +} + +func TestPathKey_String(t *testing.T) { + pk := PathKey{From: "openai", To: "anthropic"} + expected := "openai->anthropic" + if pk.String() != expected { + t.Errorf("expected %s, got %s", expected, pk.String()) + } +} + +func TestCompareBenchmarks(t *testing.T) { + r1 := BenchmarkResult{ + Iterations: 100, + Avg: 10 * time.Millisecond, + P95: 15 * time.Millisecond, + } + + r2 := BenchmarkResult{ + Iterations: 100, + Avg: 20 * time.Millisecond, + P95: 30 * time.Millisecond, + } + + comp := CompareBenchmarks("fast", r1, "slow", r2) + + if comp.Faster != "fast" { + t.Errorf("expected 'fast' to be faster, got %s", comp.Faster) + } + if comp.SpeedupRatio != 2.0 { + t.Errorf("expected 2x speedup, got %f", comp.SpeedupRatio) + } + if comp.AvgDiff != -10*time.Millisecond { + t.Errorf("expected -10ms diff, got %v", comp.AvgDiff) + } +} + +func TestBenchmarkSuite(t *testing.T) { + from := Format("suite-from") + to := Format("suite-to") + + Register(from, to, func(model string, rawJSON []byte, stream bool) []byte { + return rawJSON + }, ResponseTransform{}) + + suite := NewBenchmarkSuite(nil). + WithConfig(BenchmarkConfig{ + WarmupIterations: 1, + DisableCache: true, + }). + Run("test1", from, to, "model", []byte("data1"), 5). + Run("test2", from, to, "model", []byte("data2"), 5) + + results := suite.GetResults() + if len(results) < 2 { + t.Errorf("expected at least 2 results, got %d", len(results)) + } + + r, ok := suite.GetResult("test1") + if !ok { + t.Error("expected to find test1 result") + } + if r.Iterations != 5 { + t.Errorf("expected 5 iterations, got %d", r.Iterations) + } + + _, ok = suite.GetResult("nonexistent") + if ok { + t.Error("expected nonexistent to not be found") + } +} + +func TestBenchmarkSuite_Compare(t *testing.T) { + from := Format("compare-from") + to := Format("compare-to") + + suite := NewBenchmarkSuite(nil). + Run("a", from, to, "m", []byte("d"), 5). + Run("b", from, to, "m", []byte("d"), 5) + + comp, ok := suite.Compare("a", "b") + if !ok { + t.Error("expected comparison to succeed") + } + if comp.Name1 != "a" || comp.Name2 != "b" { + t.Error("comparison names don't match") + } + + _, ok = suite.Compare("a", "nonexistent") + if ok { + t.Error("expected comparison to fail for nonexistent") + } +} + +func TestBenchmarkSuite_Report(t *testing.T) { + from := Format("report-from") + to := Format("report-to") + + suite := NewBenchmarkSuite(nil). + Run("slow", from, to, "m", []byte("d"), 5). + Run("fast", from, to, "m", []byte("d"), 5) + + report := suite.Report() + if len(report) != 2 { + t.Errorf("expected 2 entries, got %d", len(report)) + } + + // Report should be sorted by avg duration + for i := 0; i < len(report)-1; i++ { + if report[i].Result.Avg > report[i+1].Result.Avg { + t.Error("report should be sorted by avg duration") + } + } +} + +func TestBenchmarkWithContext(t *testing.T) { + from := Format("ctx-from") + to := Format("ctx-to") + + result, err := BenchmarkWithContext(context.Background(), from, to, "m", []byte("d"), 10) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if result.Iterations != 10 { + t.Errorf("expected 10 iterations, got %d", result.Iterations) + } +} + +func TestBenchmarkWithContext_Timeout(t *testing.T) { + from := Format("timeout-from") + to := Format("timeout-to") + + // Register a slow transformer + defaultRegistry.Register(from, to, func(model string, rawJSON []byte, stream bool) []byte { + time.Sleep(50 * time.Millisecond) + return rawJSON + }, ResponseTransform{}) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + result, err := BenchmarkWithContext(ctx, from, to, "m", []byte("d"), 100) + + // Should be cancelled before all iterations complete + if err == nil { + t.Log("benchmark completed before timeout (this is OK if machine is fast)") + } + if result.Iterations == 0 { + t.Error("should have completed at least some iterations") + } +} + +func TestBenchmarkWithContext_NilContext(t *testing.T) { + from := Format("nil-ctx-from") + to := Format("nil-ctx-to") + + result, err := BenchmarkWithContext(nil, from, to, "m", []byte("d"), 5) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if result.Iterations != 5 { + t.Errorf("expected 5 iterations, got %d", result.Iterations) + } +} + +func TestBenchmarkMemory(t *testing.T) { + from := Format("mem-from") + to := Format("mem-to") + + result := BenchmarkMemory(from, to, "m", []byte("test data")) + + // Duration can be 0 on very fast machines + // Memory stats might be 0 if GC runs during measurement + // Just check they're not negative + if result.AllocsDiff < 0 { + t.Error("allocs diff should not be negative") + } +} + +func TestBenchmarkRegistry(t *testing.T) { + reg := NewRegistry() + + from := Format("reg-from") + to := Format("reg-to") + + reg.Register(from, to, func(model string, rawJSON []byte, stream bool) []byte { + return rawJSON + }, ResponseTransform{}) + + result := BenchmarkRegistry(reg, from, to, "model", []byte("test"), 10) + + if result.Iterations != 10 { + t.Errorf("expected 10 iterations, got %d", result.Iterations) + } +} + +func TestDefaultBenchmarkConfig(t *testing.T) { + config := DefaultBenchmarkConfig() + + if config.WarmupIterations != 10 { + t.Errorf("expected 10 warmup iterations, got %d", config.WarmupIterations) + } + if config.Parallel != 1 { + t.Errorf("expected 1 parallel, got %d", config.Parallel) + } + if !config.DisableCache { + t.Error("expected DisableCache to be true by default") + } +}