feat(translator): add translation benchmarking utilities

Add benchmark suite for measuring translation performance including
P50/P95/P99 latency percentiles and comparison tools.
This commit is contained in:
Finessse
2025-12-28 07:07:04 +07:00
parent 185dcc46dd
commit 5de9cfa8ea
2 changed files with 764 additions and 0 deletions
+427
View File
@@ -0,0 +1,427 @@
package translator
import (
"context"
"fmt"
"runtime"
"sort"
"time"
)
// BenchmarkResult contains timing statistics from a benchmark run.
type BenchmarkResult struct {
Iterations int `json:"iterations"`
Min time.Duration `json:"min"`
Max time.Duration `json:"max"`
Avg time.Duration `json:"avg"`
P50 time.Duration `json:"p50"`
P95 time.Duration `json:"p95"`
P99 time.Duration `json:"p99"`
Total time.Duration `json:"total"`
OpsPerSec float64 `json:"ops_per_sec"`
Errors int `json:"errors"`
}
// BenchmarkConfig configures benchmark behavior.
type BenchmarkConfig struct {
// WarmupIterations runs before measurement starts
WarmupIterations int
// ForceGC runs garbage collection between iterations
ForceGC bool
// DisableCache disables caching during benchmark
DisableCache bool
// Parallel runs iterations in parallel with this many goroutines
Parallel int
}
// DefaultBenchmarkConfig returns sensible defaults.
func DefaultBenchmarkConfig() BenchmarkConfig {
return BenchmarkConfig{
WarmupIterations: 10,
ForceGC: false,
DisableCache: true,
Parallel: 1,
}
}
// BenchmarkTranslation measures translation performance.
func BenchmarkTranslation(from, to Format, model string, payload []byte, iterations int) BenchmarkResult {
return BenchmarkTranslationWithConfig(from, to, model, payload, iterations, DefaultBenchmarkConfig())
}
// BenchmarkTranslationWithConfig measures translation with custom configuration.
func BenchmarkTranslationWithConfig(from, to Format, model string, payload []byte, iterations int, config BenchmarkConfig) BenchmarkResult {
if iterations < 1 {
iterations = 1
}
// Save and restore cache state if disabling
if config.DisableCache {
wasEnabled := defaultCache.IsEnabled()
defaultCache.SetCacheEnabled(false)
defer defaultCache.SetCacheEnabled(wasEnabled)
}
// Warmup
for i := 0; i < config.WarmupIterations; i++ {
_ = TranslateRequest(from, to, model, payload, false)
}
if config.ForceGC {
runtime.GC()
}
durations := make([]time.Duration, iterations)
errors := 0
if config.Parallel > 1 {
durations, errors = runParallelBenchmark(from, to, model, payload, iterations, config.Parallel)
} else {
for i := 0; i < iterations; i++ {
start := time.Now()
result := TranslateRequest(from, to, model, payload, false)
durations[i] = time.Since(start)
if result == nil {
errors++
}
}
}
return calculateBenchmarkResult(durations, errors)
}
// runParallelBenchmark runs iterations across multiple goroutines.
func runParallelBenchmark(from, to Format, model string, payload []byte, iterations, parallel int) ([]time.Duration, int) {
durations := make([]time.Duration, iterations)
errCount := make(chan int, parallel)
iterPerWorker := iterations / parallel
remainder := iterations % parallel
done := make(chan struct{})
idx := 0
for w := 0; w < parallel; w++ {
count := iterPerWorker
if w < remainder {
count++
}
startIdx := idx
idx += count
go func(start, cnt int) {
localErrors := 0
for i := 0; i < cnt; i++ {
s := time.Now()
result := TranslateRequest(from, to, model, payload, false)
durations[start+i] = time.Since(s)
if result == nil {
localErrors++
}
}
errCount <- localErrors
done <- struct{}{}
}(startIdx, count)
}
totalErrors := 0
for i := 0; i < parallel; i++ {
<-done
totalErrors += <-errCount
}
return durations, totalErrors
}
// calculateBenchmarkResult computes statistics from durations.
func calculateBenchmarkResult(durations []time.Duration, errors int) BenchmarkResult {
if len(durations) == 0 {
return BenchmarkResult{}
}
// Sort for percentile calculation
sorted := make([]time.Duration, len(durations))
copy(sorted, durations)
sort.Slice(sorted, func(i, j int) bool {
return sorted[i] < sorted[j]
})
var total time.Duration
for _, d := range durations {
total += d
}
n := len(durations)
result := BenchmarkResult{
Iterations: n,
Min: sorted[0],
Max: sorted[n-1],
Avg: total / time.Duration(n),
P50: sorted[n*50/100],
P95: sorted[n*95/100],
P99: sorted[n*99/100],
Total: total,
Errors: errors,
}
if total > 0 {
result.OpsPerSec = float64(n) / total.Seconds()
}
return result
}
// PathKey represents a translation path for benchmarking.
type PathKey struct {
From Format
To Format
}
// String returns a string representation of the path.
func (pk PathKey) String() string {
return fmt.Sprintf("%s->%s", pk.From, pk.To)
}
// BenchmarkAllPaths benchmarks all registered translation paths.
func BenchmarkAllPaths(model string, payload []byte, iterations int) map[string]BenchmarkResult {
return BenchmarkAllPathsWithConfig(model, payload, iterations, DefaultBenchmarkConfig())
}
// BenchmarkAllPathsWithConfig benchmarks all paths with custom configuration.
func BenchmarkAllPathsWithConfig(model string, payload []byte, iterations int, config BenchmarkConfig) map[string]BenchmarkResult {
results := make(map[string]BenchmarkResult)
defaultRegistry.mu.RLock()
paths := make([]PathKey, 0)
for from, targets := range defaultRegistry.requests {
for to := range targets {
paths = append(paths, PathKey{From: from, To: to})
}
}
defaultRegistry.mu.RUnlock()
for _, path := range paths {
key := path.String()
results[key] = BenchmarkTranslationWithConfig(path.From, path.To, model, payload, iterations, config)
}
return results
}
// BenchmarkRegistry benchmarks a specific registry instance.
func BenchmarkRegistry(r *Registry, from, to Format, model string, payload []byte, iterations int) BenchmarkResult {
if iterations < 1 {
iterations = 1
}
config := DefaultBenchmarkConfig()
// Warmup
for i := 0; i < config.WarmupIterations; i++ {
_ = r.TranslateRequest(from, to, model, payload, false)
}
durations := make([]time.Duration, iterations)
errors := 0
for i := 0; i < iterations; i++ {
start := time.Now()
result := r.TranslateRequest(from, to, model, payload, false)
durations[i] = time.Since(start)
if result == nil {
errors++
}
}
return calculateBenchmarkResult(durations, errors)
}
// CompareBenchmarks compares two benchmark results.
type BenchmarkComparison struct {
Name1 string `json:"name1"`
Name2 string `json:"name2"`
AvgDiff time.Duration `json:"avg_diff"`
AvgDiffPct float64 `json:"avg_diff_pct"`
P95Diff time.Duration `json:"p95_diff"`
P95DiffPct float64 `json:"p95_diff_pct"`
Faster string `json:"faster"`
SpeedupRatio float64 `json:"speedup_ratio"`
}
// CompareBenchmarks compares two benchmark results.
func CompareBenchmarks(name1 string, result1 BenchmarkResult, name2 string, result2 BenchmarkResult) BenchmarkComparison {
comp := BenchmarkComparison{
Name1: name1,
Name2: name2,
AvgDiff: result1.Avg - result2.Avg,
P95Diff: result1.P95 - result2.P95,
}
if result2.Avg > 0 {
comp.AvgDiffPct = float64(comp.AvgDiff) / float64(result2.Avg) * 100
}
if result2.P95 > 0 {
comp.P95DiffPct = float64(comp.P95Diff) / float64(result2.P95) * 100
}
if result1.Avg < result2.Avg {
comp.Faster = name1
if result1.Avg > 0 {
comp.SpeedupRatio = float64(result2.Avg) / float64(result1.Avg)
}
} else {
comp.Faster = name2
if result2.Avg > 0 {
comp.SpeedupRatio = float64(result1.Avg) / float64(result2.Avg)
}
}
return comp
}
// BenchmarkSuite provides a comprehensive benchmarking API.
type BenchmarkSuite struct {
registry *Registry
config BenchmarkConfig
results map[string]BenchmarkResult
}
// NewBenchmarkSuite creates a new benchmark suite.
func NewBenchmarkSuite(registry *Registry) *BenchmarkSuite {
if registry == nil {
registry = defaultRegistry
}
return &BenchmarkSuite{
registry: registry,
config: DefaultBenchmarkConfig(),
results: make(map[string]BenchmarkResult),
}
}
// WithConfig sets the benchmark configuration.
func (bs *BenchmarkSuite) WithConfig(config BenchmarkConfig) *BenchmarkSuite {
bs.config = config
return bs
}
// Run benchmarks a single translation path.
func (bs *BenchmarkSuite) Run(name string, from, to Format, model string, payload []byte, iterations int) *BenchmarkSuite {
result := BenchmarkTranslationWithConfig(from, to, model, payload, iterations, bs.config)
bs.results[name] = result
return bs
}
// RunAll benchmarks all registered paths.
func (bs *BenchmarkSuite) RunAll(model string, payload []byte, iterations int) *BenchmarkSuite {
allResults := BenchmarkAllPathsWithConfig(model, payload, iterations, bs.config)
for k, v := range allResults {
bs.results[k] = v
}
return bs
}
// GetResults returns all benchmark results.
func (bs *BenchmarkSuite) GetResults() map[string]BenchmarkResult {
return bs.results
}
// GetResult returns a specific benchmark result.
func (bs *BenchmarkSuite) GetResult(name string) (BenchmarkResult, bool) {
r, ok := bs.results[name]
return r, ok
}
// Compare compares two named benchmarks in the suite.
func (bs *BenchmarkSuite) Compare(name1, name2 string) (BenchmarkComparison, bool) {
r1, ok1 := bs.results[name1]
r2, ok2 := bs.results[name2]
if !ok1 || !ok2 {
return BenchmarkComparison{}, false
}
return CompareBenchmarks(name1, r1, name2, r2), true
}
// Report generates a summary report of all benchmarks.
func (bs *BenchmarkSuite) Report() []BenchmarkReportEntry {
entries := make([]BenchmarkReportEntry, 0, len(bs.results))
for name, result := range bs.results {
entries = append(entries, BenchmarkReportEntry{
Name: name,
Result: result,
})
}
sort.Slice(entries, func(i, j int) bool {
return entries[i].Result.Avg < entries[j].Result.Avg
})
return entries
}
// BenchmarkReportEntry is a single entry in a benchmark report.
type BenchmarkReportEntry struct {
Name string `json:"name"`
Result BenchmarkResult `json:"result"`
}
// BenchmarkWithContext benchmarks a translation with context timeout support.
func BenchmarkWithContext(ctx context.Context, from, to Format, model string, payload []byte, iterations int) (BenchmarkResult, error) {
_ = DefaultBenchmarkConfig() // Config available for future use
if ctx == nil {
ctx = context.Background()
}
durations := make([]time.Duration, 0, iterations)
errors := 0
for i := 0; i < iterations; i++ {
select {
case <-ctx.Done():
// Return partial results
return calculateBenchmarkResult(durations, errors), ctx.Err()
default:
start := time.Now()
result := TranslateRequest(from, to, model, payload, false)
durations = append(durations, time.Since(start))
if result == nil {
errors++
}
}
}
return calculateBenchmarkResult(durations, errors), nil
}
// MemoryBenchmark measures memory allocation during translation.
type MemoryBenchmark struct {
AllocsBefore uint64 `json:"allocs_before"`
AllocsAfter uint64 `json:"allocs_after"`
AllocsDiff uint64 `json:"allocs_diff"`
BytesBefore uint64 `json:"bytes_before"`
BytesAfter uint64 `json:"bytes_after"`
BytesDiff uint64 `json:"bytes_diff"`
Duration time.Duration `json:"duration"`
}
// BenchmarkMemory measures memory usage of a translation.
func BenchmarkMemory(from, to Format, model string, payload []byte) MemoryBenchmark {
var memBefore, memAfter runtime.MemStats
runtime.GC()
runtime.ReadMemStats(&memBefore)
start := time.Now()
_ = TranslateRequest(from, to, model, payload, false)
duration := time.Since(start)
runtime.ReadMemStats(&memAfter)
return MemoryBenchmark{
AllocsBefore: memBefore.Mallocs,
AllocsAfter: memAfter.Mallocs,
AllocsDiff: memAfter.Mallocs - memBefore.Mallocs,
BytesBefore: memBefore.TotalAlloc,
BytesAfter: memAfter.TotalAlloc,
BytesDiff: memAfter.TotalAlloc - memBefore.TotalAlloc,
Duration: duration,
}
}
+337
View File
@@ -0,0 +1,337 @@
package translator
import (
"context"
"testing"
"time"
)
func TestBenchmarkTranslation_Basic(t *testing.T) {
reg := NewRegistry()
from := Format("test-from")
to := Format("test-to")
reg.Register(from, to, func(model string, rawJSON []byte, stream bool) []byte {
// Simulate some work
time.Sleep(1 * time.Millisecond)
return rawJSON
}, ResponseTransform{})
// Use the default registry for the benchmark
defaultRegistry.Register(from, to, func(model string, rawJSON []byte, stream bool) []byte {
time.Sleep(1 * time.Millisecond)
return rawJSON
}, ResponseTransform{})
result := BenchmarkTranslation(from, to, "model", []byte("test payload"), 10)
if result.Iterations != 10 {
t.Errorf("expected 10 iterations, got %d", result.Iterations)
}
if result.Min == 0 {
t.Error("expected non-zero min duration")
}
if result.Max == 0 {
t.Error("expected non-zero max duration")
}
if result.Avg == 0 {
t.Error("expected non-zero avg duration")
}
if result.Min > result.Avg || result.Avg > result.Max {
t.Error("invalid min/avg/max relationship")
}
if result.OpsPerSec == 0 {
t.Error("expected non-zero ops/sec")
}
}
func TestBenchmarkTranslation_WithConfig(t *testing.T) {
from := Format("test-from")
to := Format("test-to")
config := BenchmarkConfig{
WarmupIterations: 2,
ForceGC: true,
DisableCache: true,
Parallel: 1,
}
result := BenchmarkTranslationWithConfig(from, to, "model", []byte("test"), 5, config)
if result.Iterations != 5 {
t.Errorf("expected 5 iterations, got %d", result.Iterations)
}
}
func TestBenchmarkTranslation_Parallel(t *testing.T) {
from := Format("test-from")
to := Format("test-to")
config := BenchmarkConfig{
WarmupIterations: 1,
Parallel: 4,
}
result := BenchmarkTranslationWithConfig(from, to, "model", []byte("test"), 20, config)
if result.Iterations != 20 {
t.Errorf("expected 20 iterations, got %d", result.Iterations)
}
}
func TestBenchmarkResult_Percentiles(t *testing.T) {
from := Format("test-from")
to := Format("test-to")
// Need enough iterations for meaningful percentiles
result := BenchmarkTranslation(from, to, "model", []byte("test"), 100)
if result.P50 == 0 {
t.Error("expected non-zero P50")
}
if result.P95 == 0 {
t.Error("expected non-zero P95")
}
if result.P99 == 0 {
t.Error("expected non-zero P99")
}
if result.P50 > result.P95 || result.P95 > result.P99 {
t.Error("percentiles should be in ascending order")
}
}
func TestBenchmarkAllPaths(t *testing.T) {
// Register some test translators on default registry
from1 := Format("bench-from1")
to1 := Format("bench-to1")
from2 := Format("bench-from2")
to2 := Format("bench-to2")
Register(from1, to1, func(model string, rawJSON []byte, stream bool) []byte {
return rawJSON
}, ResponseTransform{})
Register(from2, to2, func(model string, rawJSON []byte, stream bool) []byte {
return rawJSON
}, ResponseTransform{})
results := BenchmarkAllPaths("model", []byte("test"), 5)
if len(results) < 2 {
t.Errorf("expected at least 2 paths benchmarked, got %d", len(results))
}
// Check specific paths exist
key1 := PathKey{From: from1, To: to1}.String()
if _, ok := results[key1]; !ok {
t.Errorf("expected results for %s", key1)
}
}
func TestPathKey_String(t *testing.T) {
pk := PathKey{From: "openai", To: "anthropic"}
expected := "openai->anthropic"
if pk.String() != expected {
t.Errorf("expected %s, got %s", expected, pk.String())
}
}
func TestCompareBenchmarks(t *testing.T) {
r1 := BenchmarkResult{
Iterations: 100,
Avg: 10 * time.Millisecond,
P95: 15 * time.Millisecond,
}
r2 := BenchmarkResult{
Iterations: 100,
Avg: 20 * time.Millisecond,
P95: 30 * time.Millisecond,
}
comp := CompareBenchmarks("fast", r1, "slow", r2)
if comp.Faster != "fast" {
t.Errorf("expected 'fast' to be faster, got %s", comp.Faster)
}
if comp.SpeedupRatio != 2.0 {
t.Errorf("expected 2x speedup, got %f", comp.SpeedupRatio)
}
if comp.AvgDiff != -10*time.Millisecond {
t.Errorf("expected -10ms diff, got %v", comp.AvgDiff)
}
}
func TestBenchmarkSuite(t *testing.T) {
from := Format("suite-from")
to := Format("suite-to")
Register(from, to, func(model string, rawJSON []byte, stream bool) []byte {
return rawJSON
}, ResponseTransform{})
suite := NewBenchmarkSuite(nil).
WithConfig(BenchmarkConfig{
WarmupIterations: 1,
DisableCache: true,
}).
Run("test1", from, to, "model", []byte("data1"), 5).
Run("test2", from, to, "model", []byte("data2"), 5)
results := suite.GetResults()
if len(results) < 2 {
t.Errorf("expected at least 2 results, got %d", len(results))
}
r, ok := suite.GetResult("test1")
if !ok {
t.Error("expected to find test1 result")
}
if r.Iterations != 5 {
t.Errorf("expected 5 iterations, got %d", r.Iterations)
}
_, ok = suite.GetResult("nonexistent")
if ok {
t.Error("expected nonexistent to not be found")
}
}
func TestBenchmarkSuite_Compare(t *testing.T) {
from := Format("compare-from")
to := Format("compare-to")
suite := NewBenchmarkSuite(nil).
Run("a", from, to, "m", []byte("d"), 5).
Run("b", from, to, "m", []byte("d"), 5)
comp, ok := suite.Compare("a", "b")
if !ok {
t.Error("expected comparison to succeed")
}
if comp.Name1 != "a" || comp.Name2 != "b" {
t.Error("comparison names don't match")
}
_, ok = suite.Compare("a", "nonexistent")
if ok {
t.Error("expected comparison to fail for nonexistent")
}
}
func TestBenchmarkSuite_Report(t *testing.T) {
from := Format("report-from")
to := Format("report-to")
suite := NewBenchmarkSuite(nil).
Run("slow", from, to, "m", []byte("d"), 5).
Run("fast", from, to, "m", []byte("d"), 5)
report := suite.Report()
if len(report) != 2 {
t.Errorf("expected 2 entries, got %d", len(report))
}
// Report should be sorted by avg duration
for i := 0; i < len(report)-1; i++ {
if report[i].Result.Avg > report[i+1].Result.Avg {
t.Error("report should be sorted by avg duration")
}
}
}
func TestBenchmarkWithContext(t *testing.T) {
from := Format("ctx-from")
to := Format("ctx-to")
result, err := BenchmarkWithContext(context.Background(), from, to, "m", []byte("d"), 10)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if result.Iterations != 10 {
t.Errorf("expected 10 iterations, got %d", result.Iterations)
}
}
func TestBenchmarkWithContext_Timeout(t *testing.T) {
from := Format("timeout-from")
to := Format("timeout-to")
// Register a slow transformer
defaultRegistry.Register(from, to, func(model string, rawJSON []byte, stream bool) []byte {
time.Sleep(50 * time.Millisecond)
return rawJSON
}, ResponseTransform{})
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
result, err := BenchmarkWithContext(ctx, from, to, "m", []byte("d"), 100)
// Should be cancelled before all iterations complete
if err == nil {
t.Log("benchmark completed before timeout (this is OK if machine is fast)")
}
if result.Iterations == 0 {
t.Error("should have completed at least some iterations")
}
}
func TestBenchmarkWithContext_NilContext(t *testing.T) {
from := Format("nil-ctx-from")
to := Format("nil-ctx-to")
result, err := BenchmarkWithContext(nil, from, to, "m", []byte("d"), 5)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if result.Iterations != 5 {
t.Errorf("expected 5 iterations, got %d", result.Iterations)
}
}
func TestBenchmarkMemory(t *testing.T) {
from := Format("mem-from")
to := Format("mem-to")
result := BenchmarkMemory(from, to, "m", []byte("test data"))
// Duration can be 0 on very fast machines
// Memory stats might be 0 if GC runs during measurement
// Just check they're not negative
if result.AllocsDiff < 0 {
t.Error("allocs diff should not be negative")
}
}
func TestBenchmarkRegistry(t *testing.T) {
reg := NewRegistry()
from := Format("reg-from")
to := Format("reg-to")
reg.Register(from, to, func(model string, rawJSON []byte, stream bool) []byte {
return rawJSON
}, ResponseTransform{})
result := BenchmarkRegistry(reg, from, to, "model", []byte("test"), 10)
if result.Iterations != 10 {
t.Errorf("expected 10 iterations, got %d", result.Iterations)
}
}
func TestDefaultBenchmarkConfig(t *testing.T) {
config := DefaultBenchmarkConfig()
if config.WarmupIterations != 10 {
t.Errorf("expected 10 warmup iterations, got %d", config.WarmupIterations)
}
if config.Parallel != 1 {
t.Errorf("expected 1 parallel, got %d", config.Parallel)
}
if !config.DisableCache {
t.Error("expected DisableCache to be true by default")
}
}