mirror of
https://github.com/Finesssee/ProxyPilot.git
synced 2026-11-19 02:17:46 +00:00
feat(translator): add translation benchmarking utilities
Add benchmark suite for measuring translation performance including P50/P95/P99 latency percentiles and comparison tools.
This commit is contained in:
@@ -0,0 +1,427 @@
|
||||
package translator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// BenchmarkResult contains timing statistics from a benchmark run.
|
||||
type BenchmarkResult struct {
|
||||
Iterations int `json:"iterations"`
|
||||
Min time.Duration `json:"min"`
|
||||
Max time.Duration `json:"max"`
|
||||
Avg time.Duration `json:"avg"`
|
||||
P50 time.Duration `json:"p50"`
|
||||
P95 time.Duration `json:"p95"`
|
||||
P99 time.Duration `json:"p99"`
|
||||
Total time.Duration `json:"total"`
|
||||
OpsPerSec float64 `json:"ops_per_sec"`
|
||||
Errors int `json:"errors"`
|
||||
}
|
||||
|
||||
// BenchmarkConfig configures benchmark behavior.
|
||||
type BenchmarkConfig struct {
|
||||
// WarmupIterations runs before measurement starts
|
||||
WarmupIterations int
|
||||
// ForceGC runs garbage collection between iterations
|
||||
ForceGC bool
|
||||
// DisableCache disables caching during benchmark
|
||||
DisableCache bool
|
||||
// Parallel runs iterations in parallel with this many goroutines
|
||||
Parallel int
|
||||
}
|
||||
|
||||
// DefaultBenchmarkConfig returns sensible defaults.
|
||||
func DefaultBenchmarkConfig() BenchmarkConfig {
|
||||
return BenchmarkConfig{
|
||||
WarmupIterations: 10,
|
||||
ForceGC: false,
|
||||
DisableCache: true,
|
||||
Parallel: 1,
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkTranslation measures translation performance.
|
||||
func BenchmarkTranslation(from, to Format, model string, payload []byte, iterations int) BenchmarkResult {
|
||||
return BenchmarkTranslationWithConfig(from, to, model, payload, iterations, DefaultBenchmarkConfig())
|
||||
}
|
||||
|
||||
// BenchmarkTranslationWithConfig measures translation with custom configuration.
|
||||
func BenchmarkTranslationWithConfig(from, to Format, model string, payload []byte, iterations int, config BenchmarkConfig) BenchmarkResult {
|
||||
if iterations < 1 {
|
||||
iterations = 1
|
||||
}
|
||||
|
||||
// Save and restore cache state if disabling
|
||||
if config.DisableCache {
|
||||
wasEnabled := defaultCache.IsEnabled()
|
||||
defaultCache.SetCacheEnabled(false)
|
||||
defer defaultCache.SetCacheEnabled(wasEnabled)
|
||||
}
|
||||
|
||||
// Warmup
|
||||
for i := 0; i < config.WarmupIterations; i++ {
|
||||
_ = TranslateRequest(from, to, model, payload, false)
|
||||
}
|
||||
|
||||
if config.ForceGC {
|
||||
runtime.GC()
|
||||
}
|
||||
|
||||
durations := make([]time.Duration, iterations)
|
||||
errors := 0
|
||||
|
||||
if config.Parallel > 1 {
|
||||
durations, errors = runParallelBenchmark(from, to, model, payload, iterations, config.Parallel)
|
||||
} else {
|
||||
for i := 0; i < iterations; i++ {
|
||||
start := time.Now()
|
||||
result := TranslateRequest(from, to, model, payload, false)
|
||||
durations[i] = time.Since(start)
|
||||
if result == nil {
|
||||
errors++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return calculateBenchmarkResult(durations, errors)
|
||||
}
|
||||
|
||||
// runParallelBenchmark runs iterations across multiple goroutines.
|
||||
func runParallelBenchmark(from, to Format, model string, payload []byte, iterations, parallel int) ([]time.Duration, int) {
|
||||
durations := make([]time.Duration, iterations)
|
||||
errCount := make(chan int, parallel)
|
||||
|
||||
iterPerWorker := iterations / parallel
|
||||
remainder := iterations % parallel
|
||||
|
||||
done := make(chan struct{})
|
||||
idx := 0
|
||||
|
||||
for w := 0; w < parallel; w++ {
|
||||
count := iterPerWorker
|
||||
if w < remainder {
|
||||
count++
|
||||
}
|
||||
startIdx := idx
|
||||
idx += count
|
||||
|
||||
go func(start, cnt int) {
|
||||
localErrors := 0
|
||||
for i := 0; i < cnt; i++ {
|
||||
s := time.Now()
|
||||
result := TranslateRequest(from, to, model, payload, false)
|
||||
durations[start+i] = time.Since(s)
|
||||
if result == nil {
|
||||
localErrors++
|
||||
}
|
||||
}
|
||||
errCount <- localErrors
|
||||
done <- struct{}{}
|
||||
}(startIdx, count)
|
||||
}
|
||||
|
||||
totalErrors := 0
|
||||
for i := 0; i < parallel; i++ {
|
||||
<-done
|
||||
totalErrors += <-errCount
|
||||
}
|
||||
|
||||
return durations, totalErrors
|
||||
}
|
||||
|
||||
// calculateBenchmarkResult computes statistics from durations.
|
||||
func calculateBenchmarkResult(durations []time.Duration, errors int) BenchmarkResult {
|
||||
if len(durations) == 0 {
|
||||
return BenchmarkResult{}
|
||||
}
|
||||
|
||||
// Sort for percentile calculation
|
||||
sorted := make([]time.Duration, len(durations))
|
||||
copy(sorted, durations)
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
return sorted[i] < sorted[j]
|
||||
})
|
||||
|
||||
var total time.Duration
|
||||
for _, d := range durations {
|
||||
total += d
|
||||
}
|
||||
|
||||
n := len(durations)
|
||||
result := BenchmarkResult{
|
||||
Iterations: n,
|
||||
Min: sorted[0],
|
||||
Max: sorted[n-1],
|
||||
Avg: total / time.Duration(n),
|
||||
P50: sorted[n*50/100],
|
||||
P95: sorted[n*95/100],
|
||||
P99: sorted[n*99/100],
|
||||
Total: total,
|
||||
Errors: errors,
|
||||
}
|
||||
|
||||
if total > 0 {
|
||||
result.OpsPerSec = float64(n) / total.Seconds()
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// PathKey represents a translation path for benchmarking.
|
||||
type PathKey struct {
|
||||
From Format
|
||||
To Format
|
||||
}
|
||||
|
||||
// String returns a string representation of the path.
|
||||
func (pk PathKey) String() string {
|
||||
return fmt.Sprintf("%s->%s", pk.From, pk.To)
|
||||
}
|
||||
|
||||
// BenchmarkAllPaths benchmarks all registered translation paths.
|
||||
func BenchmarkAllPaths(model string, payload []byte, iterations int) map[string]BenchmarkResult {
|
||||
return BenchmarkAllPathsWithConfig(model, payload, iterations, DefaultBenchmarkConfig())
|
||||
}
|
||||
|
||||
// BenchmarkAllPathsWithConfig benchmarks all paths with custom configuration.
|
||||
func BenchmarkAllPathsWithConfig(model string, payload []byte, iterations int, config BenchmarkConfig) map[string]BenchmarkResult {
|
||||
results := make(map[string]BenchmarkResult)
|
||||
|
||||
defaultRegistry.mu.RLock()
|
||||
paths := make([]PathKey, 0)
|
||||
for from, targets := range defaultRegistry.requests {
|
||||
for to := range targets {
|
||||
paths = append(paths, PathKey{From: from, To: to})
|
||||
}
|
||||
}
|
||||
defaultRegistry.mu.RUnlock()
|
||||
|
||||
for _, path := range paths {
|
||||
key := path.String()
|
||||
results[key] = BenchmarkTranslationWithConfig(path.From, path.To, model, payload, iterations, config)
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
// BenchmarkRegistry benchmarks a specific registry instance.
|
||||
func BenchmarkRegistry(r *Registry, from, to Format, model string, payload []byte, iterations int) BenchmarkResult {
|
||||
if iterations < 1 {
|
||||
iterations = 1
|
||||
}
|
||||
|
||||
config := DefaultBenchmarkConfig()
|
||||
|
||||
// Warmup
|
||||
for i := 0; i < config.WarmupIterations; i++ {
|
||||
_ = r.TranslateRequest(from, to, model, payload, false)
|
||||
}
|
||||
|
||||
durations := make([]time.Duration, iterations)
|
||||
errors := 0
|
||||
|
||||
for i := 0; i < iterations; i++ {
|
||||
start := time.Now()
|
||||
result := r.TranslateRequest(from, to, model, payload, false)
|
||||
durations[i] = time.Since(start)
|
||||
if result == nil {
|
||||
errors++
|
||||
}
|
||||
}
|
||||
|
||||
return calculateBenchmarkResult(durations, errors)
|
||||
}
|
||||
|
||||
// CompareBenchmarks compares two benchmark results.
|
||||
type BenchmarkComparison struct {
|
||||
Name1 string `json:"name1"`
|
||||
Name2 string `json:"name2"`
|
||||
AvgDiff time.Duration `json:"avg_diff"`
|
||||
AvgDiffPct float64 `json:"avg_diff_pct"`
|
||||
P95Diff time.Duration `json:"p95_diff"`
|
||||
P95DiffPct float64 `json:"p95_diff_pct"`
|
||||
Faster string `json:"faster"`
|
||||
SpeedupRatio float64 `json:"speedup_ratio"`
|
||||
}
|
||||
|
||||
// CompareBenchmarks compares two benchmark results.
|
||||
func CompareBenchmarks(name1 string, result1 BenchmarkResult, name2 string, result2 BenchmarkResult) BenchmarkComparison {
|
||||
comp := BenchmarkComparison{
|
||||
Name1: name1,
|
||||
Name2: name2,
|
||||
AvgDiff: result1.Avg - result2.Avg,
|
||||
P95Diff: result1.P95 - result2.P95,
|
||||
}
|
||||
|
||||
if result2.Avg > 0 {
|
||||
comp.AvgDiffPct = float64(comp.AvgDiff) / float64(result2.Avg) * 100
|
||||
}
|
||||
if result2.P95 > 0 {
|
||||
comp.P95DiffPct = float64(comp.P95Diff) / float64(result2.P95) * 100
|
||||
}
|
||||
|
||||
if result1.Avg < result2.Avg {
|
||||
comp.Faster = name1
|
||||
if result1.Avg > 0 {
|
||||
comp.SpeedupRatio = float64(result2.Avg) / float64(result1.Avg)
|
||||
}
|
||||
} else {
|
||||
comp.Faster = name2
|
||||
if result2.Avg > 0 {
|
||||
comp.SpeedupRatio = float64(result1.Avg) / float64(result2.Avg)
|
||||
}
|
||||
}
|
||||
|
||||
return comp
|
||||
}
|
||||
|
||||
// BenchmarkSuite provides a comprehensive benchmarking API.
|
||||
type BenchmarkSuite struct {
|
||||
registry *Registry
|
||||
config BenchmarkConfig
|
||||
results map[string]BenchmarkResult
|
||||
}
|
||||
|
||||
// NewBenchmarkSuite creates a new benchmark suite.
|
||||
func NewBenchmarkSuite(registry *Registry) *BenchmarkSuite {
|
||||
if registry == nil {
|
||||
registry = defaultRegistry
|
||||
}
|
||||
return &BenchmarkSuite{
|
||||
registry: registry,
|
||||
config: DefaultBenchmarkConfig(),
|
||||
results: make(map[string]BenchmarkResult),
|
||||
}
|
||||
}
|
||||
|
||||
// WithConfig sets the benchmark configuration.
|
||||
func (bs *BenchmarkSuite) WithConfig(config BenchmarkConfig) *BenchmarkSuite {
|
||||
bs.config = config
|
||||
return bs
|
||||
}
|
||||
|
||||
// Run benchmarks a single translation path.
|
||||
func (bs *BenchmarkSuite) Run(name string, from, to Format, model string, payload []byte, iterations int) *BenchmarkSuite {
|
||||
result := BenchmarkTranslationWithConfig(from, to, model, payload, iterations, bs.config)
|
||||
bs.results[name] = result
|
||||
return bs
|
||||
}
|
||||
|
||||
// RunAll benchmarks all registered paths.
|
||||
func (bs *BenchmarkSuite) RunAll(model string, payload []byte, iterations int) *BenchmarkSuite {
|
||||
allResults := BenchmarkAllPathsWithConfig(model, payload, iterations, bs.config)
|
||||
for k, v := range allResults {
|
||||
bs.results[k] = v
|
||||
}
|
||||
return bs
|
||||
}
|
||||
|
||||
// GetResults returns all benchmark results.
|
||||
func (bs *BenchmarkSuite) GetResults() map[string]BenchmarkResult {
|
||||
return bs.results
|
||||
}
|
||||
|
||||
// GetResult returns a specific benchmark result.
|
||||
func (bs *BenchmarkSuite) GetResult(name string) (BenchmarkResult, bool) {
|
||||
r, ok := bs.results[name]
|
||||
return r, ok
|
||||
}
|
||||
|
||||
// Compare compares two named benchmarks in the suite.
|
||||
func (bs *BenchmarkSuite) Compare(name1, name2 string) (BenchmarkComparison, bool) {
|
||||
r1, ok1 := bs.results[name1]
|
||||
r2, ok2 := bs.results[name2]
|
||||
if !ok1 || !ok2 {
|
||||
return BenchmarkComparison{}, false
|
||||
}
|
||||
return CompareBenchmarks(name1, r1, name2, r2), true
|
||||
}
|
||||
|
||||
// Report generates a summary report of all benchmarks.
|
||||
func (bs *BenchmarkSuite) Report() []BenchmarkReportEntry {
|
||||
entries := make([]BenchmarkReportEntry, 0, len(bs.results))
|
||||
for name, result := range bs.results {
|
||||
entries = append(entries, BenchmarkReportEntry{
|
||||
Name: name,
|
||||
Result: result,
|
||||
})
|
||||
}
|
||||
sort.Slice(entries, func(i, j int) bool {
|
||||
return entries[i].Result.Avg < entries[j].Result.Avg
|
||||
})
|
||||
return entries
|
||||
}
|
||||
|
||||
// BenchmarkReportEntry is a single entry in a benchmark report.
|
||||
type BenchmarkReportEntry struct {
|
||||
Name string `json:"name"`
|
||||
Result BenchmarkResult `json:"result"`
|
||||
}
|
||||
|
||||
// BenchmarkWithContext benchmarks a translation with context timeout support.
|
||||
func BenchmarkWithContext(ctx context.Context, from, to Format, model string, payload []byte, iterations int) (BenchmarkResult, error) {
|
||||
_ = DefaultBenchmarkConfig() // Config available for future use
|
||||
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
|
||||
durations := make([]time.Duration, 0, iterations)
|
||||
errors := 0
|
||||
|
||||
for i := 0; i < iterations; i++ {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Return partial results
|
||||
return calculateBenchmarkResult(durations, errors), ctx.Err()
|
||||
default:
|
||||
start := time.Now()
|
||||
result := TranslateRequest(from, to, model, payload, false)
|
||||
durations = append(durations, time.Since(start))
|
||||
if result == nil {
|
||||
errors++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return calculateBenchmarkResult(durations, errors), nil
|
||||
}
|
||||
|
||||
// MemoryBenchmark measures memory allocation during translation.
|
||||
type MemoryBenchmark struct {
|
||||
AllocsBefore uint64 `json:"allocs_before"`
|
||||
AllocsAfter uint64 `json:"allocs_after"`
|
||||
AllocsDiff uint64 `json:"allocs_diff"`
|
||||
BytesBefore uint64 `json:"bytes_before"`
|
||||
BytesAfter uint64 `json:"bytes_after"`
|
||||
BytesDiff uint64 `json:"bytes_diff"`
|
||||
Duration time.Duration `json:"duration"`
|
||||
}
|
||||
|
||||
// BenchmarkMemory measures memory usage of a translation.
|
||||
func BenchmarkMemory(from, to Format, model string, payload []byte) MemoryBenchmark {
|
||||
var memBefore, memAfter runtime.MemStats
|
||||
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&memBefore)
|
||||
|
||||
start := time.Now()
|
||||
_ = TranslateRequest(from, to, model, payload, false)
|
||||
duration := time.Since(start)
|
||||
|
||||
runtime.ReadMemStats(&memAfter)
|
||||
|
||||
return MemoryBenchmark{
|
||||
AllocsBefore: memBefore.Mallocs,
|
||||
AllocsAfter: memAfter.Mallocs,
|
||||
AllocsDiff: memAfter.Mallocs - memBefore.Mallocs,
|
||||
BytesBefore: memBefore.TotalAlloc,
|
||||
BytesAfter: memAfter.TotalAlloc,
|
||||
BytesDiff: memAfter.TotalAlloc - memBefore.TotalAlloc,
|
||||
Duration: duration,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
package translator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestBenchmarkTranslation_Basic(t *testing.T) {
|
||||
reg := NewRegistry()
|
||||
|
||||
from := Format("test-from")
|
||||
to := Format("test-to")
|
||||
|
||||
reg.Register(from, to, func(model string, rawJSON []byte, stream bool) []byte {
|
||||
// Simulate some work
|
||||
time.Sleep(1 * time.Millisecond)
|
||||
return rawJSON
|
||||
}, ResponseTransform{})
|
||||
|
||||
// Use the default registry for the benchmark
|
||||
defaultRegistry.Register(from, to, func(model string, rawJSON []byte, stream bool) []byte {
|
||||
time.Sleep(1 * time.Millisecond)
|
||||
return rawJSON
|
||||
}, ResponseTransform{})
|
||||
|
||||
result := BenchmarkTranslation(from, to, "model", []byte("test payload"), 10)
|
||||
|
||||
if result.Iterations != 10 {
|
||||
t.Errorf("expected 10 iterations, got %d", result.Iterations)
|
||||
}
|
||||
if result.Min == 0 {
|
||||
t.Error("expected non-zero min duration")
|
||||
}
|
||||
if result.Max == 0 {
|
||||
t.Error("expected non-zero max duration")
|
||||
}
|
||||
if result.Avg == 0 {
|
||||
t.Error("expected non-zero avg duration")
|
||||
}
|
||||
if result.Min > result.Avg || result.Avg > result.Max {
|
||||
t.Error("invalid min/avg/max relationship")
|
||||
}
|
||||
if result.OpsPerSec == 0 {
|
||||
t.Error("expected non-zero ops/sec")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBenchmarkTranslation_WithConfig(t *testing.T) {
|
||||
from := Format("test-from")
|
||||
to := Format("test-to")
|
||||
|
||||
config := BenchmarkConfig{
|
||||
WarmupIterations: 2,
|
||||
ForceGC: true,
|
||||
DisableCache: true,
|
||||
Parallel: 1,
|
||||
}
|
||||
|
||||
result := BenchmarkTranslationWithConfig(from, to, "model", []byte("test"), 5, config)
|
||||
|
||||
if result.Iterations != 5 {
|
||||
t.Errorf("expected 5 iterations, got %d", result.Iterations)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBenchmarkTranslation_Parallel(t *testing.T) {
|
||||
from := Format("test-from")
|
||||
to := Format("test-to")
|
||||
|
||||
config := BenchmarkConfig{
|
||||
WarmupIterations: 1,
|
||||
Parallel: 4,
|
||||
}
|
||||
|
||||
result := BenchmarkTranslationWithConfig(from, to, "model", []byte("test"), 20, config)
|
||||
|
||||
if result.Iterations != 20 {
|
||||
t.Errorf("expected 20 iterations, got %d", result.Iterations)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBenchmarkResult_Percentiles(t *testing.T) {
|
||||
from := Format("test-from")
|
||||
to := Format("test-to")
|
||||
|
||||
// Need enough iterations for meaningful percentiles
|
||||
result := BenchmarkTranslation(from, to, "model", []byte("test"), 100)
|
||||
|
||||
if result.P50 == 0 {
|
||||
t.Error("expected non-zero P50")
|
||||
}
|
||||
if result.P95 == 0 {
|
||||
t.Error("expected non-zero P95")
|
||||
}
|
||||
if result.P99 == 0 {
|
||||
t.Error("expected non-zero P99")
|
||||
}
|
||||
if result.P50 > result.P95 || result.P95 > result.P99 {
|
||||
t.Error("percentiles should be in ascending order")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBenchmarkAllPaths(t *testing.T) {
|
||||
// Register some test translators on default registry
|
||||
from1 := Format("bench-from1")
|
||||
to1 := Format("bench-to1")
|
||||
from2 := Format("bench-from2")
|
||||
to2 := Format("bench-to2")
|
||||
|
||||
Register(from1, to1, func(model string, rawJSON []byte, stream bool) []byte {
|
||||
return rawJSON
|
||||
}, ResponseTransform{})
|
||||
|
||||
Register(from2, to2, func(model string, rawJSON []byte, stream bool) []byte {
|
||||
return rawJSON
|
||||
}, ResponseTransform{})
|
||||
|
||||
results := BenchmarkAllPaths("model", []byte("test"), 5)
|
||||
|
||||
if len(results) < 2 {
|
||||
t.Errorf("expected at least 2 paths benchmarked, got %d", len(results))
|
||||
}
|
||||
|
||||
// Check specific paths exist
|
||||
key1 := PathKey{From: from1, To: to1}.String()
|
||||
if _, ok := results[key1]; !ok {
|
||||
t.Errorf("expected results for %s", key1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathKey_String(t *testing.T) {
|
||||
pk := PathKey{From: "openai", To: "anthropic"}
|
||||
expected := "openai->anthropic"
|
||||
if pk.String() != expected {
|
||||
t.Errorf("expected %s, got %s", expected, pk.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareBenchmarks(t *testing.T) {
|
||||
r1 := BenchmarkResult{
|
||||
Iterations: 100,
|
||||
Avg: 10 * time.Millisecond,
|
||||
P95: 15 * time.Millisecond,
|
||||
}
|
||||
|
||||
r2 := BenchmarkResult{
|
||||
Iterations: 100,
|
||||
Avg: 20 * time.Millisecond,
|
||||
P95: 30 * time.Millisecond,
|
||||
}
|
||||
|
||||
comp := CompareBenchmarks("fast", r1, "slow", r2)
|
||||
|
||||
if comp.Faster != "fast" {
|
||||
t.Errorf("expected 'fast' to be faster, got %s", comp.Faster)
|
||||
}
|
||||
if comp.SpeedupRatio != 2.0 {
|
||||
t.Errorf("expected 2x speedup, got %f", comp.SpeedupRatio)
|
||||
}
|
||||
if comp.AvgDiff != -10*time.Millisecond {
|
||||
t.Errorf("expected -10ms diff, got %v", comp.AvgDiff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBenchmarkSuite(t *testing.T) {
|
||||
from := Format("suite-from")
|
||||
to := Format("suite-to")
|
||||
|
||||
Register(from, to, func(model string, rawJSON []byte, stream bool) []byte {
|
||||
return rawJSON
|
||||
}, ResponseTransform{})
|
||||
|
||||
suite := NewBenchmarkSuite(nil).
|
||||
WithConfig(BenchmarkConfig{
|
||||
WarmupIterations: 1,
|
||||
DisableCache: true,
|
||||
}).
|
||||
Run("test1", from, to, "model", []byte("data1"), 5).
|
||||
Run("test2", from, to, "model", []byte("data2"), 5)
|
||||
|
||||
results := suite.GetResults()
|
||||
if len(results) < 2 {
|
||||
t.Errorf("expected at least 2 results, got %d", len(results))
|
||||
}
|
||||
|
||||
r, ok := suite.GetResult("test1")
|
||||
if !ok {
|
||||
t.Error("expected to find test1 result")
|
||||
}
|
||||
if r.Iterations != 5 {
|
||||
t.Errorf("expected 5 iterations, got %d", r.Iterations)
|
||||
}
|
||||
|
||||
_, ok = suite.GetResult("nonexistent")
|
||||
if ok {
|
||||
t.Error("expected nonexistent to not be found")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBenchmarkSuite_Compare(t *testing.T) {
|
||||
from := Format("compare-from")
|
||||
to := Format("compare-to")
|
||||
|
||||
suite := NewBenchmarkSuite(nil).
|
||||
Run("a", from, to, "m", []byte("d"), 5).
|
||||
Run("b", from, to, "m", []byte("d"), 5)
|
||||
|
||||
comp, ok := suite.Compare("a", "b")
|
||||
if !ok {
|
||||
t.Error("expected comparison to succeed")
|
||||
}
|
||||
if comp.Name1 != "a" || comp.Name2 != "b" {
|
||||
t.Error("comparison names don't match")
|
||||
}
|
||||
|
||||
_, ok = suite.Compare("a", "nonexistent")
|
||||
if ok {
|
||||
t.Error("expected comparison to fail for nonexistent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBenchmarkSuite_Report(t *testing.T) {
|
||||
from := Format("report-from")
|
||||
to := Format("report-to")
|
||||
|
||||
suite := NewBenchmarkSuite(nil).
|
||||
Run("slow", from, to, "m", []byte("d"), 5).
|
||||
Run("fast", from, to, "m", []byte("d"), 5)
|
||||
|
||||
report := suite.Report()
|
||||
if len(report) != 2 {
|
||||
t.Errorf("expected 2 entries, got %d", len(report))
|
||||
}
|
||||
|
||||
// Report should be sorted by avg duration
|
||||
for i := 0; i < len(report)-1; i++ {
|
||||
if report[i].Result.Avg > report[i+1].Result.Avg {
|
||||
t.Error("report should be sorted by avg duration")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBenchmarkWithContext(t *testing.T) {
|
||||
from := Format("ctx-from")
|
||||
to := Format("ctx-to")
|
||||
|
||||
result, err := BenchmarkWithContext(context.Background(), from, to, "m", []byte("d"), 10)
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
if result.Iterations != 10 {
|
||||
t.Errorf("expected 10 iterations, got %d", result.Iterations)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBenchmarkWithContext_Timeout(t *testing.T) {
|
||||
from := Format("timeout-from")
|
||||
to := Format("timeout-to")
|
||||
|
||||
// Register a slow transformer
|
||||
defaultRegistry.Register(from, to, func(model string, rawJSON []byte, stream bool) []byte {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
return rawJSON
|
||||
}, ResponseTransform{})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
result, err := BenchmarkWithContext(ctx, from, to, "m", []byte("d"), 100)
|
||||
|
||||
// Should be cancelled before all iterations complete
|
||||
if err == nil {
|
||||
t.Log("benchmark completed before timeout (this is OK if machine is fast)")
|
||||
}
|
||||
if result.Iterations == 0 {
|
||||
t.Error("should have completed at least some iterations")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBenchmarkWithContext_NilContext(t *testing.T) {
|
||||
from := Format("nil-ctx-from")
|
||||
to := Format("nil-ctx-to")
|
||||
|
||||
result, err := BenchmarkWithContext(nil, from, to, "m", []byte("d"), 5)
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
if result.Iterations != 5 {
|
||||
t.Errorf("expected 5 iterations, got %d", result.Iterations)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBenchmarkMemory(t *testing.T) {
|
||||
from := Format("mem-from")
|
||||
to := Format("mem-to")
|
||||
|
||||
result := BenchmarkMemory(from, to, "m", []byte("test data"))
|
||||
|
||||
// Duration can be 0 on very fast machines
|
||||
// Memory stats might be 0 if GC runs during measurement
|
||||
// Just check they're not negative
|
||||
if result.AllocsDiff < 0 {
|
||||
t.Error("allocs diff should not be negative")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBenchmarkRegistry(t *testing.T) {
|
||||
reg := NewRegistry()
|
||||
|
||||
from := Format("reg-from")
|
||||
to := Format("reg-to")
|
||||
|
||||
reg.Register(from, to, func(model string, rawJSON []byte, stream bool) []byte {
|
||||
return rawJSON
|
||||
}, ResponseTransform{})
|
||||
|
||||
result := BenchmarkRegistry(reg, from, to, "model", []byte("test"), 10)
|
||||
|
||||
if result.Iterations != 10 {
|
||||
t.Errorf("expected 10 iterations, got %d", result.Iterations)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultBenchmarkConfig(t *testing.T) {
|
||||
config := DefaultBenchmarkConfig()
|
||||
|
||||
if config.WarmupIterations != 10 {
|
||||
t.Errorf("expected 10 warmup iterations, got %d", config.WarmupIterations)
|
||||
}
|
||||
if config.Parallel != 1 {
|
||||
t.Errorf("expected 1 parallel, got %d", config.Parallel)
|
||||
}
|
||||
if !config.DisableCache {
|
||||
t.Error("expected DisableCache to be true by default")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user