mirror of
https://github.com/Finesssee/ProxyPilot.git
synced 2026-11-18 18:04:20 +00:00
Upstream changes: - Codex responses compaction payload compatibility - Codex usage_limit_reached retry-after, deterministic parsing - Codex refresh_token_reused non-retryable fix - Qwen rate limiting and quota error handling - Context management compatibility simplification - Antigravity tool_result image fix, model list backfill, deep copy cache - Auth auto-refresh interval/concurrency limiting - Auth hook OnResult refactor (replaces MarkResult) - Gemini-3.1-flash-image support - Registry cleanup of outdated model definitions Fork resolution: - Accept upstream's consolidated conductor.go (auto-refresh, round-tripper) - Delete stale conductor_oauth.go, conductor_token.go (upstream reversed split) - Delete stale provider_rotation_test.go (references removed method)
389 lines
11 KiB
Go
389 lines
11 KiB
Go
package translator
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/tidwall/gjson"
|
|
)
|
|
|
|
// TestCase represents a single translation test case.
|
|
type TestCase struct {
|
|
Name string // Descriptive name for the test case
|
|
Input []byte // Input payload
|
|
ExpectedOutput []byte // Expected output payload (optional, for exact matching)
|
|
FromFormat Format // Source format
|
|
ToFormat Format // Target format
|
|
Model string // Model name to use for translation
|
|
Stream bool // Whether this is a streaming request
|
|
ShouldFail bool // Whether the translation should fail
|
|
SkipValidation bool // Skip schema validation of output
|
|
ValidateFields []string // Fields to validate in output (if ExpectedOutput is nil)
|
|
}
|
|
|
|
// TestResult contains the result of running a test case.
|
|
type TestResult struct {
|
|
Name string
|
|
Passed bool
|
|
Error error
|
|
Input []byte
|
|
Output []byte
|
|
Details string
|
|
}
|
|
|
|
// TestSuite represents a collection of test cases.
|
|
type TestSuite struct {
|
|
Name string
|
|
Description string
|
|
Cases []TestCase
|
|
}
|
|
|
|
// RunTestCases executes a slice of test cases against the default registry.
|
|
func RunTestCases(t *testing.T, cases []TestCase) {
|
|
t.Helper()
|
|
RunTestCasesWithRegistry(t, Default(), cases)
|
|
}
|
|
|
|
// RunTestCasesWithRegistry executes a slice of test cases against a specific registry.
|
|
func RunTestCasesWithRegistry(t *testing.T, registry *Registry, cases []TestCase) {
|
|
t.Helper()
|
|
|
|
for _, tc := range cases {
|
|
tc := tc // capture range variable
|
|
t.Run(tc.Name, func(t *testing.T) {
|
|
result := runSingleTestCase(registry, tc)
|
|
if !result.Passed {
|
|
t.Errorf("Test case %q failed: %v\nDetails: %s", tc.Name, result.Error, result.Details)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// runSingleTestCase executes a single test case and returns the result.
|
|
func runSingleTestCase(registry *Registry, tc TestCase) TestResult {
|
|
result := TestResult{
|
|
Name: tc.Name,
|
|
Input: tc.Input,
|
|
}
|
|
|
|
// Perform translation
|
|
output := registry.TranslateRequest(tc.FromFormat, tc.ToFormat, tc.Model, tc.Input, tc.Stream)
|
|
result.Output = output
|
|
|
|
// Check if translation was expected to fail
|
|
if tc.ShouldFail {
|
|
// For fail cases, we check if the output equals input (passthrough means no translator)
|
|
if string(output) == string(tc.Input) {
|
|
result.Passed = true
|
|
return result
|
|
}
|
|
result.Error = fmt.Errorf("expected translation to fail, but it succeeded")
|
|
return result
|
|
}
|
|
|
|
// Validate output schema if not skipped
|
|
if !tc.SkipValidation {
|
|
if err := ValidateSchema(tc.ToFormat, output); err != nil {
|
|
result.Error = fmt.Errorf("output validation failed: %w", err)
|
|
result.Details = fmt.Sprintf("Output: %s", truncateForTest(output, 500))
|
|
return result
|
|
}
|
|
}
|
|
|
|
// If ExpectedOutput is provided, do exact comparison
|
|
if tc.ExpectedOutput != nil {
|
|
if !jsonEqual(output, tc.ExpectedOutput) {
|
|
result.Error = fmt.Errorf("output does not match expected")
|
|
result.Details = fmt.Sprintf("Expected: %s\nGot: %s",
|
|
truncateForTest(tc.ExpectedOutput, 500),
|
|
truncateForTest(output, 500))
|
|
return result
|
|
}
|
|
}
|
|
|
|
// If ValidateFields is provided, check those specific fields
|
|
if len(tc.ValidateFields) > 0 {
|
|
outputParsed := gjson.ParseBytes(output)
|
|
for _, field := range tc.ValidateFields {
|
|
if !outputParsed.Get(field).Exists() {
|
|
result.Error = fmt.Errorf("expected field %q not found in output", field)
|
|
result.Details = fmt.Sprintf("Output: %s", truncateForTest(output, 500))
|
|
return result
|
|
}
|
|
}
|
|
}
|
|
|
|
result.Passed = true
|
|
return result
|
|
}
|
|
|
|
// RunTestSuite executes a test suite against the default registry.
|
|
func RunTestSuite(t *testing.T, suite TestSuite) {
|
|
t.Helper()
|
|
t.Run(suite.Name, func(t *testing.T) {
|
|
if suite.Description != "" {
|
|
t.Logf("Suite: %s - %s", suite.Name, suite.Description)
|
|
}
|
|
RunTestCases(t, suite.Cases)
|
|
})
|
|
}
|
|
|
|
// LoadTestCasesFromJSON loads test cases from a JSON file.
|
|
// The JSON format should be an array of test case objects.
|
|
func LoadTestCasesFromJSON(path string) ([]TestCase, error) {
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read test file: %w", err)
|
|
}
|
|
|
|
var rawCases []struct {
|
|
Name string `json:"name"`
|
|
Input json.RawMessage `json:"input"`
|
|
ExpectedOutput json.RawMessage `json:"expected_output,omitempty"`
|
|
FromFormat string `json:"from_format"`
|
|
ToFormat string `json:"to_format"`
|
|
Model string `json:"model,omitempty"`
|
|
Stream bool `json:"stream,omitempty"`
|
|
ShouldFail bool `json:"should_fail,omitempty"`
|
|
SkipValidation bool `json:"skip_validation,omitempty"`
|
|
ValidateFields []string `json:"validate_fields,omitempty"`
|
|
}
|
|
|
|
if err := json.Unmarshal(data, &rawCases); err != nil {
|
|
return nil, fmt.Errorf("failed to parse test file: %w", err)
|
|
}
|
|
|
|
cases := make([]TestCase, len(rawCases))
|
|
for i, rc := range rawCases {
|
|
cases[i] = TestCase{
|
|
Name: rc.Name,
|
|
Input: rc.Input,
|
|
ExpectedOutput: rc.ExpectedOutput,
|
|
FromFormat: Format(rc.FromFormat),
|
|
ToFormat: Format(rc.ToFormat),
|
|
Model: rc.Model,
|
|
Stream: rc.Stream,
|
|
ShouldFail: rc.ShouldFail,
|
|
SkipValidation: rc.SkipValidation,
|
|
ValidateFields: rc.ValidateFields,
|
|
}
|
|
}
|
|
|
|
return cases, nil
|
|
}
|
|
|
|
// LoadTestSuiteFromJSON loads a test suite from a JSON file.
|
|
func LoadTestSuiteFromJSON(path string) (*TestSuite, error) {
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read test suite file: %w", err)
|
|
}
|
|
|
|
var rawSuite struct {
|
|
Name string `json:"name"`
|
|
Description string `json:"description,omitempty"`
|
|
Cases json.RawMessage `json:"cases"`
|
|
}
|
|
|
|
if err := json.Unmarshal(data, &rawSuite); err != nil {
|
|
return nil, fmt.Errorf("failed to parse test suite file: %w", err)
|
|
}
|
|
|
|
cases, err := parseRawCases(rawSuite.Cases)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &TestSuite{
|
|
Name: rawSuite.Name,
|
|
Description: rawSuite.Description,
|
|
Cases: cases,
|
|
}, nil
|
|
}
|
|
|
|
// LoadTestCasesFromDir loads all JSON test case files from a directory.
|
|
func LoadTestCasesFromDir(dir string) ([]TestCase, error) {
|
|
var allCases []TestCase
|
|
|
|
entries, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read test directory: %w", err)
|
|
}
|
|
|
|
for _, entry := range entries {
|
|
if entry.IsDir() {
|
|
continue
|
|
}
|
|
if filepath.Ext(entry.Name()) != ".json" {
|
|
continue
|
|
}
|
|
|
|
path := filepath.Join(dir, entry.Name())
|
|
cases, err := LoadTestCasesFromJSON(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to load %s: %w", entry.Name(), err)
|
|
}
|
|
allCases = append(allCases, cases...)
|
|
}
|
|
|
|
return allCases, nil
|
|
}
|
|
|
|
// parseRawCases parses raw JSON cases into TestCase structs.
|
|
func parseRawCases(data json.RawMessage) ([]TestCase, error) {
|
|
var rawCases []struct {
|
|
Name string `json:"name"`
|
|
Input json.RawMessage `json:"input"`
|
|
ExpectedOutput json.RawMessage `json:"expected_output,omitempty"`
|
|
FromFormat string `json:"from_format"`
|
|
ToFormat string `json:"to_format"`
|
|
Model string `json:"model,omitempty"`
|
|
Stream bool `json:"stream,omitempty"`
|
|
ShouldFail bool `json:"should_fail,omitempty"`
|
|
SkipValidation bool `json:"skip_validation,omitempty"`
|
|
ValidateFields []string `json:"validate_fields,omitempty"`
|
|
}
|
|
|
|
if err := json.Unmarshal(data, &rawCases); err != nil {
|
|
return nil, fmt.Errorf("failed to parse cases: %w", err)
|
|
}
|
|
|
|
cases := make([]TestCase, len(rawCases))
|
|
for i, rc := range rawCases {
|
|
cases[i] = TestCase{
|
|
Name: rc.Name,
|
|
Input: rc.Input,
|
|
ExpectedOutput: rc.ExpectedOutput,
|
|
FromFormat: Format(rc.FromFormat),
|
|
ToFormat: Format(rc.ToFormat),
|
|
Model: rc.Model,
|
|
Stream: rc.Stream,
|
|
ShouldFail: rc.ShouldFail,
|
|
SkipValidation: rc.SkipValidation,
|
|
ValidateFields: rc.ValidateFields,
|
|
}
|
|
}
|
|
|
|
return cases, nil
|
|
}
|
|
|
|
// jsonEqual compares two JSON byte slices for semantic equality.
|
|
func jsonEqual(a, b []byte) bool {
|
|
var aVal, bVal interface{}
|
|
if err := json.Unmarshal(a, &aVal); err != nil {
|
|
return false
|
|
}
|
|
if err := json.Unmarshal(b, &bVal); err != nil {
|
|
return false
|
|
}
|
|
|
|
aBytes, err := json.Marshal(aVal)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
bBytes, err := json.Marshal(bVal)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
|
|
return string(aBytes) == string(bBytes)
|
|
}
|
|
|
|
// truncateForTest truncates a byte slice for test output.
|
|
func truncateForTest(data []byte, maxLen int) string {
|
|
if len(data) <= maxLen {
|
|
return string(data)
|
|
}
|
|
return string(data[:maxLen]) + "...[truncated]"
|
|
}
|
|
|
|
// AssertTranslation is a helper for simple translation assertions in tests.
|
|
func AssertTranslation(t *testing.T, from, to Format, input []byte, validateFields ...string) {
|
|
t.Helper()
|
|
|
|
output := TranslateRequest(from, to, "", input, false)
|
|
if output == nil {
|
|
t.Fatalf("Translation %s -> %s returned nil", from, to)
|
|
}
|
|
|
|
if err := ValidateSchema(to, output); err != nil {
|
|
t.Errorf("Output validation failed: %v\nOutput: %s", err, truncateForTest(output, 500))
|
|
}
|
|
|
|
if len(validateFields) > 0 {
|
|
parsed := gjson.ParseBytes(output)
|
|
for _, field := range validateFields {
|
|
if !parsed.Get(field).Exists() {
|
|
t.Errorf("Expected field %q not found in output", field)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// AssertRoundtrip is a helper for roundtrip translation assertions in tests.
|
|
func AssertRoundtrip(t *testing.T, format Format, input []byte) {
|
|
t.Helper()
|
|
|
|
preserved, err := TestRoundtrip(format, input)
|
|
if err != nil {
|
|
t.Fatalf("Roundtrip test failed: %v", err)
|
|
}
|
|
if !preserved {
|
|
result, _ := TestRoundtripDetailed(format, FormatOpenAI, input)
|
|
if result != nil {
|
|
t.Errorf("Roundtrip did not preserve data: %v", result.Differences)
|
|
} else {
|
|
t.Error("Roundtrip did not preserve data")
|
|
}
|
|
}
|
|
}
|
|
|
|
// NewTestCase creates a new TestCase with common defaults.
|
|
func NewTestCase(name string, from, to Format, input []byte) TestCase {
|
|
return TestCase{
|
|
Name: name,
|
|
Input: input,
|
|
FromFormat: from,
|
|
ToFormat: to,
|
|
}
|
|
}
|
|
|
|
// WithExpectedOutput sets the expected output for a test case.
|
|
func (tc TestCase) WithExpectedOutput(output []byte) TestCase {
|
|
tc.ExpectedOutput = output
|
|
return tc
|
|
}
|
|
|
|
// WithModel sets the model for a test case.
|
|
func (tc TestCase) WithModel(model string) TestCase {
|
|
tc.Model = model
|
|
return tc
|
|
}
|
|
|
|
// WithStream sets the stream flag for a test case.
|
|
func (tc TestCase) WithStream(stream bool) TestCase {
|
|
tc.Stream = stream
|
|
return tc
|
|
}
|
|
|
|
// WithShouldFail marks a test case as expected to fail.
|
|
func (tc TestCase) WithShouldFail() TestCase {
|
|
tc.ShouldFail = true
|
|
return tc
|
|
}
|
|
|
|
// WithSkipValidation skips output schema validation for a test case.
|
|
func (tc TestCase) WithSkipValidation() TestCase {
|
|
tc.SkipValidation = true
|
|
return tc
|
|
}
|
|
|
|
// WithValidateFields sets fields to validate in the output.
|
|
func (tc TestCase) WithValidateFields(fields ...string) TestCase {
|
|
tc.ValidateFields = fields
|
|
return tc
|
|
}
|