From 556fac16676aaf46d7fa0ba2bb62eba52f8111fa Mon Sep 17 00:00:00 2001 From: Finessse Date: Wed, 31 Dec 2025 02:08:39 +0700 Subject: [PATCH] feat(examples): add CountTokens to custom provider example Demonstrate token counting implementation in custom provider. --- examples/custom-provider/main.go | 230 ++++++++++++++++++++++++++++++- 1 file changed, 228 insertions(+), 2 deletions(-) diff --git a/examples/custom-provider/main.go b/examples/custom-provider/main.go index bdb83810..4d42a3e1 100644 --- a/examples/custom-provider/main.go +++ b/examples/custom-provider/main.go @@ -14,12 +14,14 @@ import ( "bytes" "context" "errors" + "fmt" "io" "net/http" "net/url" "os" "path/filepath" "strings" + "sync" "time" "github.com/gin-gonic/gin" @@ -31,6 +33,8 @@ import ( "github.com/router-for-me/CLIProxyAPI/v6/sdk/config" "github.com/router-for-me/CLIProxyAPI/v6/sdk/logging" sdktr "github.com/router-for-me/CLIProxyAPI/v6/sdk/translator" + "github.com/tidwall/gjson" + "github.com/tiktoken-go/tokenizer" ) const ( @@ -61,6 +65,185 @@ func init() { ) } +// ============================================================================ +// Token Counting Utilities +// ============================================================================ +// +// This section demonstrates how to implement token counting for a custom provider. +// Token counting is important for: +// - Quota management and billing estimation +// - Ensuring requests don't exceed model context limits +// - Client-side optimization and request batching +// +// There are several approaches you can use: +// +// 1. SIMPLE ESTIMATION (shown in estimateTokens): +// - Fast and requires no external dependencies beyond string operations +// - Uses character count / 4 as a rough approximation +// - Good for quick estimates, less accurate for complex text +// +// 2. TIKTOKEN (shown in countWithTiktoken): +// - Uses OpenAI's tiktoken library for accurate BPE token counting +// - Best for OpenAI-compatible models (GPT-4, GPT-3.5, etc.) +// - Very accurate but requires the tiktoken-go package +// +// 3. UPSTREAM API (not shown): +// - Call your provider's token counting API endpoint +// - Most accurate but adds latency and API costs +// - Implement this if your provider offers a /count_tokens endpoint +// +// Choose the approach that best fits your accuracy needs and performance requirements. + +var ( + // tokenizerCache stores tokenizer instances to avoid repeated initialization. + // The sync.Map is safe for concurrent access. + tokenizerCache sync.Map +) + +// getOrCreateTokenizer returns a cached tokenizer codec for the given model. +// This improves performance by avoiding repeated tokenizer initialization. +// +// Customize this function to map your provider's model names to appropriate +// tokenizer encodings. The tiktoken-go library provides encodings for: +// - O200kBase: GPT-4o, GPT-5, and newer models (default fallback) +// - Cl100kBase: GPT-4, GPT-3.5-turbo +// - P50kBase: text-davinci-003, Codex models +func getOrCreateTokenizer(model string) (tokenizer.Codec, error) { + // Check cache first + if cached, ok := tokenizerCache.Load(model); ok { + return cached.(tokenizer.Codec), nil + } + + // Select encoding based on model prefix + // Customize this mapping for your provider's model naming convention + var enc tokenizer.Codec + var err error + + sanitized := strings.ToLower(strings.TrimSpace(model)) + switch { + // Example: Map your custom models to appropriate encodings + case strings.HasPrefix(sanitized, "myprov-gpt4"): + enc, err = tokenizer.ForModel(tokenizer.GPT4) + case strings.HasPrefix(sanitized, "myprov-gpt3"): + enc, err = tokenizer.ForModel(tokenizer.GPT35Turbo) + case strings.HasPrefix(sanitized, "gpt-4o"), strings.HasPrefix(sanitized, "gpt-4.1"): + enc, err = tokenizer.ForModel(tokenizer.GPT4o) + case strings.HasPrefix(sanitized, "gpt-4"): + enc, err = tokenizer.ForModel(tokenizer.GPT4) + case strings.HasPrefix(sanitized, "gpt-3.5"): + enc, err = tokenizer.ForModel(tokenizer.GPT35Turbo) + default: + // Default to O200kBase - the most recent encoding for modern models + enc, err = tokenizer.Get(tokenizer.O200kBase) + } + + if err != nil { + return nil, err + } + + // Store in cache (use LoadOrStore to handle race conditions) + actual, _ := tokenizerCache.LoadOrStore(model, enc) + return actual.(tokenizer.Codec), nil +} + +// estimateTokens provides a simple token count estimation without external libraries. +// This is useful as a fallback when tiktoken is unavailable or for quick approximations. +// +// The algorithm uses character count / 4, which is based on the observation that +// English text averages about 4 characters per token. This is less accurate for: +// - Code (typically more tokens per character) +// - Non-English text (varies by language) +// - Text with many special characters or numbers +// +// For production use, consider tiktoken for better accuracy. +func estimateTokens(text string) int { + if len(text) == 0 { + return 0 + } + // Simple estimation: ~4 characters per token on average + // This is a reasonable approximation for English text + tokens := len(text) / 4 + if tokens == 0 && len(text) > 0 { + tokens = 1 // At least 1 token for non-empty text + } + return tokens +} + +// countWithTiktoken counts tokens using the tiktoken BPE tokenizer. +// This provides accurate token counts for OpenAI-compatible models. +// +// For custom providers with different tokenization schemes, you may need to: +// - Apply an adjustment factor (e.g., multiply by 1.1 for Claude models) +// - Use a custom tokenizer that matches your provider's implementation +// - Call your provider's native token counting API instead +func countWithTiktoken(enc tokenizer.Codec, text string) (int, error) { + if enc == nil { + return 0, fmt.Errorf("tokenizer is nil") + } + _, tokens, err := enc.Encode(text) + if err != nil { + return 0, err + } + return len(tokens), nil +} + +// extractMessagesText extracts all text content from an OpenAI-format chat request. +// This handles the standard messages array structure with role/content fields. +// +// Customize this function if your provider uses a different request format. +func extractMessagesText(payload []byte) string { + if len(payload) == 0 { + return "" + } + + root := gjson.ParseBytes(payload) + var segments []string + + // Extract from messages array (OpenAI chat format) + messages := root.Get("messages") + if messages.Exists() && messages.IsArray() { + messages.ForEach(func(_, msg gjson.Result) bool { + // Extract role for context (adds ~1 token overhead per message) + if role := msg.Get("role").String(); role != "" { + segments = append(segments, role) + } + // Extract text content + content := msg.Get("content") + if content.IsArray() { + // Handle array content (e.g., with images) + content.ForEach(func(_, part gjson.Result) bool { + if part.Get("type").String() == "text" { + if text := part.Get("text").String(); text != "" { + segments = append(segments, text) + } + } + return true + }) + } else if text := content.String(); text != "" { + segments = append(segments, text) + } + return true + }) + } + + // Extract from prompt field (completion format) + if prompt := root.Get("prompt").String(); prompt != "" { + segments = append(segments, prompt) + } + + // Extract from input field (some APIs use this) + if input := root.Get("input").String(); input != "" { + segments = append(segments, input) + } + + // Extract system prompt if present at top level + if system := root.Get("system").String(); system != "" { + segments = append(segments, system) + } + + return strings.Join(segments, "\n") +} + // MyExecutor is a minimal provider implementation for demonstration purposes. // It implements the Executor interface to handle requests to a custom AI provider. type MyExecutor struct{} @@ -137,8 +320,51 @@ func (MyExecutor) Execute(ctx context.Context, a *coreauth.Auth, req clipexec.Re return clipexec.Response{Payload: body}, nil } -func (MyExecutor) CountTokens(context.Context, *coreauth.Auth, clipexec.Request, clipexec.Options) (clipexec.Response, error) { - return clipexec.Response{}, errors.New("count tokens not implemented") +// CountTokens counts tokens in the request payload. +// This implementation demonstrates a layered approach: +// 1. Try tiktoken for accurate BPE-based counting +// 2. Fall back to simple character-based estimation if tiktoken fails +// +// The response format follows the Anthropic API convention: {"count": } +// Adjust the response format to match your provider's expected format. +// +// For production use, consider: +// - Caching token counts for identical payloads +// - Using your provider's native token counting API if available +// - Applying model-specific adjustment factors +func (MyExecutor) CountTokens(_ context.Context, _ *coreauth.Auth, req clipexec.Request, _ clipexec.Options) (clipexec.Response, error) { + var tokenCount int + + // Extract text content from the request payload + text := extractMessagesText(req.Payload) + + // If no text was extracted from structured fields, use raw payload + if text == "" && len(req.Payload) > 0 { + text = string(req.Payload) + } + + // Try tiktoken first for accurate counting + enc, err := getOrCreateTokenizer(req.Model) + if err == nil { + // Successfully got a tokenizer, use it for accurate counting + count, countErr := countWithTiktoken(enc, text) + if countErr == nil { + tokenCount = count + } else { + // Tiktoken encoding failed, fall back to estimation + tokenCount = estimateTokens(text) + } + } else { + // Could not get tokenizer, fall back to simple estimation + tokenCount = estimateTokens(text) + } + + // Return response in the expected format + // The format follows Anthropic's count_tokens response: {"count": } + // Adjust this format to match your provider's API specification + response := fmt.Sprintf(`{"count":%d}`, tokenCount) + + return clipexec.Response{Payload: []byte(response)}, nil } func (MyExecutor) Embed(context.Context, *coreauth.Auth, clipexec.Request, clipexec.Options) (clipexec.Response, error) {