middleware: model-aware prompt budget

This commit is contained in:
Finessse
2025-12-17 20:30:03 +07:00
parent efb6965ef3
commit d23af0518d
+28 -2
View File
@@ -10,6 +10,7 @@ import (
"sync"
"github.com/gin-gonic/gin"
"github.com/router-for-me/CLIProxyAPI/v6/internal/registry"
"github.com/tidwall/gjson"
"github.com/tidwall/sjson"
)
@@ -70,7 +71,7 @@ func CodexPromptBudgetMiddleware() gin.HandlerFunc {
ua := strings.ToLower(req.Header.Get("User-Agent"))
isStainless := req.Header.Get("X-Stainless-Lang") != "" || req.Header.Get("X-Stainless-Package-Version") != ""
isAgenticCLI := strings.Contains(ua, "openai codex") || strings.Contains(ua, "factory-cli") || strings.Contains(ua, "warp") || isStainless
isAgenticCLI := strings.Contains(ua, "openai codex") || strings.Contains(ua, "factory-cli") || strings.Contains(ua, "warp") || strings.Contains(ua, "droid") || isStainless
if !isAgenticCLI {
c.Next()
return
@@ -110,7 +111,7 @@ func CodexPromptBudgetMiddleware() gin.HandlerFunc {
}
originalLen := len(body)
maxBytes := agenticMaxBodyBytes()
maxBytes := agenticMaxBodyBytesForModel(body)
if originalLen <= maxBytes {
req.Body = io.NopCloser(bytes.NewReader(body))
req.ContentLength = int64(originalLen)
@@ -143,6 +144,31 @@ func CodexPromptBudgetMiddleware() gin.HandlerFunc {
}
}
func agenticMaxBodyBytesForModel(body []byte) int {
maxBytes := agenticMaxBodyBytes()
model := gjson.GetBytes(body, "model").String()
if model == "" {
return maxBytes
}
info := registry.GetGlobalRegistry().GetModelInfo(model)
if info == nil || info.ContextLength <= 0 {
return maxBytes
}
// Heuristic: ~4 bytes per token (UTF-8 text + JSON overhead). We only scale DOWN
// from the global default to avoid upstream "prompt too long" for small-context models.
estimated := info.ContextLength * 4
const minBytes = 32 * 1024
if estimated < minBytes {
estimated = minBytes
}
if estimated < maxBytes {
return estimated
}
return maxBytes
}
// trimOpenAIChatCompletions trims an OpenAI Chat Completions payload by shortening the messages array.
func trimOpenAIChatCompletions(body []byte, maxBytes int) []byte {
root := gjson.ParseBytes(body)