refactor(middleware): split codex_prompt_budget.go into logical modules

Split the 2,762 line codex_prompt_budget.go into focused modules:
- codex_prompt_budget.go (1,272 lines): Main middleware, scaffold injection
- codex_budget_types.go (498 lines): Constants, package vars, config functions
- codex_budget_calculator.go (944 lines): Token analysis, trim functions

Co-Authored-By: Claude (gemini-claude-opus-4-5-thinking) <noreply@anthropic.com>
This commit is contained in:
RCD
2026-02-01 12:28:00 +07:00
co-authored by Claude
parent 75ca091a73
commit 7cc4e9d8f5
3 changed files with 1503 additions and 1551 deletions
@@ -0,0 +1,944 @@
package middleware
import (
"strconv"
"strings"
"github.com/router-for-me/CLIProxyAPI/v6/internal/memory"
"github.com/router-for-me/CLIProxyAPI/v6/internal/registry"
"github.com/router-for-me/CLIProxyAPI/v6/internal/runtime/executor"
"github.com/tidwall/gjson"
"github.com/tidwall/sjson"
)
// tokenAwareCompressionResult contains the result of token-aware analysis
type tokenAwareCompressionResult struct {
ShouldTrim bool // Whether trimming is needed
CurrentTokens int64 // Estimated current token count
ContextWindow int // Model's context window
TargetTokens int64 // Target token count after trimming
TargetMaxBytes int // Approximate bytes to achieve target tokens
Model string // The model name
}
// analyzeTokenBudget checks if the request exceeds the token threshold and calculates trimming targets.
func analyzeTokenBudget(body []byte) *tokenAwareCompressionResult {
result := &tokenAwareCompressionResult{
ShouldTrim: false,
TargetMaxBytes: agenticMaxBodyBytes(),
}
if !agenticTokenAwareEnabled() {
return result
}
model := gjson.GetBytes(body, "model").String()
if model == "" {
return result
}
result.Model = model
contextWindow := getModelContextWindow(model)
result.ContextWindow = contextWindow
currentTokens, err := executor.EstimateRequestTokens(model, body)
if err != nil {
currentTokens = int64(len(body) / 4)
}
result.CurrentTokens = currentTokens
reserveTokens := agenticReserveTokens()
availableContext := contextWindow - reserveTokens
if availableContext < 1000 {
availableContext = contextWindow / 2
}
threshold := agenticCompressionThreshold()
maxInputTokens := int64(float64(availableContext) * threshold)
if currentTokens <= maxInputTokens {
result.ShouldTrim = false
return result
}
targetRatio := threshold * 0.9
targetTokens := int64(float64(availableContext) * targetRatio)
result.TargetTokens = targetTokens
result.ShouldTrim = true
targetBytes := int(targetTokens * 4)
minBytes := 32 * 1024
if targetBytes < minBytes {
targetBytes = minBytes
}
maxBytes := agenticMaxBodyBytes()
if targetBytes > maxBytes {
targetBytes = maxBytes
}
result.TargetMaxBytes = targetBytes
return result
}
// getTokenAwareMaxBytes returns the maximum body size based on token analysis.
func getTokenAwareMaxBytes(body []byte) int {
if !agenticTokenAwareEnabled() {
return agenticMaxBodyBytesForModel(body)
}
result := analyzeTokenBudget(body)
if result.ShouldTrim {
return result.TargetMaxBytes
}
return agenticMaxBodyBytesForModel(body)
}
func agenticMaxBodyBytesForModel(body []byte) int {
maxBytes := agenticMaxBodyBytes()
model := gjson.GetBytes(body, "model").String()
if model == "" {
return maxBytes
}
info := registry.GetGlobalRegistry().GetModelInfo(model, "")
if info == nil || info.ContextLength <= 0 {
return maxBytes
}
estimated := info.ContextLength * 4
const minBytes = 32 * 1024
if estimated < minBytes {
estimated = minBytes
}
if estimated < maxBytes {
return estimated
}
return maxBytes
}
// trimWithMemoryResult holds the result of trimming with memory extraction.
type trimWithMemoryResult struct {
Body []byte
Query string
Dropped []memory.Event
Shape string // "chat", "responses", or "claude"
}
// trimOpenAIChatCompletions trims an OpenAI Chat Completions payload by shortening the messages array.
func trimOpenAIChatCompletions(body []byte, maxBytes int, mustKeepTools bool) []byte {
root := gjson.ParseBytes(body)
msgs := root.Get("messages")
if !msgs.IsArray() {
return body
}
arr := msgs.Array()
if len(arr) == 0 {
return body
}
firstSystem := gjson.Result{}
for i := 0; i < len(arr); i++ {
if strings.EqualFold(arr[i].Get("role").String(), "system") {
firstSystem = arr[i]
break
}
}
isToolResultMsg := func(m gjson.Result) bool {
role := strings.ToLower(strings.TrimSpace(m.Get("role").String()))
return role == "tool" || role == "function"
}
assistantHasToolCall := func(m gjson.Result) bool {
if !strings.EqualFold(m.Get("role").String(), "assistant") {
return false
}
if tc := m.Get("tool_calls"); tc.Exists() && tc.IsArray() && len(tc.Array()) > 0 {
return true
}
if fc := m.Get("function_call"); fc.Exists() {
return true
}
return false
}
keep := 20
perTextLimit := 20_000
dropTools := false
minKeep := agenticMinKeepMessages()
for keep >= minKeep {
outBody := body
if dropTools && !mustKeepTools {
outBody, _ = sjson.DeleteBytes(outBody, "tools")
outBody, _ = sjson.SetBytes(outBody, "tool_choice", "none")
}
newMsgs := make([]string, 0, keep+1)
if firstSystem.Exists() {
newMsgs = append(newMsgs, truncateMessageContent(firstSystem.Raw, perTextLimit))
}
required := make(map[int]struct{}, 8)
tailKept := 0
for i := len(arr) - 1; i >= 0 && (tailKept < keep || len(required) > 0); i-- {
if strings.EqualFold(arr[i].Get("role").String(), "system") {
continue
}
_, req := required[i]
if !req && tailKept >= keep {
continue
}
newMsgs = append(newMsgs, truncateMessageContent(arr[i].Raw, perTextLimit))
if !req {
tailKept++
} else {
delete(required, i)
}
if isToolResultMsg(arr[i]) {
prev := i - 1
if prev >= 0 && assistantHasToolCall(arr[prev]) {
required[prev] = struct{}{}
}
}
}
if firstSystem.Exists() {
reverseStrings(newMsgs[1:])
} else {
reverseStrings(newMsgs)
}
out := setJSONArrayBytes(outBody, "messages", newMsgs)
if len(out) <= maxBytes {
return out
}
keep = keep / 2
if perTextLimit > 5_000 {
perTextLimit = perTextLimit / 2
}
dropTools = true
}
return body
}
// trimOpenAIResponses trims an OpenAI Responses payload by shortening the input array.
func trimOpenAIResponses(body []byte, maxBytes int, mustKeepTools bool) []byte {
root := gjson.ParseBytes(body)
input := root.Get("input")
if !input.Exists() || !input.IsArray() {
return body
}
arr := input.Array()
if len(arr) == 0 {
return body
}
keep := 30
perTextLimit := 20_000
dropTools := false
minKeep := agenticMinKeepMessages()
for keep >= minKeep {
outBody := body
if dropTools && !mustKeepTools {
outBody, _ = sjson.DeleteBytes(outBody, "tools")
outBody, _ = sjson.SetBytes(outBody, "tool_choice", "none")
}
if inst := root.Get("instructions"); inst.Exists() && inst.Type == gjson.String {
s := inst.String()
instructionsLimit := perTextLimit
if instructionsLimit > 2048 {
instructionsLimit = 2048
}
if len(s) > instructionsLimit {
outBody, _ = sjson.SetBytes(outBody, "instructions", s[:instructionsLimit]+"\n...[truncated]...")
}
}
callByID := make(map[string]string, 16)
for i := 0; i < len(arr); i++ {
item := arr[i]
t := item.Get("type").String()
if t == "" && item.Get("role").String() != "" {
t = "message"
}
if t != "function_call" {
continue
}
callID := item.Get("call_id").String()
if callID == "" {
continue
}
if _, ok := callByID[callID]; !ok {
callByID[callID] = item.Raw
}
}
needCall := make(map[string]struct{}, 8)
newItems := make([]string, 0, keep+8)
kept := 0
for i := len(arr) - 1; i >= 0 && kept < keep; i-- {
item := arr[i]
t := item.Get("type").String()
if t == "" && item.Get("role").String() != "" {
t = "message"
}
if dropTools && !mustKeepTools && (t == "function_call" || t == "function_call_output") {
continue
}
if t == "function_call_output" {
callID := item.Get("call_id").String()
if callID != "" {
needCall[callID] = struct{}{}
}
}
if t == "function_call" {
callID := item.Get("call_id").String()
if callID != "" {
delete(needCall, callID)
}
}
newItems = append(newItems, truncateMessageContent(item.Raw, perTextLimit))
kept++
}
reverseStrings(newItems)
if !dropTools && len(needCall) > 0 {
prefix := make([]string, 0, len(needCall))
for callID := range needCall {
if raw, ok := callByID[callID]; ok {
prefix = append(prefix, raw)
}
}
if len(prefix) > 0 {
ordered := make([]string, 0, len(prefix))
for i := 0; i < len(arr); i++ {
item := arr[i]
if item.Get("type").String() != "function_call" {
continue
}
callID := item.Get("call_id").String()
if callID == "" {
continue
}
if _, ok := needCall[callID]; ok {
if raw, ok2 := callByID[callID]; ok2 {
ordered = append(ordered, raw)
}
}
}
newItems = append(ordered, newItems...)
}
}
out := setJSONArrayBytes(outBody, "input", newItems)
if len(out) <= maxBytes {
return out
}
keep = keep / 2
if perTextLimit > 5_000 {
perTextLimit = perTextLimit / 2
}
if !mustKeepTools {
dropTools = true
}
}
return body
}
func trimOpenAIChatCompletionsWithMemory(body []byte, maxBytes int, mustKeepTools bool) *trimWithMemoryResult {
root := gjson.ParseBytes(body)
msgs := root.Get("messages")
if !msgs.IsArray() {
return &trimWithMemoryResult{Body: body, Shape: "chat"}
}
arr := msgs.Array()
if len(arr) == 0 {
return &trimWithMemoryResult{Body: body, Shape: "chat"}
}
firstSystem := gjson.Result{}
firstSystemIndex := -1
for i := 0; i < len(arr); i++ {
if strings.EqualFold(arr[i].Get("role").String(), "system") {
firstSystem = arr[i]
firstSystemIndex = i
break
}
}
query := extractLastUserTextFromChat(arr)
minKeep := agenticMinKeepMessages()
keep := 20
perTextLimit := 20_000
dropTools := false
for keep >= minKeep {
outBody := body
if dropTools && !mustKeepTools {
outBody, _ = sjson.DeleteBytes(outBody, "tools")
outBody, _ = sjson.SetBytes(outBody, "tool_choice", "none")
}
newMsgs := make([]string, 0, keep+1)
keptIdx := make(map[int]struct{}, keep+2)
if firstSystem.Exists() {
newMsgs = append(newMsgs, truncateMessageContent(firstSystem.Raw, perTextLimit))
keptIdx[firstSystemIndex] = struct{}{}
}
isToolResultMsg := func(m gjson.Result) bool {
role := strings.ToLower(strings.TrimSpace(m.Get("role").String()))
return role == "tool" || role == "function"
}
assistantHasToolCall := func(m gjson.Result) bool {
if !strings.EqualFold(m.Get("role").String(), "assistant") {
return false
}
if tc := m.Get("tool_calls"); tc.Exists() && tc.IsArray() && len(tc.Array()) > 0 {
return true
}
if fc := m.Get("function_call"); fc.Exists() {
return true
}
return false
}
required := make(map[int]struct{}, 8)
tailKept := 0
for i := len(arr) - 1; i >= 0 && (tailKept < keep || len(required) > 0); i-- {
if strings.EqualFold(arr[i].Get("role").String(), "system") {
continue
}
_, req := required[i]
if !req && tailKept >= keep {
continue
}
newMsgs = append(newMsgs, truncateMessageContent(arr[i].Raw, perTextLimit))
keptIdx[i] = struct{}{}
if !req {
tailKept++
} else {
delete(required, i)
}
if isToolResultMsg(arr[i]) {
prev := i - 1
if prev >= 0 && assistantHasToolCall(arr[prev]) {
required[prev] = struct{}{}
}
}
}
if firstSystem.Exists() {
reverseStrings(newMsgs[1:])
} else {
reverseStrings(newMsgs)
}
out := setJSONArrayBytes(outBody, "messages", newMsgs)
if len(out) <= maxBytes {
dropped := collectDroppedChat(arr, keptIdx)
return &trimWithMemoryResult{Body: out, Query: query, Dropped: dropped, Shape: "chat"}
}
keep = keep / 2
if perTextLimit > 5_000 {
perTextLimit = perTextLimit / 2
}
dropTools = true
}
return &trimWithMemoryResult{Body: body, Query: query, Shape: "chat"}
}
func collectDroppedChat(arr []gjson.Result, kept map[int]struct{}) []memory.Event {
out := make([]memory.Event, 0, 32)
for i := 0; i < len(arr); i++ {
if _, ok := kept[i]; ok {
continue
}
role := arr[i].Get("role").String()
txt := extractTextFromChatMessage(arr[i])
if strings.TrimSpace(txt) == "" {
continue
}
out = append(out, memory.Event{Kind: "dropped_chat", Role: role, Text: txt})
}
return out
}
func extractLastUserTextFromChat(arr []gjson.Result) string {
for i := len(arr) - 1; i >= 0; i-- {
if !strings.EqualFold(arr[i].Get("role").String(), "user") {
continue
}
txt := extractTextFromChatMessage(arr[i])
if strings.TrimSpace(txt) != "" {
return txt
}
}
return ""
}
func extractTextFromChatMessage(msg gjson.Result) string {
content := msg.Get("content")
switch {
case content.Type == gjson.String:
return content.String()
case content.IsArray():
var b strings.Builder
for _, it := range content.Array() {
partType := it.Get("type").String()
if partType == "thinking" || partType == "reasoning" {
continue
}
if t := it.Get("text"); t.Exists() && t.Type == gjson.String {
if b.Len() > 0 {
b.WriteString("\n")
}
b.WriteString(t.String())
}
}
return b.String()
default:
return ""
}
}
func trimOpenAIResponsesWithMemory(body []byte, maxBytes int, mustKeepTools bool) *trimWithMemoryResult {
root := gjson.ParseBytes(body)
input := root.Get("input")
if !input.Exists() || !input.IsArray() {
return &trimWithMemoryResult{Body: body, Shape: "responses"}
}
arr := input.Array()
if len(arr) == 0 {
return &trimWithMemoryResult{Body: body, Shape: "responses"}
}
query := extractLastUserTextFromResponses(arr)
minKeep := agenticMinKeepMessages()
keep := 30
perTextLimit := 20_000
dropTools := false
for keep >= minKeep {
outBody := body
if dropTools && !mustKeepTools {
outBody, _ = sjson.DeleteBytes(outBody, "tools")
outBody, _ = sjson.SetBytes(outBody, "tool_choice", "none")
}
if inst := root.Get("instructions"); inst.Exists() && inst.Type == gjson.String {
s := inst.String()
instructionsLimit := perTextLimit
if instructionsLimit > 2048 {
instructionsLimit = 2048
}
if len(s) > instructionsLimit {
outBody, _ = sjson.SetBytes(outBody, "instructions", s[:instructionsLimit]+"\n...[truncated]...")
}
}
callByID := make(map[string]gjson.Result, 16)
for i := 0; i < len(arr); i++ {
item := arr[i]
t := item.Get("type").String()
if t == "" && item.Get("role").String() != "" {
t = "message"
}
if t != "function_call" {
continue
}
callID := item.Get("call_id").String()
if callID == "" {
continue
}
if _, ok := callByID[callID]; !ok {
callByID[callID] = item
}
}
needCall := make(map[string]struct{}, 8)
newItems := make([]string, 0, keep+8)
keptIdx := make(map[int]struct{}, keep+16)
kept := 0
for i := len(arr) - 1; i >= 0 && kept < keep; i-- {
item := arr[i]
t := item.Get("type").String()
if t == "" && item.Get("role").String() != "" {
t = "message"
}
if dropTools && !mustKeepTools && (t == "function_call" || t == "function_call_output") {
continue
}
if t == "function_call_output" {
callID := item.Get("call_id").String()
if callID != "" {
needCall[callID] = struct{}{}
}
}
if t == "function_call" {
callID := item.Get("call_id").String()
if callID != "" {
delete(needCall, callID)
}
}
newItems = append(newItems, truncateMessageContent(item.Raw, perTextLimit))
keptIdx[i] = struct{}{}
kept++
}
reverseStrings(newItems)
if !dropTools && len(needCall) > 0 {
ordered := make([]string, 0, len(needCall))
for i := 0; i < len(arr); i++ {
item := arr[i]
if item.Get("type").String() != "function_call" {
continue
}
callID := item.Get("call_id").String()
if callID == "" {
continue
}
if _, ok := needCall[callID]; ok {
if call, ok2 := callByID[callID]; ok2 {
ordered = append(ordered, call.Raw)
keptIdx[i] = struct{}{}
}
}
}
if len(ordered) > 0 {
newItems = append(ordered, newItems...)
}
}
out := setJSONArrayBytes(outBody, "input", newItems)
if len(out) <= maxBytes {
dropped := collectDroppedResponses(arr, keptIdx)
return &trimWithMemoryResult{Body: out, Query: query, Dropped: dropped, Shape: "responses"}
}
keep = keep / 2
if perTextLimit > 5_000 {
perTextLimit = perTextLimit / 2
}
if !mustKeepTools {
dropTools = true
}
}
return &trimWithMemoryResult{Body: body, Query: query, Shape: "responses"}
}
func collectDroppedResponses(arr []gjson.Result, kept map[int]struct{}) []memory.Event {
out := make([]memory.Event, 0, 64)
for i := 0; i < len(arr); i++ {
if _, ok := kept[i]; ok {
continue
}
item := arr[i]
t := item.Get("type").String()
role := item.Get("role").String()
txt := extractTextFromResponsesItem(item)
if strings.TrimSpace(txt) == "" {
continue
}
out = append(out, memory.Event{Kind: "dropped_responses", Type: t, Role: role, Text: txt})
}
return out
}
func extractLastUserTextFromResponses(arr []gjson.Result) string {
for i := len(arr) - 1; i >= 0; i-- {
item := arr[i]
role := item.Get("role").String()
if !strings.EqualFold(role, "user") {
continue
}
txt := extractTextFromResponsesItem(item)
if strings.TrimSpace(txt) != "" {
return txt
}
}
return ""
}
func extractTextFromResponsesItem(item gjson.Result) string {
content := item.Get("content")
if content.IsArray() {
var b strings.Builder
for _, part := range content.Array() {
partType := part.Get("type").String()
if partType == "thinking" || partType == "reasoning" {
continue
}
text := part.Get("text")
if !text.Exists() || text.Type != gjson.String {
continue
}
if b.Len() > 0 {
b.WriteString("\n")
}
b.WriteString(text.String())
}
return b.String()
}
if content.Type == gjson.String {
return content.String()
}
if t := item.Get("text"); t.Exists() && t.Type == gjson.String {
return t.String()
}
if out := item.Get("output"); out.Exists() && out.Type == gjson.String {
return out.String()
}
return ""
}
func trimClaudeMessagesWithMemory(body []byte, maxBytes int, mustKeepTools bool) *trimWithMemoryResult {
root := gjson.ParseBytes(body)
msgs := root.Get("messages")
if !msgs.IsArray() {
return &trimWithMemoryResult{Body: body, Shape: "claude"}
}
arr := msgs.Array()
if len(arr) == 0 {
return &trimWithMemoryResult{Body: body, Shape: "claude"}
}
query := extractLastUserTextFromClaude(arr)
minKeep := agenticMinKeepMessages()
keep := 20
perTextLimit := 20_000
dropTools := false
for keep >= minKeep {
outBody := body
if dropTools && !mustKeepTools {
outBody, _ = sjson.DeleteBytes(outBody, "tools")
outBody, _ = sjson.SetBytes(outBody, "tool_choice", map[string]any{"type": "none"})
}
newMsgs := make([]string, 0, keep+1)
keptIdx := make(map[int]struct{}, keep+2)
isToolResultMsg := func(m gjson.Result) bool {
content := m.Get("content")
if !content.IsArray() {
return false
}
for _, part := range content.Array() {
if part.Get("type").String() == "tool_result" {
return true
}
}
return false
}
assistantHasToolUse := func(m gjson.Result) bool {
if !strings.EqualFold(m.Get("role").String(), "assistant") {
return false
}
content := m.Get("content")
if !content.IsArray() {
return false
}
for _, part := range content.Array() {
if part.Get("type").String() == "tool_use" {
return true
}
}
return false
}
required := make(map[int]struct{}, 8)
tailKept := 0
for i := len(arr) - 1; i >= 0 && (tailKept < keep || len(required) > 0); i-- {
_, req := required[i]
if !req && tailKept >= keep {
continue
}
newMsgs = append(newMsgs, truncateClaudeMessageContent(arr[i].Raw, perTextLimit))
keptIdx[i] = struct{}{}
if !req {
tailKept++
} else {
delete(required, i)
}
if isToolResultMsg(arr[i]) {
prev := i - 1
if prev >= 0 && assistantHasToolUse(arr[prev]) {
required[prev] = struct{}{}
}
}
}
reverseStrings(newMsgs)
out := setJSONArrayBytes(outBody, "messages", newMsgs)
if len(out) <= maxBytes {
dropped := collectDroppedClaude(arr, keptIdx)
return &trimWithMemoryResult{Body: out, Query: query, Dropped: dropped, Shape: "claude"}
}
keep = keep / 2
if perTextLimit > 5_000 {
perTextLimit = perTextLimit / 2
}
dropTools = true
}
return &trimWithMemoryResult{Body: body, Query: query, Shape: "claude"}
}
func collectDroppedClaude(arr []gjson.Result, kept map[int]struct{}) []memory.Event {
out := make([]memory.Event, 0, 32)
for i := 0; i < len(arr); i++ {
if _, ok := kept[i]; ok {
continue
}
role := arr[i].Get("role").String()
txt := extractTextFromClaudeMessage(arr[i])
if strings.TrimSpace(txt) == "" {
continue
}
out = append(out, memory.Event{Kind: "dropped_claude", Role: role, Text: txt})
}
return out
}
func extractLastUserTextFromClaude(arr []gjson.Result) string {
for i := len(arr) - 1; i >= 0; i-- {
if !strings.EqualFold(arr[i].Get("role").String(), "user") {
continue
}
txt := extractTextFromClaudeMessage(arr[i])
if strings.TrimSpace(txt) != "" {
return txt
}
}
return ""
}
func extractTextFromClaudeMessage(msg gjson.Result) string {
content := msg.Get("content")
switch {
case content.Type == gjson.String:
return content.String()
case content.IsArray():
var b strings.Builder
for _, part := range content.Array() {
partType := part.Get("type").String()
if partType == "text" {
if t := part.Get("text"); t.Exists() && t.Type == gjson.String {
if b.Len() > 0 {
b.WriteString("\n")
}
b.WriteString(t.String())
}
}
}
return b.String()
default:
return ""
}
}
func truncateMessageContent(msgRaw string, maxTextChars int) string {
msg := msgRaw
if maxTextChars <= 0 {
return msg
}
content := gjson.Get(msg, "content")
switch {
case content.Type == gjson.String:
s := content.String()
if len(s) > maxTextChars {
s = s[:maxTextChars] + "\n...[truncated]..."
msg, _ = sjson.Set(msg, "content", s)
}
return msg
case content.IsArray():
items := content.Array()
for i := 0; i < len(items); i++ {
text := items[i].Get("text")
if !text.Exists() || text.Type != gjson.String {
continue
}
s := text.String()
if len(s) > maxTextChars {
s = s[:maxTextChars] + "\n...[truncated]..."
msg, _ = sjson.Set(msg, "content."+strconv.Itoa(i)+".text", s)
}
}
return msg
default:
return msg
}
}
func truncateClaudeMessageContent(msgRaw string, maxTextChars int) string {
msg := msgRaw
if maxTextChars <= 0 {
return msg
}
content := gjson.Get(msg, "content")
switch {
case content.Type == gjson.String:
s := content.String()
if len(s) > maxTextChars {
s = s[:maxTextChars] + "\n...[truncated]..."
msg, _ = sjson.Set(msg, "content", s)
}
return msg
case content.IsArray():
items := content.Array()
for i := 0; i < len(items); i++ {
partType := items[i].Get("type").String()
if partType != "text" {
continue
}
text := items[i].Get("text")
if !text.Exists() || text.Type != gjson.String {
continue
}
s := text.String()
if len(s) > maxTextChars {
s = s[:maxTextChars] + "\n...[truncated]..."
msg, _ = sjson.Set(msg, "content."+strconv.Itoa(i)+".text", s)
}
}
return msg
default:
return msg
}
}
func setJSONArrayBytes(body []byte, key string, rawItems []string) []byte {
out := body
out, _ = sjson.SetRawBytes(out, key, []byte("[]"))
for i := range rawItems {
out, _ = sjson.SetRawBytes(out, key+".-1", []byte(rawItems[i]))
}
return out
}
func reverseStrings(items []string) {
for i, j := 0, len(items)-1; i < j; i, j = i+1, j-1 {
items[i], items[j] = items[j], items[i]
}
}
@@ -0,0 +1,498 @@
package middleware
import (
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/router-for-me/CLIProxyAPI/v6/internal/embeddings"
"github.com/router-for-me/CLIProxyAPI/v6/internal/memory"
"github.com/router-for-me/CLIProxyAPI/v6/internal/registry"
"github.com/router-for-me/CLIProxyAPI/v6/internal/util"
)
const (
// codexHardReadLimit is a safety ceiling to avoid unbounded memory reads.
codexHardReadLimit = 10 * 1024 * 1024
// codexMaxBodyBytesDefault is a best-effort budget to keep agentic CLI requests under common model limits.
// It is intentionally conservative to avoid upstream "prompt too long" failures.
codexMaxBodyBytesDefault = 200 * 1024
specModePrompt = `SPEC MODE (do not code yet).
1) Produce a complete, reviewable specification (requirements, acceptance criteria, architecture, and file-level plan).
2) Wait for explicit approval before editing code.
3) If clarification is needed, ask now before writing code.`
)
var (
codexMaxBodyBytesOnce sync.Once
codexMaxBodyBytes int
memOnce sync.Once
memStore memory.Store
embedOnce sync.Once
embedClient *embeddings.OllamaClient
embedQueueOnce sync.Once
embedQueue *semanticEmbedQueue
pruneMu sync.Mutex
lastPrune time.Time
limiterMu sync.Mutex
memoryLimiters = map[string]*rateLimiter{}
semanticLimiters = map[string]*rateLimiter{}
)
type rateLimiter struct {
mu sync.Mutex
tokens float64
last time.Time
}
func (r *rateLimiter) Allow(ratePerSec float64, burst float64) bool {
if ratePerSec <= 0 {
return true
}
if burst <= 0 {
burst = ratePerSec
}
r.mu.Lock()
defer r.mu.Unlock()
now := time.Now()
if r.last.IsZero() {
r.last = now
r.tokens = burst
}
elapsed := now.Sub(r.last).Seconds()
if elapsed > 0 {
r.tokens += elapsed * ratePerSec
if r.tokens > burst {
r.tokens = burst
}
r.last = now
}
if r.tokens >= 1 {
r.tokens -= 1
return true
}
return false
}
func agenticMaxBodyBytes() int {
codexMaxBodyBytesOnce.Do(func() {
codexMaxBodyBytes = codexMaxBodyBytesDefault
// Optional override (bytes). Useful when running behind very large-context models.
if v := strings.TrimSpace(os.Getenv("CLIPROXY_AGENTIC_MAX_BODY_BYTES")); v != "" {
if n, err := strconv.Atoi(v); err == nil {
if n < 32*1024 {
n = 32 * 1024
}
if n > 2*1024*1024 {
n = 2 * 1024 * 1024
}
codexMaxBodyBytes = n
}
}
})
return codexMaxBodyBytes
}
func agenticMemoryStore() memory.Store {
memOnce.Do(func() {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_ENABLED")); v != "" {
if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") {
memStore = nil
return
}
}
base := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_DIR"))
if base == "" {
if w := util.WritablePath(); w != "" {
base = filepath.Join(w, ".proxypilot", "memory")
} else {
base = filepath.Join(".proxypilot", "memory")
}
}
memStore = memory.NewFileStore(base)
if agenticLLMSummaryEnabled() {
if fs, ok := memStore.(*memory.FileStore); ok {
config := memory.DefaultSummarizerConfig()
summarizer := memory.NewSummarizer(config, memory.NewNoOpSummarizerExecutor())
fs.SetSummarizer(summarizer)
}
}
})
return memStore
}
// SetSummarizerExecutor configures the LLM executor for context summarization.
func SetSummarizerExecutor(executor memory.SummarizerExecutor) {
store := agenticMemoryStore()
if store == nil {
return
}
fs, ok := store.(*memory.FileStore)
if !ok || fs == nil {
return
}
summarizer := fs.GetSummarizer()
if summarizer == nil {
config := memory.DefaultSummarizerConfig()
summarizer = memory.NewSummarizer(config, executor)
fs.SetSummarizer(summarizer)
} else {
config := memory.DefaultSummarizerConfig()
fs.SetSummarizer(memory.NewSummarizer(config, executor))
}
}
// InitSummarizerWithAuthManager configures LLM-based summarization using the core auth manager.
func InitSummarizerWithAuthManager(manager memory.CoreManagerExecutor, providers []string) {
if manager == nil {
return
}
adapter := memory.NewManagerAuthAdapter(manager)
baseExecutor := memory.NewPipelineSummarizerExecutor(adapter, providers)
executor := memory.NewSummaryModelFallbackExecutor(baseExecutor)
SetSummarizerExecutor(executor)
}
// GetSummaryModel returns the configured summary model, defaulting to gemini-3-flash.
func GetSummaryModel() string {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SUMMARY_MODEL")); v != "" {
return v
}
return memory.DefaultSummaryModel
}
func agenticSemanticEnabled() bool {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_ENABLED")); v != "" {
if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") {
return false
}
}
return true
}
func agenticSemanticModel() string {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MODEL")); v != "" {
return v
}
return "embeddinggemma"
}
func agenticSemanticBaseURL() string {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_BASE_URL")); v != "" {
return v
}
return "http://127.0.0.1:11434"
}
func agenticSemanticMaxSnips() int {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_SNIPS")); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
return n
}
}
return 4
}
func agenticSemanticMaxChars() int {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_CHARS")); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
return n
}
}
return 3000
}
func agenticSemanticClient() *embeddings.OllamaClient {
embedOnce.Do(func() {
embedClient = &embeddings.OllamaClient{
BaseURL: agenticSemanticBaseURL(),
Model: agenticSemanticModel(),
Client: &http.Client{Timeout: 8 * time.Second},
}
})
return embedClient
}
type semanticEmbedTask struct {
namespace string
session string
texts []string
roles []string
source string
}
type semanticEmbedQueue struct {
ch chan semanticEmbedTask
fs *memory.FileStore
}
func agenticTodoEnabled() bool {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_TODO_ENABLED")); v != "" {
if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") {
return false
}
}
return true
}
func agenticTodoMaxChars() int {
v := strings.TrimSpace(os.Getenv("CLIPROXY_TODO_MAX_CHARS"))
if v == "" {
return 4000
}
n, err := strconv.Atoi(v)
if err != nil {
return 4000
}
if n < 512 {
return 512
}
if n > 20_000 {
return 20_000
}
return n
}
func agenticMemoryMaxAgeDays() int {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_MAX_AGE_DAYS")); v != "" {
if n, err := strconv.Atoi(v); err == nil {
return n
}
}
return 0
}
func agenticMemoryMaxSessions() int {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_MAX_SESSIONS")); v != "" {
if n, err := strconv.Atoi(v); err == nil {
return n
}
}
return 0
}
func agenticMemoryMaxBytesPerSession() int64 {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_MAX_BYTES_PER_SESSION")); v != "" {
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
return n
}
}
return 0
}
func agenticSemanticMaxNamespaces() int {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_NAMESPACES")); v != "" {
if n, err := strconv.Atoi(v); err == nil {
return n
}
}
return 0
}
func agenticSemanticMaxBytesPerNamespace() int64 {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_BYTES_PER_NAMESPACE")); v != "" {
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
return n
}
}
return 0
}
func agenticMemoryMaxWritesPerMin() int {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_MAX_WRITES_PER_MIN")); v != "" {
if n, err := strconv.Atoi(v); err == nil {
return n
}
}
return 120
}
func agenticSemanticMaxWritesPerMin() int {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_WRITES_PER_MIN")); v != "" {
if n, err := strconv.Atoi(v); err == nil {
return n
}
}
return 120
}
func agenticSemanticQueryMaxChars() int {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_QUERY_MAX_CHARS")); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
return n
}
}
return 512
}
func agenticAnchorAppendOnly() bool {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_ANCHOR_APPEND_ONLY")); v != "" {
if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") {
return false
}
if strings.EqualFold(v, "1") || strings.EqualFold(v, "true") || strings.EqualFold(v, "on") || strings.EqualFold(v, "yes") {
return true
}
}
return true
}
func agenticAnchorSummaryMaxChars() int {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_ANCHOR_SUMMARY_MAX_CHARS")); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
return n
}
}
return 14_000
}
func agenticCompressionThreshold() float64 {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_COMPRESSION_THRESHOLD")); v != "" {
if f, err := strconv.ParseFloat(v, 64); err == nil && f > 0 && f < 1 {
return f
}
}
return 0.85
}
func agenticMinKeepMessages() int {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_MIN_KEEP_MESSAGES")); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
return n
}
}
return 25
}
func agenticLLMSummaryEnabled() bool {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_LLM_SUMMARY_ENABLED")); v != "" {
return strings.EqualFold(v, "1") || strings.EqualFold(v, "true") || strings.EqualFold(v, "on")
}
return true
}
func getModelContextWindow(model string) int {
info := registry.GetGlobalRegistry().GetModelInfo(model, "")
if info != nil && info.ContextLength > 0 {
return info.ContextLength
}
lowerModel := strings.ToLower(model)
switch {
case strings.Contains(lowerModel, "claude-3.5"), strings.Contains(lowerModel, "claude-3-5"):
return 200000
case strings.Contains(lowerModel, "claude-3"):
return 200000
case strings.Contains(lowerModel, "claude"):
return 100000
case strings.Contains(lowerModel, "gpt-4-turbo"), strings.Contains(lowerModel, "gpt-4o"):
return 128000
case strings.Contains(lowerModel, "gpt-4"):
return 8192
case strings.Contains(lowerModel, "gpt-3.5"):
return 16384
case strings.Contains(lowerModel, "gemini"):
return 1000000
case strings.Contains(lowerModel, "o1"), strings.Contains(lowerModel, "o3"):
return 200000
default:
return 100000
}
}
func agenticTokenAwareEnabled() bool {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_TOKEN_AWARE_ENABLED")); v != "" {
return strings.EqualFold(v, "1") || strings.EqualFold(v, "true") || strings.EqualFold(v, "on")
}
return true
}
func agenticReserveTokens() int {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_RESERVE_TOKENS")); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
return n
}
}
return 8192
}
func agenticScaffoldEnabled() bool {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SCAFFOLD_ENABLED")); v != "" {
if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") {
return false
}
}
return true
}
func agenticScaffoldAppendOnly() bool {
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SCAFFOLD_APPEND_ONLY")); v != "" {
if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") {
return false
}
}
return true
}
func allowMemoryWrite(session string) bool {
limit := agenticMemoryMaxWritesPerMin()
if limit <= 0 || session == "" {
return true
}
limiter := getSessionLimiter(memoryLimiters, session)
rate := float64(limit) / 60.0
burst := float64(limit) / 6.0
if burst < 5 {
burst = 5
}
return limiter.Allow(rate, burst)
}
func allowSemanticWrite(session string) bool {
limit := agenticSemanticMaxWritesPerMin()
if limit <= 0 || session == "" {
return true
}
limiter := getSessionLimiter(semanticLimiters, session)
rate := float64(limit) / 60.0
burst := float64(limit) / 6.0
if burst < 5 {
burst = 5
}
return limiter.Allow(rate, burst)
}
func getSessionLimiter(store map[string]*rateLimiter, session string) *rateLimiter {
limiterMu.Lock()
defer limiterMu.Unlock()
if limiter, ok := store[session]; ok {
return limiter
}
limiter := &rateLimiter{}
store[session] = limiter
for key, lim := range store {
if lim == nil {
delete(store, key)
continue
}
lim.mu.Lock()
last := lim.last
lim.mu.Unlock()
if !last.IsZero() && time.Since(last) > 15*time.Minute {
delete(store, key)
}
}
return limiter
}
File diff suppressed because it is too large Load Diff