From 7cc4e9d8f565d4c524dbd84b7ab333af4f47a4ce Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sun, 1 Feb 2026 12:28:00 +0700 Subject: [PATCH] refactor(middleware): split codex_prompt_budget.go into logical modules Split the 2,762 line codex_prompt_budget.go into focused modules: - codex_prompt_budget.go (1,272 lines): Main middleware, scaffold injection - codex_budget_types.go (498 lines): Constants, package vars, config functions - codex_budget_calculator.go (944 lines): Token analysis, trim functions Co-Authored-By: Claude (gemini-claude-opus-4-5-thinking) --- .../api/middleware/codex_budget_calculator.go | 944 ++++++++++ internal/api/middleware/codex_budget_types.go | 498 +++++ .../api/middleware/codex_prompt_budget.go | 1612 +---------------- 3 files changed, 1503 insertions(+), 1551 deletions(-) create mode 100644 internal/api/middleware/codex_budget_calculator.go create mode 100644 internal/api/middleware/codex_budget_types.go diff --git a/internal/api/middleware/codex_budget_calculator.go b/internal/api/middleware/codex_budget_calculator.go new file mode 100644 index 00000000..d9bc3d27 --- /dev/null +++ b/internal/api/middleware/codex_budget_calculator.go @@ -0,0 +1,944 @@ +package middleware + +import ( + "strconv" + "strings" + + "github.com/router-for-me/CLIProxyAPI/v6/internal/memory" + "github.com/router-for-me/CLIProxyAPI/v6/internal/registry" + "github.com/router-for-me/CLIProxyAPI/v6/internal/runtime/executor" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" +) + +// tokenAwareCompressionResult contains the result of token-aware analysis +type tokenAwareCompressionResult struct { + ShouldTrim bool // Whether trimming is needed + CurrentTokens int64 // Estimated current token count + ContextWindow int // Model's context window + TargetTokens int64 // Target token count after trimming + TargetMaxBytes int // Approximate bytes to achieve target tokens + Model string // The model name +} + +// analyzeTokenBudget checks if the request exceeds the token threshold and calculates trimming targets. +func analyzeTokenBudget(body []byte) *tokenAwareCompressionResult { + result := &tokenAwareCompressionResult{ + ShouldTrim: false, + TargetMaxBytes: agenticMaxBodyBytes(), + } + + if !agenticTokenAwareEnabled() { + return result + } + + model := gjson.GetBytes(body, "model").String() + if model == "" { + return result + } + result.Model = model + + contextWindow := getModelContextWindow(model) + result.ContextWindow = contextWindow + + currentTokens, err := executor.EstimateRequestTokens(model, body) + if err != nil { + currentTokens = int64(len(body) / 4) + } + result.CurrentTokens = currentTokens + + reserveTokens := agenticReserveTokens() + availableContext := contextWindow - reserveTokens + if availableContext < 1000 { + availableContext = contextWindow / 2 + } + + threshold := agenticCompressionThreshold() + maxInputTokens := int64(float64(availableContext) * threshold) + + if currentTokens <= maxInputTokens { + result.ShouldTrim = false + return result + } + + targetRatio := threshold * 0.9 + targetTokens := int64(float64(availableContext) * targetRatio) + result.TargetTokens = targetTokens + result.ShouldTrim = true + + targetBytes := int(targetTokens * 4) + + minBytes := 32 * 1024 + if targetBytes < minBytes { + targetBytes = minBytes + } + + maxBytes := agenticMaxBodyBytes() + if targetBytes > maxBytes { + targetBytes = maxBytes + } + + result.TargetMaxBytes = targetBytes + return result +} + +// getTokenAwareMaxBytes returns the maximum body size based on token analysis. +func getTokenAwareMaxBytes(body []byte) int { + if !agenticTokenAwareEnabled() { + return agenticMaxBodyBytesForModel(body) + } + + result := analyzeTokenBudget(body) + if result.ShouldTrim { + return result.TargetMaxBytes + } + + return agenticMaxBodyBytesForModel(body) +} + +func agenticMaxBodyBytesForModel(body []byte) int { + maxBytes := agenticMaxBodyBytes() + model := gjson.GetBytes(body, "model").String() + if model == "" { + return maxBytes + } + + info := registry.GetGlobalRegistry().GetModelInfo(model, "") + if info == nil || info.ContextLength <= 0 { + return maxBytes + } + + estimated := info.ContextLength * 4 + const minBytes = 32 * 1024 + if estimated < minBytes { + estimated = minBytes + } + if estimated < maxBytes { + return estimated + } + return maxBytes +} + +// trimWithMemoryResult holds the result of trimming with memory extraction. +type trimWithMemoryResult struct { + Body []byte + Query string + Dropped []memory.Event + Shape string // "chat", "responses", or "claude" +} + +// trimOpenAIChatCompletions trims an OpenAI Chat Completions payload by shortening the messages array. +func trimOpenAIChatCompletions(body []byte, maxBytes int, mustKeepTools bool) []byte { + root := gjson.ParseBytes(body) + msgs := root.Get("messages") + if !msgs.IsArray() { + return body + } + arr := msgs.Array() + if len(arr) == 0 { + return body + } + + firstSystem := gjson.Result{} + for i := 0; i < len(arr); i++ { + if strings.EqualFold(arr[i].Get("role").String(), "system") { + firstSystem = arr[i] + break + } + } + + isToolResultMsg := func(m gjson.Result) bool { + role := strings.ToLower(strings.TrimSpace(m.Get("role").String())) + return role == "tool" || role == "function" + } + assistantHasToolCall := func(m gjson.Result) bool { + if !strings.EqualFold(m.Get("role").String(), "assistant") { + return false + } + if tc := m.Get("tool_calls"); tc.Exists() && tc.IsArray() && len(tc.Array()) > 0 { + return true + } + if fc := m.Get("function_call"); fc.Exists() { + return true + } + return false + } + + keep := 20 + perTextLimit := 20_000 + dropTools := false + minKeep := agenticMinKeepMessages() + for keep >= minKeep { + outBody := body + if dropTools && !mustKeepTools { + outBody, _ = sjson.DeleteBytes(outBody, "tools") + outBody, _ = sjson.SetBytes(outBody, "tool_choice", "none") + } + + newMsgs := make([]string, 0, keep+1) + if firstSystem.Exists() { + newMsgs = append(newMsgs, truncateMessageContent(firstSystem.Raw, perTextLimit)) + } + + required := make(map[int]struct{}, 8) + tailKept := 0 + for i := len(arr) - 1; i >= 0 && (tailKept < keep || len(required) > 0); i-- { + if strings.EqualFold(arr[i].Get("role").String(), "system") { + continue + } + + _, req := required[i] + if !req && tailKept >= keep { + continue + } + + newMsgs = append(newMsgs, truncateMessageContent(arr[i].Raw, perTextLimit)) + if !req { + tailKept++ + } else { + delete(required, i) + } + + if isToolResultMsg(arr[i]) { + prev := i - 1 + if prev >= 0 && assistantHasToolCall(arr[prev]) { + required[prev] = struct{}{} + } + } + } + + if firstSystem.Exists() { + reverseStrings(newMsgs[1:]) + } else { + reverseStrings(newMsgs) + } + + out := setJSONArrayBytes(outBody, "messages", newMsgs) + if len(out) <= maxBytes { + return out + } + + keep = keep / 2 + if perTextLimit > 5_000 { + perTextLimit = perTextLimit / 2 + } + dropTools = true + } + + return body +} + +// trimOpenAIResponses trims an OpenAI Responses payload by shortening the input array. +func trimOpenAIResponses(body []byte, maxBytes int, mustKeepTools bool) []byte { + root := gjson.ParseBytes(body) + input := root.Get("input") + if !input.Exists() || !input.IsArray() { + return body + } + arr := input.Array() + if len(arr) == 0 { + return body + } + + keep := 30 + perTextLimit := 20_000 + dropTools := false + minKeep := agenticMinKeepMessages() + for keep >= minKeep { + outBody := body + if dropTools && !mustKeepTools { + outBody, _ = sjson.DeleteBytes(outBody, "tools") + outBody, _ = sjson.SetBytes(outBody, "tool_choice", "none") + } + if inst := root.Get("instructions"); inst.Exists() && inst.Type == gjson.String { + s := inst.String() + instructionsLimit := perTextLimit + if instructionsLimit > 2048 { + instructionsLimit = 2048 + } + if len(s) > instructionsLimit { + outBody, _ = sjson.SetBytes(outBody, "instructions", s[:instructionsLimit]+"\n...[truncated]...") + } + } + + callByID := make(map[string]string, 16) + for i := 0; i < len(arr); i++ { + item := arr[i] + t := item.Get("type").String() + if t == "" && item.Get("role").String() != "" { + t = "message" + } + if t != "function_call" { + continue + } + callID := item.Get("call_id").String() + if callID == "" { + continue + } + if _, ok := callByID[callID]; !ok { + callByID[callID] = item.Raw + } + } + + needCall := make(map[string]struct{}, 8) + newItems := make([]string, 0, keep+8) + kept := 0 + for i := len(arr) - 1; i >= 0 && kept < keep; i-- { + item := arr[i] + t := item.Get("type").String() + if t == "" && item.Get("role").String() != "" { + t = "message" + } + + if dropTools && !mustKeepTools && (t == "function_call" || t == "function_call_output") { + continue + } + + if t == "function_call_output" { + callID := item.Get("call_id").String() + if callID != "" { + needCall[callID] = struct{}{} + } + } + if t == "function_call" { + callID := item.Get("call_id").String() + if callID != "" { + delete(needCall, callID) + } + } + + newItems = append(newItems, truncateMessageContent(item.Raw, perTextLimit)) + kept++ + } + reverseStrings(newItems) + + if !dropTools && len(needCall) > 0 { + prefix := make([]string, 0, len(needCall)) + for callID := range needCall { + if raw, ok := callByID[callID]; ok { + prefix = append(prefix, raw) + } + } + if len(prefix) > 0 { + ordered := make([]string, 0, len(prefix)) + for i := 0; i < len(arr); i++ { + item := arr[i] + if item.Get("type").String() != "function_call" { + continue + } + callID := item.Get("call_id").String() + if callID == "" { + continue + } + if _, ok := needCall[callID]; ok { + if raw, ok2 := callByID[callID]; ok2 { + ordered = append(ordered, raw) + } + } + } + newItems = append(ordered, newItems...) + } + } + + out := setJSONArrayBytes(outBody, "input", newItems) + if len(out) <= maxBytes { + return out + } + + keep = keep / 2 + if perTextLimit > 5_000 { + perTextLimit = perTextLimit / 2 + } + if !mustKeepTools { + dropTools = true + } + } + + return body +} + +func trimOpenAIChatCompletionsWithMemory(body []byte, maxBytes int, mustKeepTools bool) *trimWithMemoryResult { + root := gjson.ParseBytes(body) + msgs := root.Get("messages") + if !msgs.IsArray() { + return &trimWithMemoryResult{Body: body, Shape: "chat"} + } + arr := msgs.Array() + if len(arr) == 0 { + return &trimWithMemoryResult{Body: body, Shape: "chat"} + } + + firstSystem := gjson.Result{} + firstSystemIndex := -1 + for i := 0; i < len(arr); i++ { + if strings.EqualFold(arr[i].Get("role").String(), "system") { + firstSystem = arr[i] + firstSystemIndex = i + break + } + } + + query := extractLastUserTextFromChat(arr) + minKeep := agenticMinKeepMessages() + keep := 20 + perTextLimit := 20_000 + dropTools := false + for keep >= minKeep { + outBody := body + if dropTools && !mustKeepTools { + outBody, _ = sjson.DeleteBytes(outBody, "tools") + outBody, _ = sjson.SetBytes(outBody, "tool_choice", "none") + } + + newMsgs := make([]string, 0, keep+1) + keptIdx := make(map[int]struct{}, keep+2) + if firstSystem.Exists() { + newMsgs = append(newMsgs, truncateMessageContent(firstSystem.Raw, perTextLimit)) + keptIdx[firstSystemIndex] = struct{}{} + } + + isToolResultMsg := func(m gjson.Result) bool { + role := strings.ToLower(strings.TrimSpace(m.Get("role").String())) + return role == "tool" || role == "function" + } + assistantHasToolCall := func(m gjson.Result) bool { + if !strings.EqualFold(m.Get("role").String(), "assistant") { + return false + } + if tc := m.Get("tool_calls"); tc.Exists() && tc.IsArray() && len(tc.Array()) > 0 { + return true + } + if fc := m.Get("function_call"); fc.Exists() { + return true + } + return false + } + + required := make(map[int]struct{}, 8) + tailKept := 0 + for i := len(arr) - 1; i >= 0 && (tailKept < keep || len(required) > 0); i-- { + if strings.EqualFold(arr[i].Get("role").String(), "system") { + continue + } + + _, req := required[i] + if !req && tailKept >= keep { + continue + } + + newMsgs = append(newMsgs, truncateMessageContent(arr[i].Raw, perTextLimit)) + keptIdx[i] = struct{}{} + if !req { + tailKept++ + } else { + delete(required, i) + } + + if isToolResultMsg(arr[i]) { + prev := i - 1 + if prev >= 0 && assistantHasToolCall(arr[prev]) { + required[prev] = struct{}{} + } + } + } + + if firstSystem.Exists() { + reverseStrings(newMsgs[1:]) + } else { + reverseStrings(newMsgs) + } + + out := setJSONArrayBytes(outBody, "messages", newMsgs) + if len(out) <= maxBytes { + dropped := collectDroppedChat(arr, keptIdx) + return &trimWithMemoryResult{Body: out, Query: query, Dropped: dropped, Shape: "chat"} + } + + keep = keep / 2 + if perTextLimit > 5_000 { + perTextLimit = perTextLimit / 2 + } + dropTools = true + } + return &trimWithMemoryResult{Body: body, Query: query, Shape: "chat"} +} + +func collectDroppedChat(arr []gjson.Result, kept map[int]struct{}) []memory.Event { + out := make([]memory.Event, 0, 32) + for i := 0; i < len(arr); i++ { + if _, ok := kept[i]; ok { + continue + } + role := arr[i].Get("role").String() + txt := extractTextFromChatMessage(arr[i]) + if strings.TrimSpace(txt) == "" { + continue + } + out = append(out, memory.Event{Kind: "dropped_chat", Role: role, Text: txt}) + } + return out +} + +func extractLastUserTextFromChat(arr []gjson.Result) string { + for i := len(arr) - 1; i >= 0; i-- { + if !strings.EqualFold(arr[i].Get("role").String(), "user") { + continue + } + txt := extractTextFromChatMessage(arr[i]) + if strings.TrimSpace(txt) != "" { + return txt + } + } + return "" +} + +func extractTextFromChatMessage(msg gjson.Result) string { + content := msg.Get("content") + switch { + case content.Type == gjson.String: + return content.String() + case content.IsArray(): + var b strings.Builder + for _, it := range content.Array() { + partType := it.Get("type").String() + if partType == "thinking" || partType == "reasoning" { + continue + } + if t := it.Get("text"); t.Exists() && t.Type == gjson.String { + if b.Len() > 0 { + b.WriteString("\n") + } + b.WriteString(t.String()) + } + } + return b.String() + default: + return "" + } +} + +func trimOpenAIResponsesWithMemory(body []byte, maxBytes int, mustKeepTools bool) *trimWithMemoryResult { + root := gjson.ParseBytes(body) + input := root.Get("input") + if !input.Exists() || !input.IsArray() { + return &trimWithMemoryResult{Body: body, Shape: "responses"} + } + arr := input.Array() + if len(arr) == 0 { + return &trimWithMemoryResult{Body: body, Shape: "responses"} + } + + query := extractLastUserTextFromResponses(arr) + minKeep := agenticMinKeepMessages() + keep := 30 + perTextLimit := 20_000 + dropTools := false + for keep >= minKeep { + outBody := body + if dropTools && !mustKeepTools { + outBody, _ = sjson.DeleteBytes(outBody, "tools") + outBody, _ = sjson.SetBytes(outBody, "tool_choice", "none") + } + if inst := root.Get("instructions"); inst.Exists() && inst.Type == gjson.String { + s := inst.String() + instructionsLimit := perTextLimit + if instructionsLimit > 2048 { + instructionsLimit = 2048 + } + if len(s) > instructionsLimit { + outBody, _ = sjson.SetBytes(outBody, "instructions", s[:instructionsLimit]+"\n...[truncated]...") + } + } + + callByID := make(map[string]gjson.Result, 16) + for i := 0; i < len(arr); i++ { + item := arr[i] + t := item.Get("type").String() + if t == "" && item.Get("role").String() != "" { + t = "message" + } + if t != "function_call" { + continue + } + callID := item.Get("call_id").String() + if callID == "" { + continue + } + if _, ok := callByID[callID]; !ok { + callByID[callID] = item + } + } + + needCall := make(map[string]struct{}, 8) + newItems := make([]string, 0, keep+8) + keptIdx := make(map[int]struct{}, keep+16) + kept := 0 + for i := len(arr) - 1; i >= 0 && kept < keep; i-- { + item := arr[i] + t := item.Get("type").String() + if t == "" && item.Get("role").String() != "" { + t = "message" + } + + if dropTools && !mustKeepTools && (t == "function_call" || t == "function_call_output") { + continue + } + if t == "function_call_output" { + callID := item.Get("call_id").String() + if callID != "" { + needCall[callID] = struct{}{} + } + } + if t == "function_call" { + callID := item.Get("call_id").String() + if callID != "" { + delete(needCall, callID) + } + } + + newItems = append(newItems, truncateMessageContent(item.Raw, perTextLimit)) + keptIdx[i] = struct{}{} + kept++ + } + reverseStrings(newItems) + + if !dropTools && len(needCall) > 0 { + ordered := make([]string, 0, len(needCall)) + for i := 0; i < len(arr); i++ { + item := arr[i] + if item.Get("type").String() != "function_call" { + continue + } + callID := item.Get("call_id").String() + if callID == "" { + continue + } + if _, ok := needCall[callID]; ok { + if call, ok2 := callByID[callID]; ok2 { + ordered = append(ordered, call.Raw) + keptIdx[i] = struct{}{} + } + } + } + if len(ordered) > 0 { + newItems = append(ordered, newItems...) + } + } + + out := setJSONArrayBytes(outBody, "input", newItems) + if len(out) <= maxBytes { + dropped := collectDroppedResponses(arr, keptIdx) + return &trimWithMemoryResult{Body: out, Query: query, Dropped: dropped, Shape: "responses"} + } + + keep = keep / 2 + if perTextLimit > 5_000 { + perTextLimit = perTextLimit / 2 + } + if !mustKeepTools { + dropTools = true + } + } + + return &trimWithMemoryResult{Body: body, Query: query, Shape: "responses"} +} + +func collectDroppedResponses(arr []gjson.Result, kept map[int]struct{}) []memory.Event { + out := make([]memory.Event, 0, 64) + for i := 0; i < len(arr); i++ { + if _, ok := kept[i]; ok { + continue + } + item := arr[i] + t := item.Get("type").String() + role := item.Get("role").String() + txt := extractTextFromResponsesItem(item) + if strings.TrimSpace(txt) == "" { + continue + } + out = append(out, memory.Event{Kind: "dropped_responses", Type: t, Role: role, Text: txt}) + } + return out +} + +func extractLastUserTextFromResponses(arr []gjson.Result) string { + for i := len(arr) - 1; i >= 0; i-- { + item := arr[i] + role := item.Get("role").String() + if !strings.EqualFold(role, "user") { + continue + } + txt := extractTextFromResponsesItem(item) + if strings.TrimSpace(txt) != "" { + return txt + } + } + return "" +} + +func extractTextFromResponsesItem(item gjson.Result) string { + content := item.Get("content") + if content.IsArray() { + var b strings.Builder + for _, part := range content.Array() { + partType := part.Get("type").String() + if partType == "thinking" || partType == "reasoning" { + continue + } + text := part.Get("text") + if !text.Exists() || text.Type != gjson.String { + continue + } + if b.Len() > 0 { + b.WriteString("\n") + } + b.WriteString(text.String()) + } + return b.String() + } + if content.Type == gjson.String { + return content.String() + } + if t := item.Get("text"); t.Exists() && t.Type == gjson.String { + return t.String() + } + if out := item.Get("output"); out.Exists() && out.Type == gjson.String { + return out.String() + } + return "" +} + +func trimClaudeMessagesWithMemory(body []byte, maxBytes int, mustKeepTools bool) *trimWithMemoryResult { + root := gjson.ParseBytes(body) + msgs := root.Get("messages") + if !msgs.IsArray() { + return &trimWithMemoryResult{Body: body, Shape: "claude"} + } + arr := msgs.Array() + if len(arr) == 0 { + return &trimWithMemoryResult{Body: body, Shape: "claude"} + } + + query := extractLastUserTextFromClaude(arr) + minKeep := agenticMinKeepMessages() + keep := 20 + perTextLimit := 20_000 + dropTools := false + for keep >= minKeep { + outBody := body + if dropTools && !mustKeepTools { + outBody, _ = sjson.DeleteBytes(outBody, "tools") + outBody, _ = sjson.SetBytes(outBody, "tool_choice", map[string]any{"type": "none"}) + } + + newMsgs := make([]string, 0, keep+1) + keptIdx := make(map[int]struct{}, keep+2) + + isToolResultMsg := func(m gjson.Result) bool { + content := m.Get("content") + if !content.IsArray() { + return false + } + for _, part := range content.Array() { + if part.Get("type").String() == "tool_result" { + return true + } + } + return false + } + assistantHasToolUse := func(m gjson.Result) bool { + if !strings.EqualFold(m.Get("role").String(), "assistant") { + return false + } + content := m.Get("content") + if !content.IsArray() { + return false + } + for _, part := range content.Array() { + if part.Get("type").String() == "tool_use" { + return true + } + } + return false + } + + required := make(map[int]struct{}, 8) + tailKept := 0 + for i := len(arr) - 1; i >= 0 && (tailKept < keep || len(required) > 0); i-- { + _, req := required[i] + if !req && tailKept >= keep { + continue + } + + newMsgs = append(newMsgs, truncateClaudeMessageContent(arr[i].Raw, perTextLimit)) + keptIdx[i] = struct{}{} + if !req { + tailKept++ + } else { + delete(required, i) + } + + if isToolResultMsg(arr[i]) { + prev := i - 1 + if prev >= 0 && assistantHasToolUse(arr[prev]) { + required[prev] = struct{}{} + } + } + } + + reverseStrings(newMsgs) + + out := setJSONArrayBytes(outBody, "messages", newMsgs) + if len(out) <= maxBytes { + dropped := collectDroppedClaude(arr, keptIdx) + return &trimWithMemoryResult{Body: out, Query: query, Dropped: dropped, Shape: "claude"} + } + + keep = keep / 2 + if perTextLimit > 5_000 { + perTextLimit = perTextLimit / 2 + } + dropTools = true + } + return &trimWithMemoryResult{Body: body, Query: query, Shape: "claude"} +} + +func collectDroppedClaude(arr []gjson.Result, kept map[int]struct{}) []memory.Event { + out := make([]memory.Event, 0, 32) + for i := 0; i < len(arr); i++ { + if _, ok := kept[i]; ok { + continue + } + role := arr[i].Get("role").String() + txt := extractTextFromClaudeMessage(arr[i]) + if strings.TrimSpace(txt) == "" { + continue + } + out = append(out, memory.Event{Kind: "dropped_claude", Role: role, Text: txt}) + } + return out +} + +func extractLastUserTextFromClaude(arr []gjson.Result) string { + for i := len(arr) - 1; i >= 0; i-- { + if !strings.EqualFold(arr[i].Get("role").String(), "user") { + continue + } + txt := extractTextFromClaudeMessage(arr[i]) + if strings.TrimSpace(txt) != "" { + return txt + } + } + return "" +} + +func extractTextFromClaudeMessage(msg gjson.Result) string { + content := msg.Get("content") + switch { + case content.Type == gjson.String: + return content.String() + case content.IsArray(): + var b strings.Builder + for _, part := range content.Array() { + partType := part.Get("type").String() + if partType == "text" { + if t := part.Get("text"); t.Exists() && t.Type == gjson.String { + if b.Len() > 0 { + b.WriteString("\n") + } + b.WriteString(t.String()) + } + } + } + return b.String() + default: + return "" + } +} + +func truncateMessageContent(msgRaw string, maxTextChars int) string { + msg := msgRaw + if maxTextChars <= 0 { + return msg + } + + content := gjson.Get(msg, "content") + switch { + case content.Type == gjson.String: + s := content.String() + if len(s) > maxTextChars { + s = s[:maxTextChars] + "\n...[truncated]..." + msg, _ = sjson.Set(msg, "content", s) + } + return msg + case content.IsArray(): + items := content.Array() + for i := 0; i < len(items); i++ { + text := items[i].Get("text") + if !text.Exists() || text.Type != gjson.String { + continue + } + s := text.String() + if len(s) > maxTextChars { + s = s[:maxTextChars] + "\n...[truncated]..." + msg, _ = sjson.Set(msg, "content."+strconv.Itoa(i)+".text", s) + } + } + return msg + default: + return msg + } +} + +func truncateClaudeMessageContent(msgRaw string, maxTextChars int) string { + msg := msgRaw + if maxTextChars <= 0 { + return msg + } + + content := gjson.Get(msg, "content") + switch { + case content.Type == gjson.String: + s := content.String() + if len(s) > maxTextChars { + s = s[:maxTextChars] + "\n...[truncated]..." + msg, _ = sjson.Set(msg, "content", s) + } + return msg + case content.IsArray(): + items := content.Array() + for i := 0; i < len(items); i++ { + partType := items[i].Get("type").String() + if partType != "text" { + continue + } + text := items[i].Get("text") + if !text.Exists() || text.Type != gjson.String { + continue + } + s := text.String() + if len(s) > maxTextChars { + s = s[:maxTextChars] + "\n...[truncated]..." + msg, _ = sjson.Set(msg, "content."+strconv.Itoa(i)+".text", s) + } + } + return msg + default: + return msg + } +} + +func setJSONArrayBytes(body []byte, key string, rawItems []string) []byte { + out := body + out, _ = sjson.SetRawBytes(out, key, []byte("[]")) + for i := range rawItems { + out, _ = sjson.SetRawBytes(out, key+".-1", []byte(rawItems[i])) + } + return out +} + +func reverseStrings(items []string) { + for i, j := 0, len(items)-1; i < j; i, j = i+1, j-1 { + items[i], items[j] = items[j], items[i] + } +} diff --git a/internal/api/middleware/codex_budget_types.go b/internal/api/middleware/codex_budget_types.go new file mode 100644 index 00000000..8a711d80 --- /dev/null +++ b/internal/api/middleware/codex_budget_types.go @@ -0,0 +1,498 @@ +package middleware + +import ( + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/router-for-me/CLIProxyAPI/v6/internal/embeddings" + "github.com/router-for-me/CLIProxyAPI/v6/internal/memory" + "github.com/router-for-me/CLIProxyAPI/v6/internal/registry" + "github.com/router-for-me/CLIProxyAPI/v6/internal/util" +) + +const ( + // codexHardReadLimit is a safety ceiling to avoid unbounded memory reads. + codexHardReadLimit = 10 * 1024 * 1024 + // codexMaxBodyBytesDefault is a best-effort budget to keep agentic CLI requests under common model limits. + // It is intentionally conservative to avoid upstream "prompt too long" failures. + codexMaxBodyBytesDefault = 200 * 1024 + + specModePrompt = `SPEC MODE (do not code yet). +1) Produce a complete, reviewable specification (requirements, acceptance criteria, architecture, and file-level plan). +2) Wait for explicit approval before editing code. +3) If clarification is needed, ask now before writing code.` +) + +var ( + codexMaxBodyBytesOnce sync.Once + codexMaxBodyBytes int + + memOnce sync.Once + memStore memory.Store + + embedOnce sync.Once + embedClient *embeddings.OllamaClient + + embedQueueOnce sync.Once + embedQueue *semanticEmbedQueue + + pruneMu sync.Mutex + lastPrune time.Time + + limiterMu sync.Mutex + memoryLimiters = map[string]*rateLimiter{} + semanticLimiters = map[string]*rateLimiter{} +) + +type rateLimiter struct { + mu sync.Mutex + tokens float64 + last time.Time +} + +func (r *rateLimiter) Allow(ratePerSec float64, burst float64) bool { + if ratePerSec <= 0 { + return true + } + if burst <= 0 { + burst = ratePerSec + } + r.mu.Lock() + defer r.mu.Unlock() + now := time.Now() + if r.last.IsZero() { + r.last = now + r.tokens = burst + } + elapsed := now.Sub(r.last).Seconds() + if elapsed > 0 { + r.tokens += elapsed * ratePerSec + if r.tokens > burst { + r.tokens = burst + } + r.last = now + } + if r.tokens >= 1 { + r.tokens -= 1 + return true + } + return false +} + +func agenticMaxBodyBytes() int { + codexMaxBodyBytesOnce.Do(func() { + codexMaxBodyBytes = codexMaxBodyBytesDefault + + // Optional override (bytes). Useful when running behind very large-context models. + if v := strings.TrimSpace(os.Getenv("CLIPROXY_AGENTIC_MAX_BODY_BYTES")); v != "" { + if n, err := strconv.Atoi(v); err == nil { + if n < 32*1024 { + n = 32 * 1024 + } + if n > 2*1024*1024 { + n = 2 * 1024 * 1024 + } + codexMaxBodyBytes = n + } + } + }) + return codexMaxBodyBytes +} + +func agenticMemoryStore() memory.Store { + memOnce.Do(func() { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_ENABLED")); v != "" { + if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") { + memStore = nil + return + } + } + + base := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_DIR")) + if base == "" { + if w := util.WritablePath(); w != "" { + base = filepath.Join(w, ".proxypilot", "memory") + } else { + base = filepath.Join(".proxypilot", "memory") + } + } + memStore = memory.NewFileStore(base) + + if agenticLLMSummaryEnabled() { + if fs, ok := memStore.(*memory.FileStore); ok { + config := memory.DefaultSummarizerConfig() + summarizer := memory.NewSummarizer(config, memory.NewNoOpSummarizerExecutor()) + fs.SetSummarizer(summarizer) + } + } + }) + return memStore +} + +// SetSummarizerExecutor configures the LLM executor for context summarization. +func SetSummarizerExecutor(executor memory.SummarizerExecutor) { + store := agenticMemoryStore() + if store == nil { + return + } + fs, ok := store.(*memory.FileStore) + if !ok || fs == nil { + return + } + summarizer := fs.GetSummarizer() + if summarizer == nil { + config := memory.DefaultSummarizerConfig() + summarizer = memory.NewSummarizer(config, executor) + fs.SetSummarizer(summarizer) + } else { + config := memory.DefaultSummarizerConfig() + fs.SetSummarizer(memory.NewSummarizer(config, executor)) + } +} + +// InitSummarizerWithAuthManager configures LLM-based summarization using the core auth manager. +func InitSummarizerWithAuthManager(manager memory.CoreManagerExecutor, providers []string) { + if manager == nil { + return + } + adapter := memory.NewManagerAuthAdapter(manager) + baseExecutor := memory.NewPipelineSummarizerExecutor(adapter, providers) + executor := memory.NewSummaryModelFallbackExecutor(baseExecutor) + SetSummarizerExecutor(executor) +} + +// GetSummaryModel returns the configured summary model, defaulting to gemini-3-flash. +func GetSummaryModel() string { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_SUMMARY_MODEL")); v != "" { + return v + } + return memory.DefaultSummaryModel +} + +func agenticSemanticEnabled() bool { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_ENABLED")); v != "" { + if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") { + return false + } + } + return true +} + +func agenticSemanticModel() string { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MODEL")); v != "" { + return v + } + return "embeddinggemma" +} + +func agenticSemanticBaseURL() string { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_BASE_URL")); v != "" { + return v + } + return "http://127.0.0.1:11434" +} + +func agenticSemanticMaxSnips() int { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_SNIPS")); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + return n + } + } + return 4 +} + +func agenticSemanticMaxChars() int { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_CHARS")); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + return n + } + } + return 3000 +} + +func agenticSemanticClient() *embeddings.OllamaClient { + embedOnce.Do(func() { + embedClient = &embeddings.OllamaClient{ + BaseURL: agenticSemanticBaseURL(), + Model: agenticSemanticModel(), + Client: &http.Client{Timeout: 8 * time.Second}, + } + }) + return embedClient +} + +type semanticEmbedTask struct { + namespace string + session string + texts []string + roles []string + source string +} + +type semanticEmbedQueue struct { + ch chan semanticEmbedTask + fs *memory.FileStore +} + +func agenticTodoEnabled() bool { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_TODO_ENABLED")); v != "" { + if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") { + return false + } + } + return true +} + +func agenticTodoMaxChars() int { + v := strings.TrimSpace(os.Getenv("CLIPROXY_TODO_MAX_CHARS")) + if v == "" { + return 4000 + } + n, err := strconv.Atoi(v) + if err != nil { + return 4000 + } + if n < 512 { + return 512 + } + if n > 20_000 { + return 20_000 + } + return n +} + +func agenticMemoryMaxAgeDays() int { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_MAX_AGE_DAYS")); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return 0 +} + +func agenticMemoryMaxSessions() int { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_MAX_SESSIONS")); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return 0 +} + +func agenticMemoryMaxBytesPerSession() int64 { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_MAX_BYTES_PER_SESSION")); v != "" { + if n, err := strconv.ParseInt(v, 10, 64); err == nil { + return n + } + } + return 0 +} + +func agenticSemanticMaxNamespaces() int { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_NAMESPACES")); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return 0 +} + +func agenticSemanticMaxBytesPerNamespace() int64 { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_BYTES_PER_NAMESPACE")); v != "" { + if n, err := strconv.ParseInt(v, 10, 64); err == nil { + return n + } + } + return 0 +} + +func agenticMemoryMaxWritesPerMin() int { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_MAX_WRITES_PER_MIN")); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return 120 +} + +func agenticSemanticMaxWritesPerMin() int { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_WRITES_PER_MIN")); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return 120 +} + +func agenticSemanticQueryMaxChars() int { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_QUERY_MAX_CHARS")); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + return n + } + } + return 512 +} + +func agenticAnchorAppendOnly() bool { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_ANCHOR_APPEND_ONLY")); v != "" { + if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") { + return false + } + if strings.EqualFold(v, "1") || strings.EqualFold(v, "true") || strings.EqualFold(v, "on") || strings.EqualFold(v, "yes") { + return true + } + } + return true +} + +func agenticAnchorSummaryMaxChars() int { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_ANCHOR_SUMMARY_MAX_CHARS")); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + return n + } + } + return 14_000 +} + +func agenticCompressionThreshold() float64 { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_COMPRESSION_THRESHOLD")); v != "" { + if f, err := strconv.ParseFloat(v, 64); err == nil && f > 0 && f < 1 { + return f + } + } + return 0.85 +} + +func agenticMinKeepMessages() int { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_MIN_KEEP_MESSAGES")); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + return n + } + } + return 25 +} + +func agenticLLMSummaryEnabled() bool { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_LLM_SUMMARY_ENABLED")); v != "" { + return strings.EqualFold(v, "1") || strings.EqualFold(v, "true") || strings.EqualFold(v, "on") + } + return true +} + +func getModelContextWindow(model string) int { + info := registry.GetGlobalRegistry().GetModelInfo(model, "") + if info != nil && info.ContextLength > 0 { + return info.ContextLength + } + lowerModel := strings.ToLower(model) + switch { + case strings.Contains(lowerModel, "claude-3.5"), strings.Contains(lowerModel, "claude-3-5"): + return 200000 + case strings.Contains(lowerModel, "claude-3"): + return 200000 + case strings.Contains(lowerModel, "claude"): + return 100000 + case strings.Contains(lowerModel, "gpt-4-turbo"), strings.Contains(lowerModel, "gpt-4o"): + return 128000 + case strings.Contains(lowerModel, "gpt-4"): + return 8192 + case strings.Contains(lowerModel, "gpt-3.5"): + return 16384 + case strings.Contains(lowerModel, "gemini"): + return 1000000 + case strings.Contains(lowerModel, "o1"), strings.Contains(lowerModel, "o3"): + return 200000 + default: + return 100000 + } +} + +func agenticTokenAwareEnabled() bool { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_TOKEN_AWARE_ENABLED")); v != "" { + return strings.EqualFold(v, "1") || strings.EqualFold(v, "true") || strings.EqualFold(v, "on") + } + return true +} + +func agenticReserveTokens() int { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_RESERVE_TOKENS")); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + return n + } + } + return 8192 +} + +func agenticScaffoldEnabled() bool { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_SCAFFOLD_ENABLED")); v != "" { + if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") { + return false + } + } + return true +} + +func agenticScaffoldAppendOnly() bool { + if v := strings.TrimSpace(os.Getenv("CLIPROXY_SCAFFOLD_APPEND_ONLY")); v != "" { + if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") { + return false + } + } + return true +} + +func allowMemoryWrite(session string) bool { + limit := agenticMemoryMaxWritesPerMin() + if limit <= 0 || session == "" { + return true + } + limiter := getSessionLimiter(memoryLimiters, session) + rate := float64(limit) / 60.0 + burst := float64(limit) / 6.0 + if burst < 5 { + burst = 5 + } + return limiter.Allow(rate, burst) +} + +func allowSemanticWrite(session string) bool { + limit := agenticSemanticMaxWritesPerMin() + if limit <= 0 || session == "" { + return true + } + limiter := getSessionLimiter(semanticLimiters, session) + rate := float64(limit) / 60.0 + burst := float64(limit) / 6.0 + if burst < 5 { + burst = 5 + } + return limiter.Allow(rate, burst) +} + +func getSessionLimiter(store map[string]*rateLimiter, session string) *rateLimiter { + limiterMu.Lock() + defer limiterMu.Unlock() + if limiter, ok := store[session]; ok { + return limiter + } + limiter := &rateLimiter{} + store[session] = limiter + for key, lim := range store { + if lim == nil { + delete(store, key) + continue + } + lim.mu.Lock() + last := lim.last + lim.mu.Unlock() + if !last.IsZero() && time.Since(last) > 15*time.Minute { + delete(store, key) + } + } + return limiter +} diff --git a/internal/api/middleware/codex_prompt_budget.go b/internal/api/middleware/codex_prompt_budget.go index 264ddb5f..2a03c471 100644 --- a/internal/api/middleware/codex_prompt_budget.go +++ b/internal/api/middleware/codex_prompt_budget.go @@ -11,686 +11,14 @@ import ( "path/filepath" "strconv" "strings" - "sync" "time" "github.com/gin-gonic/gin" - "github.com/router-for-me/CLIProxyAPI/v6/internal/embeddings" "github.com/router-for-me/CLIProxyAPI/v6/internal/memory" - "github.com/router-for-me/CLIProxyAPI/v6/internal/registry" - "github.com/router-for-me/CLIProxyAPI/v6/internal/runtime/executor" - "github.com/router-for-me/CLIProxyAPI/v6/internal/util" "github.com/tidwall/gjson" "github.com/tidwall/sjson" ) -const ( - // codexHardReadLimit is a safety ceiling to avoid unbounded memory reads. - codexHardReadLimit = 10 * 1024 * 1024 - // codexMaxBodyBytesDefault is a best-effort budget to keep agentic CLI requests under common model limits. - // It is intentionally conservative to avoid upstream "prompt too long" failures. - codexMaxBodyBytesDefault = 200 * 1024 - - specModePrompt = `SPEC MODE (do not code yet). -1) Produce a complete, reviewable specification (requirements, acceptance criteria, architecture, and file-level plan). -2) Wait for explicit approval before editing code. -3) If clarification is needed, ask now before writing code.` -) - -var ( - codexMaxBodyBytesOnce sync.Once - codexMaxBodyBytes int - - memOnce sync.Once - memStore memory.Store - - embedOnce sync.Once - embedClient *embeddings.OllamaClient - - embedQueueOnce sync.Once - embedQueue *semanticEmbedQueue - - pruneMu sync.Mutex - lastPrune time.Time - - limiterMu sync.Mutex - memoryLimiters = map[string]*rateLimiter{} - semanticLimiters = map[string]*rateLimiter{} -) - -type rateLimiter struct { - mu sync.Mutex - tokens float64 - last time.Time -} - -func (r *rateLimiter) Allow(ratePerSec float64, burst float64) bool { - if ratePerSec <= 0 { - return true - } - if burst <= 0 { - burst = ratePerSec - } - r.mu.Lock() - defer r.mu.Unlock() - now := time.Now() - if r.last.IsZero() { - r.last = now - r.tokens = burst - } - elapsed := now.Sub(r.last).Seconds() - if elapsed > 0 { - r.tokens += elapsed * ratePerSec - if r.tokens > burst { - r.tokens = burst - } - r.last = now - } - if r.tokens >= 1 { - r.tokens -= 1 - return true - } - return false -} - -func agenticMaxBodyBytes() int { - codexMaxBodyBytesOnce.Do(func() { - codexMaxBodyBytes = codexMaxBodyBytesDefault - - // Optional override (bytes). Useful when running behind very large-context models. - // Examples: - // set CLIPROXY_AGENTIC_MAX_BODY_BYTES=350000 - // export CLIPROXY_AGENTIC_MAX_BODY_BYTES=350000 - if v := strings.TrimSpace(os.Getenv("CLIPROXY_AGENTIC_MAX_BODY_BYTES")); v != "" { - if n, err := strconv.Atoi(v); err == nil { - // Clamp to a sane range: 32KB..2MB - if n < 32*1024 { - n = 32 * 1024 - } - if n > 2*1024*1024 { - n = 2 * 1024 * 1024 - } - codexMaxBodyBytes = n - } - } - }) - return codexMaxBodyBytes -} - -func agenticMemoryStore() memory.Store { - memOnce.Do(func() { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_ENABLED")); v != "" { - if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") { - memStore = nil - return - } - } - - base := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_DIR")) - if base == "" { - if w := util.WritablePath(); w != "" { - base = filepath.Join(w, ".proxypilot", "memory") - } else { - base = filepath.Join(".proxypilot", "memory") - } - } - memStore = memory.NewFileStore(base) - - // Initialize summarizer with default config (uses regex fallback until executor is set) - if agenticLLMSummaryEnabled() { - if fs, ok := memStore.(*memory.FileStore); ok { - config := memory.DefaultSummarizerConfig() - // Start with NoOp executor - will fall back to regex - // Call SetSummarizerExecutor() later with a real executor to enable LLM summarization - summarizer := memory.NewSummarizer(config, memory.NewNoOpSummarizerExecutor()) - fs.SetSummarizer(summarizer) - } - } - }) - return memStore -} - -// SetSummarizerExecutor configures the LLM executor for context summarization. -// Call this from server initialization with the auth manager to enable full LLM summarization. -// Example: middleware.SetSummarizerExecutor(memory.NewPipelineSummarizerExecutor(authManager, providers)) -func SetSummarizerExecutor(executor memory.SummarizerExecutor) { - store := agenticMemoryStore() - if store == nil { - return - } - fs, ok := store.(*memory.FileStore) - if !ok || fs == nil { - return - } - summarizer := fs.GetSummarizer() - if summarizer == nil { - config := memory.DefaultSummarizerConfig() - summarizer = memory.NewSummarizer(config, executor) - fs.SetSummarizer(summarizer) - } else { - // Replace the existing summarizer with one that has the real executor - config := memory.DefaultSummarizerConfig() - fs.SetSummarizer(memory.NewSummarizer(config, executor)) - } -} - -// InitSummarizerWithAuthManager configures LLM-based summarization using the core auth manager. -// Call this after the auth manager is initialized to enable Factory.ai-style context compression. -func InitSummarizerWithAuthManager(manager memory.CoreManagerExecutor, providers []string) { - if manager == nil { - return - } - adapter := memory.NewManagerAuthAdapter(manager) - baseExecutor := memory.NewPipelineSummarizerExecutor(adapter, providers) - // Wrap with model fallback (gemini-3-flash -> gemini-3-flash-preview) - executor := memory.NewSummaryModelFallbackExecutor(baseExecutor) - SetSummarizerExecutor(executor) -} - -// GetSummaryModel returns the configured summary model, defaulting to gemini-3-flash. -func GetSummaryModel() string { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_SUMMARY_MODEL")); v != "" { - return v - } - return memory.DefaultSummaryModel -} - -func agenticSemanticEnabled() bool { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_ENABLED")); v != "" { - if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") { - return false - } - } - return true -} - -func agenticSemanticModel() string { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MODEL")); v != "" { - return v - } - return "embeddinggemma" -} - -func agenticSemanticBaseURL() string { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_BASE_URL")); v != "" { - return v - } - return "http://127.0.0.1:11434" -} - -func agenticSemanticMaxSnips() int { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_SNIPS")); v != "" { - if n, err := strconv.Atoi(v); err == nil && n > 0 { - return n - } - } - return 4 -} - -func agenticSemanticMaxChars() int { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_CHARS")); v != "" { - if n, err := strconv.Atoi(v); err == nil && n > 0 { - return n - } - } - return 3000 -} - -func agenticSemanticClient() *embeddings.OllamaClient { - embedOnce.Do(func() { - embedClient = &embeddings.OllamaClient{ - BaseURL: agenticSemanticBaseURL(), - Model: agenticSemanticModel(), - Client: &http.Client{Timeout: 8 * time.Second}, - } - }) - return embedClient -} - -type semanticEmbedTask struct { - namespace string - session string - texts []string - roles []string - source string -} - -type semanticEmbedQueue struct { - ch chan semanticEmbedTask - fs *memory.FileStore -} - -func agenticSemanticQueue() *semanticEmbedQueue { - embedQueueOnce.Do(func() { - store := agenticMemoryStore() - fs, _ := store.(*memory.FileStore) - embedQueue = &semanticEmbedQueue{ - ch: make(chan semanticEmbedTask, 64), - fs: fs, - } - go embedQueue.run() - }) - return embedQueue -} - -func (q *semanticEmbedQueue) run() { - for task := range q.ch { - if q == nil || q.fs == nil { - continue - } - if len(task.texts) == 0 { - continue - } - client := agenticSemanticClient() - if client == nil { - continue - } - vecs, err := client.Embed(context.Background(), task.texts) - if err != nil || len(vecs) != len(task.texts) { - memory.IncSemanticFailed(len(task.texts)) - time.Sleep(2 * time.Second) - continue - } - records := make([]memory.SemanticRecord, 0, len(task.texts)) - for i := range task.texts { - if len(vecs[i]) == 0 { - continue - } - role := "" - if i < len(task.roles) { - role = task.roles[i] - } - records = append(records, memory.SemanticRecord{ - Role: role, - Text: task.texts[i], - Vec: vecs[i], - Source: task.source, - Session: task.session, - Repo: task.namespace, - }) - } - if len(records) > 0 { - _ = q.fs.AppendSemantic(task.namespace, records) - memory.IncSemanticProcessed(len(records)) - } - } -} - -func enqueueSemanticEmbeds(fs *memory.FileStore, namespace string, session string, texts []string, roles []string, source string) { - if fs == nil || namespace == "" || len(texts) == 0 { - return - } - q := agenticSemanticQueue() - if q == nil { - return - } - if q.fs == nil { - q.fs = fs - } - task := semanticEmbedTask{namespace: namespace, session: session, texts: texts, roles: roles, source: source} - memory.IncSemanticQueued(len(texts)) - select { - case q.ch <- task: - default: - // Drop if the queue is full to avoid backpressure. - memory.IncSemanticDropped(len(texts)) - } -} - -func agenticTodoEnabled() bool { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_TODO_ENABLED")); v != "" { - if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") { - return false - } - } - return true -} - -func agenticTodoMaxChars() int { - v := strings.TrimSpace(os.Getenv("CLIPROXY_TODO_MAX_CHARS")) - if v == "" { - return 4000 - } - n, err := strconv.Atoi(v) - if err != nil { - return 4000 - } - if n < 512 { - return 512 - } - if n > 20_000 { - return 20_000 - } - return n -} - -func agenticMemoryMaxAgeDays() int { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_MAX_AGE_DAYS")); v != "" { - if n, err := strconv.Atoi(v); err == nil { - return n - } - } - return 0 -} - -func agenticMemoryMaxSessions() int { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_MAX_SESSIONS")); v != "" { - if n, err := strconv.Atoi(v); err == nil { - return n - } - } - return 0 -} - -func agenticMemoryMaxBytesPerSession() int64 { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_MAX_BYTES_PER_SESSION")); v != "" { - if n, err := strconv.ParseInt(v, 10, 64); err == nil { - return n - } - } - return 0 -} - -func agenticSemanticMaxNamespaces() int { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_NAMESPACES")); v != "" { - if n, err := strconv.Atoi(v); err == nil { - return n - } - } - return 0 -} - -func agenticSemanticMaxBytesPerNamespace() int64 { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_BYTES_PER_NAMESPACE")); v != "" { - if n, err := strconv.ParseInt(v, 10, 64); err == nil { - return n - } - } - return 0 -} - -func agenticMemoryMaxWritesPerMin() int { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_MAX_WRITES_PER_MIN")); v != "" { - if n, err := strconv.Atoi(v); err == nil { - return n - } - } - return 120 -} - -func agenticSemanticMaxWritesPerMin() int { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_WRITES_PER_MIN")); v != "" { - if n, err := strconv.Atoi(v); err == nil { - return n - } - } - return 120 -} - -func agenticSemanticQueryMaxChars() int { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_QUERY_MAX_CHARS")); v != "" { - if n, err := strconv.Atoi(v); err == nil && n > 0 { - return n - } - } - return 512 -} - -func agenticAnchorAppendOnly() bool { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_ANCHOR_APPEND_ONLY")); v != "" { - if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") { - return false - } - if strings.EqualFold(v, "1") || strings.EqualFold(v, "true") || strings.EqualFold(v, "on") || strings.EqualFold(v, "yes") { - return true - } - } - return true -} - -func agenticAnchorSummaryMaxChars() int { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_ANCHOR_SUMMARY_MAX_CHARS")); v != "" { - if n, err := strconv.Atoi(v); err == nil && n > 0 { - return n - } - } - return 14_000 -} - -// agenticCompressionThreshold returns the context usage ratio at which to trigger LLM compression. -// Default: 0.85 (85% of context window) -func agenticCompressionThreshold() float64 { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_COMPRESSION_THRESHOLD")); v != "" { - if f, err := strconv.ParseFloat(v, 64); err == nil && f > 0 && f < 1 { - return f - } - } - return 0.85 -} - -// agenticMinKeepMessages returns the minimum number of messages to keep during compression. -// This prevents over-aggressive compression that loses context. -// Default: 25 (approximately 12 tool round-trips worth of context) -func agenticMinKeepMessages() int { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_MIN_KEEP_MESSAGES")); v != "" { - if n, err := strconv.Atoi(v); err == nil && n > 0 { - return n - } - } - return 25 -} - -// agenticLLMSummaryEnabled returns whether LLM-based summarization is enabled. -// Default: true -func agenticLLMSummaryEnabled() bool { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_LLM_SUMMARY_ENABLED")); v != "" { - return strings.EqualFold(v, "1") || strings.EqualFold(v, "true") || strings.EqualFold(v, "on") - } - return true -} - -// getModelContextWindow returns the context window size for a model. -func getModelContextWindow(model string) int { - // Try registry first - info := registry.GetGlobalRegistry().GetModelInfo(model, "") - if info != nil && info.ContextLength > 0 { - return info.ContextLength - } - // Fallback based on model name patterns - lowerModel := strings.ToLower(model) - switch { - case strings.Contains(lowerModel, "claude-3.5"), strings.Contains(lowerModel, "claude-3-5"): - return 200000 - case strings.Contains(lowerModel, "claude-3"): - return 200000 - case strings.Contains(lowerModel, "claude"): - return 100000 - case strings.Contains(lowerModel, "gpt-4-turbo"), strings.Contains(lowerModel, "gpt-4o"): - return 128000 - case strings.Contains(lowerModel, "gpt-4"): - return 8192 - case strings.Contains(lowerModel, "gpt-3.5"): - return 16384 - case strings.Contains(lowerModel, "gemini"): - return 1000000 - case strings.Contains(lowerModel, "o1"), strings.Contains(lowerModel, "o3"): - return 200000 - default: - return 100000 - } -} - -// agenticTokenAwareEnabled returns whether token-aware compression is enabled. -// Default: true -func agenticTokenAwareEnabled() bool { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_TOKEN_AWARE_ENABLED")); v != "" { - return strings.EqualFold(v, "1") || strings.EqualFold(v, "true") || strings.EqualFold(v, "on") - } - return true -} - -// agenticReserveTokens returns the number of tokens to reserve for model output. -// Default: 8192 (reasonable output buffer) -func agenticReserveTokens() int { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_RESERVE_TOKENS")); v != "" { - if n, err := strconv.Atoi(v); err == nil && n > 0 { - return n - } - } - return 8192 -} - -// tokenAwareCompressionResult contains the result of token-aware analysis -type tokenAwareCompressionResult struct { - ShouldTrim bool // Whether trimming is needed - CurrentTokens int64 // Estimated current token count - ContextWindow int // Model's context window - TargetTokens int64 // Target token count after trimming - TargetMaxBytes int // Approximate bytes to achieve target tokens - Model string // The model name -} - -// analyzeTokenBudget checks if the request exceeds the token threshold and calculates trimming targets. -// Returns whether trimming is needed and the target byte limit. -func analyzeTokenBudget(body []byte) *tokenAwareCompressionResult { - result := &tokenAwareCompressionResult{ - ShouldTrim: false, - TargetMaxBytes: agenticMaxBodyBytes(), // Default fallback - } - - if !agenticTokenAwareEnabled() { - return result - } - - model := gjson.GetBytes(body, "model").String() - if model == "" { - return result - } - result.Model = model - - // Get context window for this model - contextWindow := getModelContextWindow(model) - result.ContextWindow = contextWindow - - // Estimate current token count - currentTokens, err := executor.EstimateRequestTokens(model, body) - if err != nil { - // Fallback to byte-based heuristic: ~4 chars per token - currentTokens = int64(len(body) / 4) - } - result.CurrentTokens = currentTokens - - // Reserve tokens for output - reserveTokens := agenticReserveTokens() - availableContext := contextWindow - reserveTokens - if availableContext < 1000 { - availableContext = contextWindow / 2 // At least half for input - } - - // Check against threshold - threshold := agenticCompressionThreshold() - maxInputTokens := int64(float64(availableContext) * threshold) - - if currentTokens <= maxInputTokens { - // Under threshold, no trimming needed - result.ShouldTrim = false - return result - } - - // Calculate target tokens (aim for 70% of threshold to give buffer) - targetRatio := threshold * 0.9 // Slightly under threshold - targetTokens := int64(float64(availableContext) * targetRatio) - result.TargetTokens = targetTokens - result.ShouldTrim = true - - // Convert target tokens to approximate byte limit - // Use ~4 bytes per token as heuristic, but be conservative - targetBytes := int(targetTokens * 4) - - // Clamp to reasonable bounds - minBytes := 32 * 1024 // 32KB minimum - if targetBytes < minBytes { - targetBytes = minBytes - } - - // Don't exceed the configured max - maxBytes := agenticMaxBodyBytes() - if targetBytes > maxBytes { - targetBytes = maxBytes - } - - result.TargetMaxBytes = targetBytes - return result -} - -// getTokenAwareMaxBytes returns the maximum body size based on token analysis. -// This is more accurate than the byte-based heuristic in agenticMaxBodyBytesForModel. -func getTokenAwareMaxBytes(body []byte) int { - if !agenticTokenAwareEnabled() { - return agenticMaxBodyBytesForModel(body) - } - - result := analyzeTokenBudget(body) - if result.ShouldTrim { - return result.TargetMaxBytes - } - - // Not over threshold, but still apply model-specific limits - return agenticMaxBodyBytesForModel(body) -} - -func allowMemoryWrite(session string) bool { - limit := agenticMemoryMaxWritesPerMin() - if limit <= 0 || session == "" { - return true - } - limiter := getSessionLimiter(memoryLimiters, session) - rate := float64(limit) / 60.0 - burst := float64(limit) / 6.0 - if burst < 5 { - burst = 5 - } - return limiter.Allow(rate, burst) -} - -func allowSemanticWrite(session string) bool { - limit := agenticSemanticMaxWritesPerMin() - if limit <= 0 || session == "" { - return true - } - limiter := getSessionLimiter(semanticLimiters, session) - rate := float64(limit) / 60.0 - burst := float64(limit) / 6.0 - if burst < 5 { - burst = 5 - } - return limiter.Allow(rate, burst) -} - -func getSessionLimiter(store map[string]*rateLimiter, session string) *rateLimiter { - limiterMu.Lock() - defer limiterMu.Unlock() - if limiter, ok := store[session]; ok { - return limiter - } - limiter := &rateLimiter{} - store[session] = limiter - // best-effort cleanup for stale entries - for key, lim := range store { - if lim == nil { - delete(store, key) - continue - } - lim.mu.Lock() - last := lim.last - lim.mu.Unlock() - if !last.IsZero() && time.Since(last) > 15*time.Minute { - delete(store, key) - } - } - return limiter -} - // CodexPromptBudgetMiddleware trims oversized OpenAI requests coming from Codex CLI. // // Rationale: Codex CLI can accumulate large workspace context and exceed upstream prompt limits. @@ -1025,24 +353,6 @@ func buildAnchorBlock(anchor string) string { return b.String() } -func agenticScaffoldEnabled() bool { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_SCAFFOLD_ENABLED")); v != "" { - if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") { - return false - } - } - return true -} - -func agenticScaffoldAppendOnly() bool { - if v := strings.TrimSpace(os.Getenv("CLIPROXY_SCAFFOLD_APPEND_ONLY")); v != "" { - if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") { - return false - } - } - return true -} - func agenticSpecModeEnabled(req *http.Request, body []byte) bool { if req == nil { return false @@ -1558,13 +868,6 @@ func appendToLastUserText(shape string, body []byte, suffix string, maxBytes int } } -type trimWithMemoryResult struct { - Body []byte - Query string - Dropped []memory.Event - Shape string // "chat", "responses", or "claude" -} - func agenticStoreAndInjectMemory(c *gin.Context, req *http.Request, session string, res *trimWithMemoryResult, maxBytes int) { if req == nil || res == nil { return @@ -1730,7 +1033,7 @@ func agenticMaybePruneMemory() { } func extractPinnedContext(req *http.Request, shape string, body []byte) string { - // Pinned is intended to capture durable “always-on” state: system instructions / policies. + // Pinned is intended to capture durable "always-on" state: system instructions / policies. // For /v1/responses use instructions; for chat prefer first system message. switch shape { case "responses": @@ -1891,872 +1194,79 @@ func extractAgenticSessionKey(req *http.Request, body []byte) string { return "ua_" + hex.EncodeToString(sum[:8]) } -func agenticMaxBodyBytesForModel(body []byte) int { - maxBytes := agenticMaxBodyBytes() - model := gjson.GetBytes(body, "model").String() - if model == "" { - return maxBytes - } - - info := registry.GetGlobalRegistry().GetModelInfo(model, "") - if info == nil || info.ContextLength <= 0 { - return maxBytes - } - - // Heuristic: ~4 bytes per token (UTF-8 text + JSON overhead). We only scale DOWN - // from the global default to avoid upstream "prompt too long" for small-context models. - estimated := info.ContextLength * 4 - const minBytes = 32 * 1024 - if estimated < minBytes { - estimated = minBytes - } - if estimated < maxBytes { - return estimated - } - return maxBytes +func agenticSemanticQueue() *semanticEmbedQueue { + embedQueueOnce.Do(func() { + store := agenticMemoryStore() + fs, _ := store.(*memory.FileStore) + embedQueue = &semanticEmbedQueue{ + ch: make(chan semanticEmbedTask, 64), + fs: fs, + } + go embedQueue.run() + }) + return embedQueue } -// trimOpenAIChatCompletions trims an OpenAI Chat Completions payload by shortening the messages array. -func trimOpenAIChatCompletions(body []byte, maxBytes int, mustKeepTools bool) []byte { - root := gjson.ParseBytes(body) - msgs := root.Get("messages") - if !msgs.IsArray() { - return body - } - arr := msgs.Array() - if len(arr) == 0 { - return body - } - - firstSystem := gjson.Result{} - for i := 0; i < len(arr); i++ { - if strings.EqualFold(arr[i].Get("role").String(), "system") { - firstSystem = arr[i] - break - } - } - - isToolResultMsg := func(m gjson.Result) bool { - role := strings.ToLower(strings.TrimSpace(m.Get("role").String())) - // OpenAI tool result message uses role:"tool". Legacy uses role:"function". - return role == "tool" || role == "function" - } - assistantHasToolCall := func(m gjson.Result) bool { - if !strings.EqualFold(m.Get("role").String(), "assistant") { - return false - } - if tc := m.Get("tool_calls"); tc.Exists() && tc.IsArray() && len(tc.Array()) > 0 { - return true - } - if fc := m.Get("function_call"); fc.Exists() { - return true - } - return false - } - - keep := 20 - perTextLimit := 20_000 - dropTools := false - minKeep := agenticMinKeepMessages() - for keep >= minKeep { - outBody := body - if dropTools && !mustKeepTools { - outBody, _ = sjson.DeleteBytes(outBody, "tools") - outBody, _ = sjson.SetBytes(outBody, "tool_choice", "none") - } - - newMsgs := make([]string, 0, keep+1) - if firstSystem.Exists() { - newMsgs = append(newMsgs, truncateMessageContent(firstSystem.Raw, perTextLimit)) - } - - // Preserve tool call/result adjacency: - // If we keep a tool result message, we must also keep the immediately preceding - // assistant tool call message, otherwise downstream Claude/OpenAI validators can reject. - required := make(map[int]struct{}, 8) - tailKept := 0 - for i := len(arr) - 1; i >= 0 && (tailKept < keep || len(required) > 0); i-- { - if strings.EqualFold(arr[i].Get("role").String(), "system") { - continue - } - - _, req := required[i] - if !req && tailKept >= keep { - continue - } - - newMsgs = append(newMsgs, truncateMessageContent(arr[i].Raw, perTextLimit)) - if !req { - tailKept++ - } else { - delete(required, i) - } - - if isToolResultMsg(arr[i]) { - prev := i - 1 - if prev >= 0 && assistantHasToolCall(arr[prev]) { - required[prev] = struct{}{} - } - } - } - - // Reverse tail section to restore order (system is at index 0 if present). - if firstSystem.Exists() { - reverseStrings(newMsgs[1:]) - } else { - reverseStrings(newMsgs) - } - - out := setJSONArrayBytes(outBody, "messages", newMsgs) - if len(out) <= maxBytes { - return out - } - - // If still too large, reduce kept messages and also tighten per-message text limit. - keep = keep / 2 - if perTextLimit > 5_000 { - perTextLimit = perTextLimit / 2 - } - dropTools = true - } - - return body -} - -// trimOpenAIResponses trims an OpenAI Responses payload by shortening the input array. -func trimOpenAIResponses(body []byte, maxBytes int, mustKeepTools bool) []byte { - root := gjson.ParseBytes(body) - input := root.Get("input") - if !input.Exists() || !input.IsArray() { - return body - } - arr := input.Array() - if len(arr) == 0 { - return body - } - - keep := 30 - perTextLimit := 20_000 - dropTools := false - minKeep := agenticMinKeepMessages() - for keep >= minKeep { - outBody := body - if dropTools && !mustKeepTools { - outBody, _ = sjson.DeleteBytes(outBody, "tools") - outBody, _ = sjson.SetBytes(outBody, "tool_choice", "none") - } - if inst := root.Get("instructions"); inst.Exists() && inst.Type == gjson.String { - s := inst.String() - // Instructions can be validated separately upstream; keep it much shorter than message text. - instructionsLimit := perTextLimit - if instructionsLimit > 2048 { - instructionsLimit = 2048 - } - if len(s) > instructionsLimit { - outBody, _ = sjson.SetBytes(outBody, "instructions", s[:instructionsLimit]+"\n...[truncated]...") - } - } - - // Preserve tool call/result pairs: - // - If we keep a function_call_output, we must also keep the matching function_call (same call_id), - // otherwise downstream Claude-style validators can reject with tool_result/tool_use mismatches. - // - If we've decided to drop tools, remove both calls and outputs from the conversation history. - callByID := make(map[string]string, 16) - for i := 0; i < len(arr); i++ { - item := arr[i] - t := item.Get("type").String() - if t == "" && item.Get("role").String() != "" { - t = "message" - } - if t != "function_call" { - continue - } - callID := item.Get("call_id").String() - if callID == "" { - continue - } - // Keep the first occurrence to avoid reordering/duplication surprises. - if _, ok := callByID[callID]; !ok { - callByID[callID] = item.Raw - } - } - - needCall := make(map[string]struct{}, 8) - newItems := make([]string, 0, keep+8) - kept := 0 - for i := len(arr) - 1; i >= 0 && kept < keep; i-- { - item := arr[i] - t := item.Get("type").String() - if t == "" && item.Get("role").String() != "" { - t = "message" - } - - if dropTools && !mustKeepTools && (t == "function_call" || t == "function_call_output") { - continue - } - - if t == "function_call_output" { - callID := item.Get("call_id").String() - if callID != "" { - needCall[callID] = struct{}{} - } - } - if t == "function_call" { - callID := item.Get("call_id").String() - if callID != "" { - delete(needCall, callID) - } - } - - newItems = append(newItems, truncateMessageContent(item.Raw, perTextLimit)) - kept++ - } - reverseStrings(newItems) - - // Prepend any missing function_call items required by kept outputs. - // If we don't have the matching call, drop the orphan outputs later in the loop by tightening keep/perTextLimit. - if !dropTools && len(needCall) > 0 { - prefix := make([]string, 0, len(needCall)) - for callID := range needCall { - if raw, ok := callByID[callID]; ok { - prefix = append(prefix, raw) - } - } - if len(prefix) > 0 { - // Keep stable order by inserting in original array order. - ordered := make([]string, 0, len(prefix)) - for i := 0; i < len(arr); i++ { - item := arr[i] - if item.Get("type").String() != "function_call" { - continue - } - callID := item.Get("call_id").String() - if callID == "" { - continue - } - if _, ok := needCall[callID]; ok { - if raw, ok2 := callByID[callID]; ok2 { - ordered = append(ordered, raw) - } - } - } - newItems = append(ordered, newItems...) - } - } - - out := setJSONArrayBytes(outBody, "input", newItems) - if len(out) <= maxBytes { - return out - } - - keep = keep / 2 - if perTextLimit > 5_000 { - perTextLimit = perTextLimit / 2 - } - if !mustKeepTools { - dropTools = true - } - } - - return body -} - -func trimOpenAIChatCompletionsWithMemory(body []byte, maxBytes int, mustKeepTools bool) *trimWithMemoryResult { - root := gjson.ParseBytes(body) - msgs := root.Get("messages") - if !msgs.IsArray() { - return &trimWithMemoryResult{Body: body, Shape: "chat"} - } - arr := msgs.Array() - if len(arr) == 0 { - return &trimWithMemoryResult{Body: body, Shape: "chat"} - } - - firstSystem := gjson.Result{} - firstSystemIndex := -1 - for i := 0; i < len(arr); i++ { - if strings.EqualFold(arr[i].Get("role").String(), "system") { - firstSystem = arr[i] - firstSystemIndex = i - break - } - } - - query := extractLastUserTextFromChat(arr) - minKeep := agenticMinKeepMessages() - keep := 20 - perTextLimit := 20_000 - dropTools := false - for keep >= minKeep { - outBody := body - if dropTools && !mustKeepTools { - outBody, _ = sjson.DeleteBytes(outBody, "tools") - outBody, _ = sjson.SetBytes(outBody, "tool_choice", "none") - } - - newMsgs := make([]string, 0, keep+1) - keptIdx := make(map[int]struct{}, keep+2) - if firstSystem.Exists() { - newMsgs = append(newMsgs, truncateMessageContent(firstSystem.Raw, perTextLimit)) - keptIdx[firstSystemIndex] = struct{}{} - } - - isToolResultMsg := func(m gjson.Result) bool { - role := strings.ToLower(strings.TrimSpace(m.Get("role").String())) - return role == "tool" || role == "function" - } - assistantHasToolCall := func(m gjson.Result) bool { - if !strings.EqualFold(m.Get("role").String(), "assistant") { - return false - } - if tc := m.Get("tool_calls"); tc.Exists() && tc.IsArray() && len(tc.Array()) > 0 { - return true - } - if fc := m.Get("function_call"); fc.Exists() { - return true - } - return false - } - - required := make(map[int]struct{}, 8) - tailKept := 0 - for i := len(arr) - 1; i >= 0 && (tailKept < keep || len(required) > 0); i-- { - if strings.EqualFold(arr[i].Get("role").String(), "system") { - continue - } - - _, req := required[i] - if !req && tailKept >= keep { - continue - } - - newMsgs = append(newMsgs, truncateMessageContent(arr[i].Raw, perTextLimit)) - keptIdx[i] = struct{}{} - if !req { - tailKept++ - } else { - delete(required, i) - } - - if isToolResultMsg(arr[i]) { - prev := i - 1 - if prev >= 0 && assistantHasToolCall(arr[prev]) { - required[prev] = struct{}{} - } - } - } - - if firstSystem.Exists() { - reverseStrings(newMsgs[1:]) - } else { - reverseStrings(newMsgs) - } - - out := setJSONArrayBytes(outBody, "messages", newMsgs) - if len(out) <= maxBytes { - dropped := collectDroppedChat(arr, keptIdx) - return &trimWithMemoryResult{Body: out, Query: query, Dropped: dropped, Shape: "chat"} - } - - keep = keep / 2 - if perTextLimit > 5_000 { - perTextLimit = perTextLimit / 2 - } - dropTools = true - } - return &trimWithMemoryResult{Body: body, Query: query, Shape: "chat"} -} - -func collectDroppedChat(arr []gjson.Result, kept map[int]struct{}) []memory.Event { - out := make([]memory.Event, 0, 32) - for i := 0; i < len(arr); i++ { - if _, ok := kept[i]; ok { +func (q *semanticEmbedQueue) run() { + for task := range q.ch { + if q == nil || q.fs == nil { continue } - role := arr[i].Get("role").String() - txt := extractTextFromChatMessage(arr[i]) - if strings.TrimSpace(txt) == "" { + if len(task.texts) == 0 { continue } - out = append(out, memory.Event{Kind: "dropped_chat", Role: role, Text: txt}) - } - return out -} - -func extractLastUserTextFromChat(arr []gjson.Result) string { - for i := len(arr) - 1; i >= 0; i-- { - if !strings.EqualFold(arr[i].Get("role").String(), "user") { + client := agenticSemanticClient() + if client == nil { continue } - txt := extractTextFromChatMessage(arr[i]) - if strings.TrimSpace(txt) != "" { - return txt + vecs, err := client.Embed(context.Background(), task.texts) + if err != nil || len(vecs) != len(task.texts) { + memory.IncSemanticFailed(len(task.texts)) + time.Sleep(2 * time.Second) + continue } - } - return "" -} - -func extractTextFromChatMessage(msg gjson.Result) string { - content := msg.Get("content") - switch { - case content.Type == gjson.String: - return content.String() - case content.IsArray(): - var b strings.Builder - for _, it := range content.Array() { - // Skip thinking/reasoning content blocks - partType := it.Get("type").String() - if partType == "thinking" || partType == "reasoning" { + records := make([]memory.SemanticRecord, 0, len(task.texts)) + for i := range task.texts { + if len(vecs[i]) == 0 { continue } - if t := it.Get("text"); t.Exists() && t.Type == gjson.String { - if b.Len() > 0 { - b.WriteString("\n") - } - b.WriteString(t.String()) + role := "" + if i < len(task.roles) { + role = task.roles[i] } + records = append(records, memory.SemanticRecord{ + Role: role, + Text: task.texts[i], + Vec: vecs[i], + Source: task.source, + Session: task.session, + Repo: task.namespace, + }) } - return b.String() + if len(records) > 0 { + _ = q.fs.AppendSemantic(task.namespace, records) + memory.IncSemanticProcessed(len(records)) + } + } +} + +func enqueueSemanticEmbeds(fs *memory.FileStore, namespace string, session string, texts []string, roles []string, source string) { + if fs == nil || namespace == "" || len(texts) == 0 { + return + } + q := agenticSemanticQueue() + if q == nil { + return + } + if q.fs == nil { + q.fs = fs + } + task := semanticEmbedTask{namespace: namespace, session: session, texts: texts, roles: roles, source: source} + memory.IncSemanticQueued(len(texts)) + select { + case q.ch <- task: default: - return "" - } -} - -func trimOpenAIResponsesWithMemory(body []byte, maxBytes int, mustKeepTools bool) *trimWithMemoryResult { - root := gjson.ParseBytes(body) - input := root.Get("input") - if !input.Exists() || !input.IsArray() { - return &trimWithMemoryResult{Body: body, Shape: "responses"} - } - arr := input.Array() - if len(arr) == 0 { - return &trimWithMemoryResult{Body: body, Shape: "responses"} - } - - query := extractLastUserTextFromResponses(arr) - minKeep := agenticMinKeepMessages() - keep := 30 - perTextLimit := 20_000 - dropTools := false - for keep >= minKeep { - outBody := body - if dropTools && !mustKeepTools { - outBody, _ = sjson.DeleteBytes(outBody, "tools") - outBody, _ = sjson.SetBytes(outBody, "tool_choice", "none") - } - if inst := root.Get("instructions"); inst.Exists() && inst.Type == gjson.String { - s := inst.String() - instructionsLimit := perTextLimit - if instructionsLimit > 2048 { - instructionsLimit = 2048 - } - if len(s) > instructionsLimit { - outBody, _ = sjson.SetBytes(outBody, "instructions", s[:instructionsLimit]+"\n...[truncated]...") - } - } - - callByID := make(map[string]gjson.Result, 16) - for i := 0; i < len(arr); i++ { - item := arr[i] - t := item.Get("type").String() - if t == "" && item.Get("role").String() != "" { - t = "message" - } - if t != "function_call" { - continue - } - callID := item.Get("call_id").String() - if callID == "" { - continue - } - if _, ok := callByID[callID]; !ok { - callByID[callID] = item - } - } - - needCall := make(map[string]struct{}, 8) - newItems := make([]string, 0, keep+8) - keptIdx := make(map[int]struct{}, keep+16) - kept := 0 - for i := len(arr) - 1; i >= 0 && kept < keep; i-- { - item := arr[i] - t := item.Get("type").String() - if t == "" && item.Get("role").String() != "" { - t = "message" - } - - if dropTools && !mustKeepTools && (t == "function_call" || t == "function_call_output") { - continue - } - if t == "function_call_output" { - callID := item.Get("call_id").String() - if callID != "" { - needCall[callID] = struct{}{} - } - } - if t == "function_call" { - callID := item.Get("call_id").String() - if callID != "" { - delete(needCall, callID) - } - } - - newItems = append(newItems, truncateMessageContent(item.Raw, perTextLimit)) - keptIdx[i] = struct{}{} - kept++ - } - reverseStrings(newItems) - - // Prepend missing function_call items required by kept outputs. - if !dropTools && len(needCall) > 0 { - ordered := make([]string, 0, len(needCall)) - for i := 0; i < len(arr); i++ { - item := arr[i] - if item.Get("type").String() != "function_call" { - continue - } - callID := item.Get("call_id").String() - if callID == "" { - continue - } - if _, ok := needCall[callID]; ok { - if call, ok2 := callByID[callID]; ok2 { - ordered = append(ordered, call.Raw) - keptIdx[i] = struct{}{} - } - } - } - if len(ordered) > 0 { - newItems = append(ordered, newItems...) - } - } - - out := setJSONArrayBytes(outBody, "input", newItems) - if len(out) <= maxBytes { - dropped := collectDroppedResponses(arr, keptIdx) - return &trimWithMemoryResult{Body: out, Query: query, Dropped: dropped, Shape: "responses"} - } - - keep = keep / 2 - if perTextLimit > 5_000 { - perTextLimit = perTextLimit / 2 - } - if !mustKeepTools { - dropTools = true - } - } - - return &trimWithMemoryResult{Body: body, Query: query, Shape: "responses"} -} - -func collectDroppedResponses(arr []gjson.Result, kept map[int]struct{}) []memory.Event { - out := make([]memory.Event, 0, 64) - for i := 0; i < len(arr); i++ { - if _, ok := kept[i]; ok { - continue - } - item := arr[i] - t := item.Get("type").String() - role := item.Get("role").String() - txt := extractTextFromResponsesItem(item) - if strings.TrimSpace(txt) == "" { - continue - } - out = append(out, memory.Event{Kind: "dropped_responses", Type: t, Role: role, Text: txt}) - } - return out -} - -func extractLastUserTextFromResponses(arr []gjson.Result) string { - for i := len(arr) - 1; i >= 0; i-- { - item := arr[i] - role := item.Get("role").String() - if !strings.EqualFold(role, "user") { - continue - } - txt := extractTextFromResponsesItem(item) - if strings.TrimSpace(txt) != "" { - return txt - } - } - return "" -} - -func extractTextFromResponsesItem(item gjson.Result) string { - // Typical Responses input item: {role:"user", content:[{type:"input_text", text:"..."}]} - content := item.Get("content") - if content.IsArray() { - var b strings.Builder - for _, part := range content.Array() { - // Skip thinking/reasoning content blocks - partType := part.Get("type").String() - if partType == "thinking" || partType == "reasoning" { - continue - } - text := part.Get("text") - if !text.Exists() || text.Type != gjson.String { - continue - } - if b.Len() > 0 { - b.WriteString("\n") - } - b.WriteString(text.String()) - } - return b.String() - } - if content.Type == gjson.String { - return content.String() - } - if t := item.Get("text"); t.Exists() && t.Type == gjson.String { - return t.String() - } - // function_call_output has output or content; capture raw-ish summary. - if out := item.Get("output"); out.Exists() && out.Type == gjson.String { - return out.String() - } - return "" -} - -func truncateMessageContent(msgRaw string, maxTextChars int) string { - msg := msgRaw - if maxTextChars <= 0 { - return msg - } - - content := gjson.Get(msg, "content") - switch { - case content.Type == gjson.String: - s := content.String() - if len(s) > maxTextChars { - s = s[:maxTextChars] + "\n...[truncated]..." - msg, _ = sjson.Set(msg, "content", s) - } - return msg - case content.IsArray(): - items := content.Array() - for i := 0; i < len(items); i++ { - // OpenAI chat: {type:"text", text:"..."} or Responses: {type:"input_text", text:"..."} - text := items[i].Get("text") - if !text.Exists() || text.Type != gjson.String { - continue - } - s := text.String() - if len(s) > maxTextChars { - s = s[:maxTextChars] + "\n...[truncated]..." - msg, _ = sjson.Set(msg, "content."+strconv.Itoa(i)+".text", s) - } - } - return msg - default: - return msg - } -} - -func setJSONArrayBytes(body []byte, key string, rawItems []string) []byte { - out := body - out, _ = sjson.SetRawBytes(out, key, []byte("[]")) - for i := range rawItems { - out, _ = sjson.SetRawBytes(out, key+".-1", []byte(rawItems[i])) - } - return out -} - -func reverseStrings(items []string) { - for i, j := 0, len(items)-1; i < j; i, j = i+1, j-1 { - items[i], items[j] = items[j], items[i] - } -} - -// trimClaudeMessagesWithMemory trims a Claude Messages API payload by shortening the messages array. -// Claude uses a similar structure to OpenAI chat but with content blocks that can include -// tool_use, tool_result, text, and other types. -func trimClaudeMessagesWithMemory(body []byte, maxBytes int, mustKeepTools bool) *trimWithMemoryResult { - root := gjson.ParseBytes(body) - msgs := root.Get("messages") - if !msgs.IsArray() { - return &trimWithMemoryResult{Body: body, Shape: "claude"} - } - arr := msgs.Array() - if len(arr) == 0 { - return &trimWithMemoryResult{Body: body, Shape: "claude"} - } - - query := extractLastUserTextFromClaude(arr) - minKeep := agenticMinKeepMessages() - keep := 20 - perTextLimit := 20_000 - dropTools := false - for keep >= minKeep { - outBody := body - if dropTools && !mustKeepTools { - outBody, _ = sjson.DeleteBytes(outBody, "tools") - outBody, _ = sjson.SetBytes(outBody, "tool_choice", map[string]any{"type": "none"}) - } - - newMsgs := make([]string, 0, keep+1) - keptIdx := make(map[int]struct{}, keep+2) - - // Claude Messages API helper functions - isToolResultMsg := func(m gjson.Result) bool { - content := m.Get("content") - if !content.IsArray() { - return false - } - for _, part := range content.Array() { - if part.Get("type").String() == "tool_result" { - return true - } - } - return false - } - assistantHasToolUse := func(m gjson.Result) bool { - if !strings.EqualFold(m.Get("role").String(), "assistant") { - return false - } - content := m.Get("content") - if !content.IsArray() { - return false - } - for _, part := range content.Array() { - if part.Get("type").String() == "tool_use" { - return true - } - } - return false - } - - required := make(map[int]struct{}, 8) - tailKept := 0 - for i := len(arr) - 1; i >= 0 && (tailKept < keep || len(required) > 0); i-- { - _, req := required[i] - if !req && tailKept >= keep { - continue - } - - newMsgs = append(newMsgs, truncateClaudeMessageContent(arr[i].Raw, perTextLimit)) - keptIdx[i] = struct{}{} - if !req { - tailKept++ - } else { - delete(required, i) - } - - // If this is a tool result message, ensure we keep the preceding assistant message with tool_use - if isToolResultMsg(arr[i]) { - prev := i - 1 - if prev >= 0 && assistantHasToolUse(arr[prev]) { - required[prev] = struct{}{} - } - } - } - - reverseStrings(newMsgs) - - out := setJSONArrayBytes(outBody, "messages", newMsgs) - if len(out) <= maxBytes { - dropped := collectDroppedClaude(arr, keptIdx) - return &trimWithMemoryResult{Body: out, Query: query, Dropped: dropped, Shape: "claude"} - } - - keep = keep / 2 - if perTextLimit > 5_000 { - perTextLimit = perTextLimit / 2 - } - dropTools = true - } - return &trimWithMemoryResult{Body: body, Query: query, Shape: "claude"} -} - -func collectDroppedClaude(arr []gjson.Result, kept map[int]struct{}) []memory.Event { - out := make([]memory.Event, 0, 32) - for i := 0; i < len(arr); i++ { - if _, ok := kept[i]; ok { - continue - } - role := arr[i].Get("role").String() - txt := extractTextFromClaudeMessage(arr[i]) - if strings.TrimSpace(txt) == "" { - continue - } - out = append(out, memory.Event{Kind: "dropped_claude", Role: role, Text: txt}) - } - return out -} - -func extractLastUserTextFromClaude(arr []gjson.Result) string { - for i := len(arr) - 1; i >= 0; i-- { - if !strings.EqualFold(arr[i].Get("role").String(), "user") { - continue - } - txt := extractTextFromClaudeMessage(arr[i]) - if strings.TrimSpace(txt) != "" { - return txt - } - } - return "" -} - -func extractTextFromClaudeMessage(msg gjson.Result) string { - content := msg.Get("content") - switch { - case content.Type == gjson.String: - return content.String() - case content.IsArray(): - var b strings.Builder - for _, part := range content.Array() { - partType := part.Get("type").String() - // Claude text blocks use type:"text" with text field - // Skip thinking blocks (type:"thinking") to avoid leaking internal reasoning - if partType == "text" { - if t := part.Get("text"); t.Exists() && t.Type == gjson.String { - if b.Len() > 0 { - b.WriteString("\n") - } - b.WriteString(t.String()) - } - } - } - return b.String() - default: - return "" - } -} - -func truncateClaudeMessageContent(msgRaw string, maxTextChars int) string { - msg := msgRaw - if maxTextChars <= 0 { - return msg - } - - content := gjson.Get(msg, "content") - switch { - case content.Type == gjson.String: - s := content.String() - if len(s) > maxTextChars { - s = s[:maxTextChars] + "\n...[truncated]..." - msg, _ = sjson.Set(msg, "content", s) - } - return msg - case content.IsArray(): - items := content.Array() - for i := 0; i < len(items); i++ { - partType := items[i].Get("type").String() - // Only truncate text blocks - if partType != "text" { - continue - } - text := items[i].Get("text") - if !text.Exists() || text.Type != gjson.String { - continue - } - s := text.String() - if len(s) > maxTextChars { - s = s[:maxTextChars] + "\n...[truncated]..." - msg, _ = sjson.Set(msg, "content."+strconv.Itoa(i)+".text", s) - } - } - return msg - default: - return msg + // Drop if the queue is full to avoid backpressure. + memory.IncSemanticDropped(len(texts)) } }