mirror of
https://github.com/Finesssee/ProxyPilot.git
synced 2026-11-20 20:34:36 +00:00
refactor(middleware): split codex_prompt_budget.go into logical modules
Split the 2,762 line codex_prompt_budget.go into focused modules: - codex_prompt_budget.go (1,272 lines): Main middleware, scaffold injection - codex_budget_types.go (498 lines): Constants, package vars, config functions - codex_budget_calculator.go (944 lines): Token analysis, trim functions Co-Authored-By: Claude (gemini-claude-opus-4-5-thinking) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,944 @@
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/router-for-me/CLIProxyAPI/v6/internal/memory"
|
||||
"github.com/router-for-me/CLIProxyAPI/v6/internal/registry"
|
||||
"github.com/router-for-me/CLIProxyAPI/v6/internal/runtime/executor"
|
||||
"github.com/tidwall/gjson"
|
||||
"github.com/tidwall/sjson"
|
||||
)
|
||||
|
||||
// tokenAwareCompressionResult contains the result of token-aware analysis
|
||||
type tokenAwareCompressionResult struct {
|
||||
ShouldTrim bool // Whether trimming is needed
|
||||
CurrentTokens int64 // Estimated current token count
|
||||
ContextWindow int // Model's context window
|
||||
TargetTokens int64 // Target token count after trimming
|
||||
TargetMaxBytes int // Approximate bytes to achieve target tokens
|
||||
Model string // The model name
|
||||
}
|
||||
|
||||
// analyzeTokenBudget checks if the request exceeds the token threshold and calculates trimming targets.
|
||||
func analyzeTokenBudget(body []byte) *tokenAwareCompressionResult {
|
||||
result := &tokenAwareCompressionResult{
|
||||
ShouldTrim: false,
|
||||
TargetMaxBytes: agenticMaxBodyBytes(),
|
||||
}
|
||||
|
||||
if !agenticTokenAwareEnabled() {
|
||||
return result
|
||||
}
|
||||
|
||||
model := gjson.GetBytes(body, "model").String()
|
||||
if model == "" {
|
||||
return result
|
||||
}
|
||||
result.Model = model
|
||||
|
||||
contextWindow := getModelContextWindow(model)
|
||||
result.ContextWindow = contextWindow
|
||||
|
||||
currentTokens, err := executor.EstimateRequestTokens(model, body)
|
||||
if err != nil {
|
||||
currentTokens = int64(len(body) / 4)
|
||||
}
|
||||
result.CurrentTokens = currentTokens
|
||||
|
||||
reserveTokens := agenticReserveTokens()
|
||||
availableContext := contextWindow - reserveTokens
|
||||
if availableContext < 1000 {
|
||||
availableContext = contextWindow / 2
|
||||
}
|
||||
|
||||
threshold := agenticCompressionThreshold()
|
||||
maxInputTokens := int64(float64(availableContext) * threshold)
|
||||
|
||||
if currentTokens <= maxInputTokens {
|
||||
result.ShouldTrim = false
|
||||
return result
|
||||
}
|
||||
|
||||
targetRatio := threshold * 0.9
|
||||
targetTokens := int64(float64(availableContext) * targetRatio)
|
||||
result.TargetTokens = targetTokens
|
||||
result.ShouldTrim = true
|
||||
|
||||
targetBytes := int(targetTokens * 4)
|
||||
|
||||
minBytes := 32 * 1024
|
||||
if targetBytes < minBytes {
|
||||
targetBytes = minBytes
|
||||
}
|
||||
|
||||
maxBytes := agenticMaxBodyBytes()
|
||||
if targetBytes > maxBytes {
|
||||
targetBytes = maxBytes
|
||||
}
|
||||
|
||||
result.TargetMaxBytes = targetBytes
|
||||
return result
|
||||
}
|
||||
|
||||
// getTokenAwareMaxBytes returns the maximum body size based on token analysis.
|
||||
func getTokenAwareMaxBytes(body []byte) int {
|
||||
if !agenticTokenAwareEnabled() {
|
||||
return agenticMaxBodyBytesForModel(body)
|
||||
}
|
||||
|
||||
result := analyzeTokenBudget(body)
|
||||
if result.ShouldTrim {
|
||||
return result.TargetMaxBytes
|
||||
}
|
||||
|
||||
return agenticMaxBodyBytesForModel(body)
|
||||
}
|
||||
|
||||
func agenticMaxBodyBytesForModel(body []byte) int {
|
||||
maxBytes := agenticMaxBodyBytes()
|
||||
model := gjson.GetBytes(body, "model").String()
|
||||
if model == "" {
|
||||
return maxBytes
|
||||
}
|
||||
|
||||
info := registry.GetGlobalRegistry().GetModelInfo(model, "")
|
||||
if info == nil || info.ContextLength <= 0 {
|
||||
return maxBytes
|
||||
}
|
||||
|
||||
estimated := info.ContextLength * 4
|
||||
const minBytes = 32 * 1024
|
||||
if estimated < minBytes {
|
||||
estimated = minBytes
|
||||
}
|
||||
if estimated < maxBytes {
|
||||
return estimated
|
||||
}
|
||||
return maxBytes
|
||||
}
|
||||
|
||||
// trimWithMemoryResult holds the result of trimming with memory extraction.
|
||||
type trimWithMemoryResult struct {
|
||||
Body []byte
|
||||
Query string
|
||||
Dropped []memory.Event
|
||||
Shape string // "chat", "responses", or "claude"
|
||||
}
|
||||
|
||||
// trimOpenAIChatCompletions trims an OpenAI Chat Completions payload by shortening the messages array.
|
||||
func trimOpenAIChatCompletions(body []byte, maxBytes int, mustKeepTools bool) []byte {
|
||||
root := gjson.ParseBytes(body)
|
||||
msgs := root.Get("messages")
|
||||
if !msgs.IsArray() {
|
||||
return body
|
||||
}
|
||||
arr := msgs.Array()
|
||||
if len(arr) == 0 {
|
||||
return body
|
||||
}
|
||||
|
||||
firstSystem := gjson.Result{}
|
||||
for i := 0; i < len(arr); i++ {
|
||||
if strings.EqualFold(arr[i].Get("role").String(), "system") {
|
||||
firstSystem = arr[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
isToolResultMsg := func(m gjson.Result) bool {
|
||||
role := strings.ToLower(strings.TrimSpace(m.Get("role").String()))
|
||||
return role == "tool" || role == "function"
|
||||
}
|
||||
assistantHasToolCall := func(m gjson.Result) bool {
|
||||
if !strings.EqualFold(m.Get("role").String(), "assistant") {
|
||||
return false
|
||||
}
|
||||
if tc := m.Get("tool_calls"); tc.Exists() && tc.IsArray() && len(tc.Array()) > 0 {
|
||||
return true
|
||||
}
|
||||
if fc := m.Get("function_call"); fc.Exists() {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
keep := 20
|
||||
perTextLimit := 20_000
|
||||
dropTools := false
|
||||
minKeep := agenticMinKeepMessages()
|
||||
for keep >= minKeep {
|
||||
outBody := body
|
||||
if dropTools && !mustKeepTools {
|
||||
outBody, _ = sjson.DeleteBytes(outBody, "tools")
|
||||
outBody, _ = sjson.SetBytes(outBody, "tool_choice", "none")
|
||||
}
|
||||
|
||||
newMsgs := make([]string, 0, keep+1)
|
||||
if firstSystem.Exists() {
|
||||
newMsgs = append(newMsgs, truncateMessageContent(firstSystem.Raw, perTextLimit))
|
||||
}
|
||||
|
||||
required := make(map[int]struct{}, 8)
|
||||
tailKept := 0
|
||||
for i := len(arr) - 1; i >= 0 && (tailKept < keep || len(required) > 0); i-- {
|
||||
if strings.EqualFold(arr[i].Get("role").String(), "system") {
|
||||
continue
|
||||
}
|
||||
|
||||
_, req := required[i]
|
||||
if !req && tailKept >= keep {
|
||||
continue
|
||||
}
|
||||
|
||||
newMsgs = append(newMsgs, truncateMessageContent(arr[i].Raw, perTextLimit))
|
||||
if !req {
|
||||
tailKept++
|
||||
} else {
|
||||
delete(required, i)
|
||||
}
|
||||
|
||||
if isToolResultMsg(arr[i]) {
|
||||
prev := i - 1
|
||||
if prev >= 0 && assistantHasToolCall(arr[prev]) {
|
||||
required[prev] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if firstSystem.Exists() {
|
||||
reverseStrings(newMsgs[1:])
|
||||
} else {
|
||||
reverseStrings(newMsgs)
|
||||
}
|
||||
|
||||
out := setJSONArrayBytes(outBody, "messages", newMsgs)
|
||||
if len(out) <= maxBytes {
|
||||
return out
|
||||
}
|
||||
|
||||
keep = keep / 2
|
||||
if perTextLimit > 5_000 {
|
||||
perTextLimit = perTextLimit / 2
|
||||
}
|
||||
dropTools = true
|
||||
}
|
||||
|
||||
return body
|
||||
}
|
||||
|
||||
// trimOpenAIResponses trims an OpenAI Responses payload by shortening the input array.
|
||||
func trimOpenAIResponses(body []byte, maxBytes int, mustKeepTools bool) []byte {
|
||||
root := gjson.ParseBytes(body)
|
||||
input := root.Get("input")
|
||||
if !input.Exists() || !input.IsArray() {
|
||||
return body
|
||||
}
|
||||
arr := input.Array()
|
||||
if len(arr) == 0 {
|
||||
return body
|
||||
}
|
||||
|
||||
keep := 30
|
||||
perTextLimit := 20_000
|
||||
dropTools := false
|
||||
minKeep := agenticMinKeepMessages()
|
||||
for keep >= minKeep {
|
||||
outBody := body
|
||||
if dropTools && !mustKeepTools {
|
||||
outBody, _ = sjson.DeleteBytes(outBody, "tools")
|
||||
outBody, _ = sjson.SetBytes(outBody, "tool_choice", "none")
|
||||
}
|
||||
if inst := root.Get("instructions"); inst.Exists() && inst.Type == gjson.String {
|
||||
s := inst.String()
|
||||
instructionsLimit := perTextLimit
|
||||
if instructionsLimit > 2048 {
|
||||
instructionsLimit = 2048
|
||||
}
|
||||
if len(s) > instructionsLimit {
|
||||
outBody, _ = sjson.SetBytes(outBody, "instructions", s[:instructionsLimit]+"\n...[truncated]...")
|
||||
}
|
||||
}
|
||||
|
||||
callByID := make(map[string]string, 16)
|
||||
for i := 0; i < len(arr); i++ {
|
||||
item := arr[i]
|
||||
t := item.Get("type").String()
|
||||
if t == "" && item.Get("role").String() != "" {
|
||||
t = "message"
|
||||
}
|
||||
if t != "function_call" {
|
||||
continue
|
||||
}
|
||||
callID := item.Get("call_id").String()
|
||||
if callID == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := callByID[callID]; !ok {
|
||||
callByID[callID] = item.Raw
|
||||
}
|
||||
}
|
||||
|
||||
needCall := make(map[string]struct{}, 8)
|
||||
newItems := make([]string, 0, keep+8)
|
||||
kept := 0
|
||||
for i := len(arr) - 1; i >= 0 && kept < keep; i-- {
|
||||
item := arr[i]
|
||||
t := item.Get("type").String()
|
||||
if t == "" && item.Get("role").String() != "" {
|
||||
t = "message"
|
||||
}
|
||||
|
||||
if dropTools && !mustKeepTools && (t == "function_call" || t == "function_call_output") {
|
||||
continue
|
||||
}
|
||||
|
||||
if t == "function_call_output" {
|
||||
callID := item.Get("call_id").String()
|
||||
if callID != "" {
|
||||
needCall[callID] = struct{}{}
|
||||
}
|
||||
}
|
||||
if t == "function_call" {
|
||||
callID := item.Get("call_id").String()
|
||||
if callID != "" {
|
||||
delete(needCall, callID)
|
||||
}
|
||||
}
|
||||
|
||||
newItems = append(newItems, truncateMessageContent(item.Raw, perTextLimit))
|
||||
kept++
|
||||
}
|
||||
reverseStrings(newItems)
|
||||
|
||||
if !dropTools && len(needCall) > 0 {
|
||||
prefix := make([]string, 0, len(needCall))
|
||||
for callID := range needCall {
|
||||
if raw, ok := callByID[callID]; ok {
|
||||
prefix = append(prefix, raw)
|
||||
}
|
||||
}
|
||||
if len(prefix) > 0 {
|
||||
ordered := make([]string, 0, len(prefix))
|
||||
for i := 0; i < len(arr); i++ {
|
||||
item := arr[i]
|
||||
if item.Get("type").String() != "function_call" {
|
||||
continue
|
||||
}
|
||||
callID := item.Get("call_id").String()
|
||||
if callID == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := needCall[callID]; ok {
|
||||
if raw, ok2 := callByID[callID]; ok2 {
|
||||
ordered = append(ordered, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
newItems = append(ordered, newItems...)
|
||||
}
|
||||
}
|
||||
|
||||
out := setJSONArrayBytes(outBody, "input", newItems)
|
||||
if len(out) <= maxBytes {
|
||||
return out
|
||||
}
|
||||
|
||||
keep = keep / 2
|
||||
if perTextLimit > 5_000 {
|
||||
perTextLimit = perTextLimit / 2
|
||||
}
|
||||
if !mustKeepTools {
|
||||
dropTools = true
|
||||
}
|
||||
}
|
||||
|
||||
return body
|
||||
}
|
||||
|
||||
func trimOpenAIChatCompletionsWithMemory(body []byte, maxBytes int, mustKeepTools bool) *trimWithMemoryResult {
|
||||
root := gjson.ParseBytes(body)
|
||||
msgs := root.Get("messages")
|
||||
if !msgs.IsArray() {
|
||||
return &trimWithMemoryResult{Body: body, Shape: "chat"}
|
||||
}
|
||||
arr := msgs.Array()
|
||||
if len(arr) == 0 {
|
||||
return &trimWithMemoryResult{Body: body, Shape: "chat"}
|
||||
}
|
||||
|
||||
firstSystem := gjson.Result{}
|
||||
firstSystemIndex := -1
|
||||
for i := 0; i < len(arr); i++ {
|
||||
if strings.EqualFold(arr[i].Get("role").String(), "system") {
|
||||
firstSystem = arr[i]
|
||||
firstSystemIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
query := extractLastUserTextFromChat(arr)
|
||||
minKeep := agenticMinKeepMessages()
|
||||
keep := 20
|
||||
perTextLimit := 20_000
|
||||
dropTools := false
|
||||
for keep >= minKeep {
|
||||
outBody := body
|
||||
if dropTools && !mustKeepTools {
|
||||
outBody, _ = sjson.DeleteBytes(outBody, "tools")
|
||||
outBody, _ = sjson.SetBytes(outBody, "tool_choice", "none")
|
||||
}
|
||||
|
||||
newMsgs := make([]string, 0, keep+1)
|
||||
keptIdx := make(map[int]struct{}, keep+2)
|
||||
if firstSystem.Exists() {
|
||||
newMsgs = append(newMsgs, truncateMessageContent(firstSystem.Raw, perTextLimit))
|
||||
keptIdx[firstSystemIndex] = struct{}{}
|
||||
}
|
||||
|
||||
isToolResultMsg := func(m gjson.Result) bool {
|
||||
role := strings.ToLower(strings.TrimSpace(m.Get("role").String()))
|
||||
return role == "tool" || role == "function"
|
||||
}
|
||||
assistantHasToolCall := func(m gjson.Result) bool {
|
||||
if !strings.EqualFold(m.Get("role").String(), "assistant") {
|
||||
return false
|
||||
}
|
||||
if tc := m.Get("tool_calls"); tc.Exists() && tc.IsArray() && len(tc.Array()) > 0 {
|
||||
return true
|
||||
}
|
||||
if fc := m.Get("function_call"); fc.Exists() {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
required := make(map[int]struct{}, 8)
|
||||
tailKept := 0
|
||||
for i := len(arr) - 1; i >= 0 && (tailKept < keep || len(required) > 0); i-- {
|
||||
if strings.EqualFold(arr[i].Get("role").String(), "system") {
|
||||
continue
|
||||
}
|
||||
|
||||
_, req := required[i]
|
||||
if !req && tailKept >= keep {
|
||||
continue
|
||||
}
|
||||
|
||||
newMsgs = append(newMsgs, truncateMessageContent(arr[i].Raw, perTextLimit))
|
||||
keptIdx[i] = struct{}{}
|
||||
if !req {
|
||||
tailKept++
|
||||
} else {
|
||||
delete(required, i)
|
||||
}
|
||||
|
||||
if isToolResultMsg(arr[i]) {
|
||||
prev := i - 1
|
||||
if prev >= 0 && assistantHasToolCall(arr[prev]) {
|
||||
required[prev] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if firstSystem.Exists() {
|
||||
reverseStrings(newMsgs[1:])
|
||||
} else {
|
||||
reverseStrings(newMsgs)
|
||||
}
|
||||
|
||||
out := setJSONArrayBytes(outBody, "messages", newMsgs)
|
||||
if len(out) <= maxBytes {
|
||||
dropped := collectDroppedChat(arr, keptIdx)
|
||||
return &trimWithMemoryResult{Body: out, Query: query, Dropped: dropped, Shape: "chat"}
|
||||
}
|
||||
|
||||
keep = keep / 2
|
||||
if perTextLimit > 5_000 {
|
||||
perTextLimit = perTextLimit / 2
|
||||
}
|
||||
dropTools = true
|
||||
}
|
||||
return &trimWithMemoryResult{Body: body, Query: query, Shape: "chat"}
|
||||
}
|
||||
|
||||
func collectDroppedChat(arr []gjson.Result, kept map[int]struct{}) []memory.Event {
|
||||
out := make([]memory.Event, 0, 32)
|
||||
for i := 0; i < len(arr); i++ {
|
||||
if _, ok := kept[i]; ok {
|
||||
continue
|
||||
}
|
||||
role := arr[i].Get("role").String()
|
||||
txt := extractTextFromChatMessage(arr[i])
|
||||
if strings.TrimSpace(txt) == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, memory.Event{Kind: "dropped_chat", Role: role, Text: txt})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func extractLastUserTextFromChat(arr []gjson.Result) string {
|
||||
for i := len(arr) - 1; i >= 0; i-- {
|
||||
if !strings.EqualFold(arr[i].Get("role").String(), "user") {
|
||||
continue
|
||||
}
|
||||
txt := extractTextFromChatMessage(arr[i])
|
||||
if strings.TrimSpace(txt) != "" {
|
||||
return txt
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractTextFromChatMessage(msg gjson.Result) string {
|
||||
content := msg.Get("content")
|
||||
switch {
|
||||
case content.Type == gjson.String:
|
||||
return content.String()
|
||||
case content.IsArray():
|
||||
var b strings.Builder
|
||||
for _, it := range content.Array() {
|
||||
partType := it.Get("type").String()
|
||||
if partType == "thinking" || partType == "reasoning" {
|
||||
continue
|
||||
}
|
||||
if t := it.Get("text"); t.Exists() && t.Type == gjson.String {
|
||||
if b.Len() > 0 {
|
||||
b.WriteString("\n")
|
||||
}
|
||||
b.WriteString(t.String())
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func trimOpenAIResponsesWithMemory(body []byte, maxBytes int, mustKeepTools bool) *trimWithMemoryResult {
|
||||
root := gjson.ParseBytes(body)
|
||||
input := root.Get("input")
|
||||
if !input.Exists() || !input.IsArray() {
|
||||
return &trimWithMemoryResult{Body: body, Shape: "responses"}
|
||||
}
|
||||
arr := input.Array()
|
||||
if len(arr) == 0 {
|
||||
return &trimWithMemoryResult{Body: body, Shape: "responses"}
|
||||
}
|
||||
|
||||
query := extractLastUserTextFromResponses(arr)
|
||||
minKeep := agenticMinKeepMessages()
|
||||
keep := 30
|
||||
perTextLimit := 20_000
|
||||
dropTools := false
|
||||
for keep >= minKeep {
|
||||
outBody := body
|
||||
if dropTools && !mustKeepTools {
|
||||
outBody, _ = sjson.DeleteBytes(outBody, "tools")
|
||||
outBody, _ = sjson.SetBytes(outBody, "tool_choice", "none")
|
||||
}
|
||||
if inst := root.Get("instructions"); inst.Exists() && inst.Type == gjson.String {
|
||||
s := inst.String()
|
||||
instructionsLimit := perTextLimit
|
||||
if instructionsLimit > 2048 {
|
||||
instructionsLimit = 2048
|
||||
}
|
||||
if len(s) > instructionsLimit {
|
||||
outBody, _ = sjson.SetBytes(outBody, "instructions", s[:instructionsLimit]+"\n...[truncated]...")
|
||||
}
|
||||
}
|
||||
|
||||
callByID := make(map[string]gjson.Result, 16)
|
||||
for i := 0; i < len(arr); i++ {
|
||||
item := arr[i]
|
||||
t := item.Get("type").String()
|
||||
if t == "" && item.Get("role").String() != "" {
|
||||
t = "message"
|
||||
}
|
||||
if t != "function_call" {
|
||||
continue
|
||||
}
|
||||
callID := item.Get("call_id").String()
|
||||
if callID == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := callByID[callID]; !ok {
|
||||
callByID[callID] = item
|
||||
}
|
||||
}
|
||||
|
||||
needCall := make(map[string]struct{}, 8)
|
||||
newItems := make([]string, 0, keep+8)
|
||||
keptIdx := make(map[int]struct{}, keep+16)
|
||||
kept := 0
|
||||
for i := len(arr) - 1; i >= 0 && kept < keep; i-- {
|
||||
item := arr[i]
|
||||
t := item.Get("type").String()
|
||||
if t == "" && item.Get("role").String() != "" {
|
||||
t = "message"
|
||||
}
|
||||
|
||||
if dropTools && !mustKeepTools && (t == "function_call" || t == "function_call_output") {
|
||||
continue
|
||||
}
|
||||
if t == "function_call_output" {
|
||||
callID := item.Get("call_id").String()
|
||||
if callID != "" {
|
||||
needCall[callID] = struct{}{}
|
||||
}
|
||||
}
|
||||
if t == "function_call" {
|
||||
callID := item.Get("call_id").String()
|
||||
if callID != "" {
|
||||
delete(needCall, callID)
|
||||
}
|
||||
}
|
||||
|
||||
newItems = append(newItems, truncateMessageContent(item.Raw, perTextLimit))
|
||||
keptIdx[i] = struct{}{}
|
||||
kept++
|
||||
}
|
||||
reverseStrings(newItems)
|
||||
|
||||
if !dropTools && len(needCall) > 0 {
|
||||
ordered := make([]string, 0, len(needCall))
|
||||
for i := 0; i < len(arr); i++ {
|
||||
item := arr[i]
|
||||
if item.Get("type").String() != "function_call" {
|
||||
continue
|
||||
}
|
||||
callID := item.Get("call_id").String()
|
||||
if callID == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := needCall[callID]; ok {
|
||||
if call, ok2 := callByID[callID]; ok2 {
|
||||
ordered = append(ordered, call.Raw)
|
||||
keptIdx[i] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(ordered) > 0 {
|
||||
newItems = append(ordered, newItems...)
|
||||
}
|
||||
}
|
||||
|
||||
out := setJSONArrayBytes(outBody, "input", newItems)
|
||||
if len(out) <= maxBytes {
|
||||
dropped := collectDroppedResponses(arr, keptIdx)
|
||||
return &trimWithMemoryResult{Body: out, Query: query, Dropped: dropped, Shape: "responses"}
|
||||
}
|
||||
|
||||
keep = keep / 2
|
||||
if perTextLimit > 5_000 {
|
||||
perTextLimit = perTextLimit / 2
|
||||
}
|
||||
if !mustKeepTools {
|
||||
dropTools = true
|
||||
}
|
||||
}
|
||||
|
||||
return &trimWithMemoryResult{Body: body, Query: query, Shape: "responses"}
|
||||
}
|
||||
|
||||
func collectDroppedResponses(arr []gjson.Result, kept map[int]struct{}) []memory.Event {
|
||||
out := make([]memory.Event, 0, 64)
|
||||
for i := 0; i < len(arr); i++ {
|
||||
if _, ok := kept[i]; ok {
|
||||
continue
|
||||
}
|
||||
item := arr[i]
|
||||
t := item.Get("type").String()
|
||||
role := item.Get("role").String()
|
||||
txt := extractTextFromResponsesItem(item)
|
||||
if strings.TrimSpace(txt) == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, memory.Event{Kind: "dropped_responses", Type: t, Role: role, Text: txt})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func extractLastUserTextFromResponses(arr []gjson.Result) string {
|
||||
for i := len(arr) - 1; i >= 0; i-- {
|
||||
item := arr[i]
|
||||
role := item.Get("role").String()
|
||||
if !strings.EqualFold(role, "user") {
|
||||
continue
|
||||
}
|
||||
txt := extractTextFromResponsesItem(item)
|
||||
if strings.TrimSpace(txt) != "" {
|
||||
return txt
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractTextFromResponsesItem(item gjson.Result) string {
|
||||
content := item.Get("content")
|
||||
if content.IsArray() {
|
||||
var b strings.Builder
|
||||
for _, part := range content.Array() {
|
||||
partType := part.Get("type").String()
|
||||
if partType == "thinking" || partType == "reasoning" {
|
||||
continue
|
||||
}
|
||||
text := part.Get("text")
|
||||
if !text.Exists() || text.Type != gjson.String {
|
||||
continue
|
||||
}
|
||||
if b.Len() > 0 {
|
||||
b.WriteString("\n")
|
||||
}
|
||||
b.WriteString(text.String())
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
if content.Type == gjson.String {
|
||||
return content.String()
|
||||
}
|
||||
if t := item.Get("text"); t.Exists() && t.Type == gjson.String {
|
||||
return t.String()
|
||||
}
|
||||
if out := item.Get("output"); out.Exists() && out.Type == gjson.String {
|
||||
return out.String()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func trimClaudeMessagesWithMemory(body []byte, maxBytes int, mustKeepTools bool) *trimWithMemoryResult {
|
||||
root := gjson.ParseBytes(body)
|
||||
msgs := root.Get("messages")
|
||||
if !msgs.IsArray() {
|
||||
return &trimWithMemoryResult{Body: body, Shape: "claude"}
|
||||
}
|
||||
arr := msgs.Array()
|
||||
if len(arr) == 0 {
|
||||
return &trimWithMemoryResult{Body: body, Shape: "claude"}
|
||||
}
|
||||
|
||||
query := extractLastUserTextFromClaude(arr)
|
||||
minKeep := agenticMinKeepMessages()
|
||||
keep := 20
|
||||
perTextLimit := 20_000
|
||||
dropTools := false
|
||||
for keep >= minKeep {
|
||||
outBody := body
|
||||
if dropTools && !mustKeepTools {
|
||||
outBody, _ = sjson.DeleteBytes(outBody, "tools")
|
||||
outBody, _ = sjson.SetBytes(outBody, "tool_choice", map[string]any{"type": "none"})
|
||||
}
|
||||
|
||||
newMsgs := make([]string, 0, keep+1)
|
||||
keptIdx := make(map[int]struct{}, keep+2)
|
||||
|
||||
isToolResultMsg := func(m gjson.Result) bool {
|
||||
content := m.Get("content")
|
||||
if !content.IsArray() {
|
||||
return false
|
||||
}
|
||||
for _, part := range content.Array() {
|
||||
if part.Get("type").String() == "tool_result" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
assistantHasToolUse := func(m gjson.Result) bool {
|
||||
if !strings.EqualFold(m.Get("role").String(), "assistant") {
|
||||
return false
|
||||
}
|
||||
content := m.Get("content")
|
||||
if !content.IsArray() {
|
||||
return false
|
||||
}
|
||||
for _, part := range content.Array() {
|
||||
if part.Get("type").String() == "tool_use" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
required := make(map[int]struct{}, 8)
|
||||
tailKept := 0
|
||||
for i := len(arr) - 1; i >= 0 && (tailKept < keep || len(required) > 0); i-- {
|
||||
_, req := required[i]
|
||||
if !req && tailKept >= keep {
|
||||
continue
|
||||
}
|
||||
|
||||
newMsgs = append(newMsgs, truncateClaudeMessageContent(arr[i].Raw, perTextLimit))
|
||||
keptIdx[i] = struct{}{}
|
||||
if !req {
|
||||
tailKept++
|
||||
} else {
|
||||
delete(required, i)
|
||||
}
|
||||
|
||||
if isToolResultMsg(arr[i]) {
|
||||
prev := i - 1
|
||||
if prev >= 0 && assistantHasToolUse(arr[prev]) {
|
||||
required[prev] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
reverseStrings(newMsgs)
|
||||
|
||||
out := setJSONArrayBytes(outBody, "messages", newMsgs)
|
||||
if len(out) <= maxBytes {
|
||||
dropped := collectDroppedClaude(arr, keptIdx)
|
||||
return &trimWithMemoryResult{Body: out, Query: query, Dropped: dropped, Shape: "claude"}
|
||||
}
|
||||
|
||||
keep = keep / 2
|
||||
if perTextLimit > 5_000 {
|
||||
perTextLimit = perTextLimit / 2
|
||||
}
|
||||
dropTools = true
|
||||
}
|
||||
return &trimWithMemoryResult{Body: body, Query: query, Shape: "claude"}
|
||||
}
|
||||
|
||||
func collectDroppedClaude(arr []gjson.Result, kept map[int]struct{}) []memory.Event {
|
||||
out := make([]memory.Event, 0, 32)
|
||||
for i := 0; i < len(arr); i++ {
|
||||
if _, ok := kept[i]; ok {
|
||||
continue
|
||||
}
|
||||
role := arr[i].Get("role").String()
|
||||
txt := extractTextFromClaudeMessage(arr[i])
|
||||
if strings.TrimSpace(txt) == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, memory.Event{Kind: "dropped_claude", Role: role, Text: txt})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func extractLastUserTextFromClaude(arr []gjson.Result) string {
|
||||
for i := len(arr) - 1; i >= 0; i-- {
|
||||
if !strings.EqualFold(arr[i].Get("role").String(), "user") {
|
||||
continue
|
||||
}
|
||||
txt := extractTextFromClaudeMessage(arr[i])
|
||||
if strings.TrimSpace(txt) != "" {
|
||||
return txt
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractTextFromClaudeMessage(msg gjson.Result) string {
|
||||
content := msg.Get("content")
|
||||
switch {
|
||||
case content.Type == gjson.String:
|
||||
return content.String()
|
||||
case content.IsArray():
|
||||
var b strings.Builder
|
||||
for _, part := range content.Array() {
|
||||
partType := part.Get("type").String()
|
||||
if partType == "text" {
|
||||
if t := part.Get("text"); t.Exists() && t.Type == gjson.String {
|
||||
if b.Len() > 0 {
|
||||
b.WriteString("\n")
|
||||
}
|
||||
b.WriteString(t.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func truncateMessageContent(msgRaw string, maxTextChars int) string {
|
||||
msg := msgRaw
|
||||
if maxTextChars <= 0 {
|
||||
return msg
|
||||
}
|
||||
|
||||
content := gjson.Get(msg, "content")
|
||||
switch {
|
||||
case content.Type == gjson.String:
|
||||
s := content.String()
|
||||
if len(s) > maxTextChars {
|
||||
s = s[:maxTextChars] + "\n...[truncated]..."
|
||||
msg, _ = sjson.Set(msg, "content", s)
|
||||
}
|
||||
return msg
|
||||
case content.IsArray():
|
||||
items := content.Array()
|
||||
for i := 0; i < len(items); i++ {
|
||||
text := items[i].Get("text")
|
||||
if !text.Exists() || text.Type != gjson.String {
|
||||
continue
|
||||
}
|
||||
s := text.String()
|
||||
if len(s) > maxTextChars {
|
||||
s = s[:maxTextChars] + "\n...[truncated]..."
|
||||
msg, _ = sjson.Set(msg, "content."+strconv.Itoa(i)+".text", s)
|
||||
}
|
||||
}
|
||||
return msg
|
||||
default:
|
||||
return msg
|
||||
}
|
||||
}
|
||||
|
||||
func truncateClaudeMessageContent(msgRaw string, maxTextChars int) string {
|
||||
msg := msgRaw
|
||||
if maxTextChars <= 0 {
|
||||
return msg
|
||||
}
|
||||
|
||||
content := gjson.Get(msg, "content")
|
||||
switch {
|
||||
case content.Type == gjson.String:
|
||||
s := content.String()
|
||||
if len(s) > maxTextChars {
|
||||
s = s[:maxTextChars] + "\n...[truncated]..."
|
||||
msg, _ = sjson.Set(msg, "content", s)
|
||||
}
|
||||
return msg
|
||||
case content.IsArray():
|
||||
items := content.Array()
|
||||
for i := 0; i < len(items); i++ {
|
||||
partType := items[i].Get("type").String()
|
||||
if partType != "text" {
|
||||
continue
|
||||
}
|
||||
text := items[i].Get("text")
|
||||
if !text.Exists() || text.Type != gjson.String {
|
||||
continue
|
||||
}
|
||||
s := text.String()
|
||||
if len(s) > maxTextChars {
|
||||
s = s[:maxTextChars] + "\n...[truncated]..."
|
||||
msg, _ = sjson.Set(msg, "content."+strconv.Itoa(i)+".text", s)
|
||||
}
|
||||
}
|
||||
return msg
|
||||
default:
|
||||
return msg
|
||||
}
|
||||
}
|
||||
|
||||
func setJSONArrayBytes(body []byte, key string, rawItems []string) []byte {
|
||||
out := body
|
||||
out, _ = sjson.SetRawBytes(out, key, []byte("[]"))
|
||||
for i := range rawItems {
|
||||
out, _ = sjson.SetRawBytes(out, key+".-1", []byte(rawItems[i]))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func reverseStrings(items []string) {
|
||||
for i, j := 0, len(items)-1; i < j; i, j = i+1, j-1 {
|
||||
items[i], items[j] = items[j], items[i]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,498 @@
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/router-for-me/CLIProxyAPI/v6/internal/embeddings"
|
||||
"github.com/router-for-me/CLIProxyAPI/v6/internal/memory"
|
||||
"github.com/router-for-me/CLIProxyAPI/v6/internal/registry"
|
||||
"github.com/router-for-me/CLIProxyAPI/v6/internal/util"
|
||||
)
|
||||
|
||||
const (
|
||||
// codexHardReadLimit is a safety ceiling to avoid unbounded memory reads.
|
||||
codexHardReadLimit = 10 * 1024 * 1024
|
||||
// codexMaxBodyBytesDefault is a best-effort budget to keep agentic CLI requests under common model limits.
|
||||
// It is intentionally conservative to avoid upstream "prompt too long" failures.
|
||||
codexMaxBodyBytesDefault = 200 * 1024
|
||||
|
||||
specModePrompt = `SPEC MODE (do not code yet).
|
||||
1) Produce a complete, reviewable specification (requirements, acceptance criteria, architecture, and file-level plan).
|
||||
2) Wait for explicit approval before editing code.
|
||||
3) If clarification is needed, ask now before writing code.`
|
||||
)
|
||||
|
||||
var (
|
||||
codexMaxBodyBytesOnce sync.Once
|
||||
codexMaxBodyBytes int
|
||||
|
||||
memOnce sync.Once
|
||||
memStore memory.Store
|
||||
|
||||
embedOnce sync.Once
|
||||
embedClient *embeddings.OllamaClient
|
||||
|
||||
embedQueueOnce sync.Once
|
||||
embedQueue *semanticEmbedQueue
|
||||
|
||||
pruneMu sync.Mutex
|
||||
lastPrune time.Time
|
||||
|
||||
limiterMu sync.Mutex
|
||||
memoryLimiters = map[string]*rateLimiter{}
|
||||
semanticLimiters = map[string]*rateLimiter{}
|
||||
)
|
||||
|
||||
type rateLimiter struct {
|
||||
mu sync.Mutex
|
||||
tokens float64
|
||||
last time.Time
|
||||
}
|
||||
|
||||
func (r *rateLimiter) Allow(ratePerSec float64, burst float64) bool {
|
||||
if ratePerSec <= 0 {
|
||||
return true
|
||||
}
|
||||
if burst <= 0 {
|
||||
burst = ratePerSec
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
now := time.Now()
|
||||
if r.last.IsZero() {
|
||||
r.last = now
|
||||
r.tokens = burst
|
||||
}
|
||||
elapsed := now.Sub(r.last).Seconds()
|
||||
if elapsed > 0 {
|
||||
r.tokens += elapsed * ratePerSec
|
||||
if r.tokens > burst {
|
||||
r.tokens = burst
|
||||
}
|
||||
r.last = now
|
||||
}
|
||||
if r.tokens >= 1 {
|
||||
r.tokens -= 1
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func agenticMaxBodyBytes() int {
|
||||
codexMaxBodyBytesOnce.Do(func() {
|
||||
codexMaxBodyBytes = codexMaxBodyBytesDefault
|
||||
|
||||
// Optional override (bytes). Useful when running behind very large-context models.
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_AGENTIC_MAX_BODY_BYTES")); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
if n < 32*1024 {
|
||||
n = 32 * 1024
|
||||
}
|
||||
if n > 2*1024*1024 {
|
||||
n = 2 * 1024 * 1024
|
||||
}
|
||||
codexMaxBodyBytes = n
|
||||
}
|
||||
}
|
||||
})
|
||||
return codexMaxBodyBytes
|
||||
}
|
||||
|
||||
func agenticMemoryStore() memory.Store {
|
||||
memOnce.Do(func() {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_ENABLED")); v != "" {
|
||||
if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") {
|
||||
memStore = nil
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
base := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_DIR"))
|
||||
if base == "" {
|
||||
if w := util.WritablePath(); w != "" {
|
||||
base = filepath.Join(w, ".proxypilot", "memory")
|
||||
} else {
|
||||
base = filepath.Join(".proxypilot", "memory")
|
||||
}
|
||||
}
|
||||
memStore = memory.NewFileStore(base)
|
||||
|
||||
if agenticLLMSummaryEnabled() {
|
||||
if fs, ok := memStore.(*memory.FileStore); ok {
|
||||
config := memory.DefaultSummarizerConfig()
|
||||
summarizer := memory.NewSummarizer(config, memory.NewNoOpSummarizerExecutor())
|
||||
fs.SetSummarizer(summarizer)
|
||||
}
|
||||
}
|
||||
})
|
||||
return memStore
|
||||
}
|
||||
|
||||
// SetSummarizerExecutor configures the LLM executor for context summarization.
|
||||
func SetSummarizerExecutor(executor memory.SummarizerExecutor) {
|
||||
store := agenticMemoryStore()
|
||||
if store == nil {
|
||||
return
|
||||
}
|
||||
fs, ok := store.(*memory.FileStore)
|
||||
if !ok || fs == nil {
|
||||
return
|
||||
}
|
||||
summarizer := fs.GetSummarizer()
|
||||
if summarizer == nil {
|
||||
config := memory.DefaultSummarizerConfig()
|
||||
summarizer = memory.NewSummarizer(config, executor)
|
||||
fs.SetSummarizer(summarizer)
|
||||
} else {
|
||||
config := memory.DefaultSummarizerConfig()
|
||||
fs.SetSummarizer(memory.NewSummarizer(config, executor))
|
||||
}
|
||||
}
|
||||
|
||||
// InitSummarizerWithAuthManager configures LLM-based summarization using the core auth manager.
|
||||
func InitSummarizerWithAuthManager(manager memory.CoreManagerExecutor, providers []string) {
|
||||
if manager == nil {
|
||||
return
|
||||
}
|
||||
adapter := memory.NewManagerAuthAdapter(manager)
|
||||
baseExecutor := memory.NewPipelineSummarizerExecutor(adapter, providers)
|
||||
executor := memory.NewSummaryModelFallbackExecutor(baseExecutor)
|
||||
SetSummarizerExecutor(executor)
|
||||
}
|
||||
|
||||
// GetSummaryModel returns the configured summary model, defaulting to gemini-3-flash.
|
||||
func GetSummaryModel() string {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SUMMARY_MODEL")); v != "" {
|
||||
return v
|
||||
}
|
||||
return memory.DefaultSummaryModel
|
||||
}
|
||||
|
||||
func agenticSemanticEnabled() bool {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_ENABLED")); v != "" {
|
||||
if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func agenticSemanticModel() string {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MODEL")); v != "" {
|
||||
return v
|
||||
}
|
||||
return "embeddinggemma"
|
||||
}
|
||||
|
||||
func agenticSemanticBaseURL() string {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_BASE_URL")); v != "" {
|
||||
return v
|
||||
}
|
||||
return "http://127.0.0.1:11434"
|
||||
}
|
||||
|
||||
func agenticSemanticMaxSnips() int {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_SNIPS")); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 4
|
||||
}
|
||||
|
||||
func agenticSemanticMaxChars() int {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_CHARS")); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 3000
|
||||
}
|
||||
|
||||
func agenticSemanticClient() *embeddings.OllamaClient {
|
||||
embedOnce.Do(func() {
|
||||
embedClient = &embeddings.OllamaClient{
|
||||
BaseURL: agenticSemanticBaseURL(),
|
||||
Model: agenticSemanticModel(),
|
||||
Client: &http.Client{Timeout: 8 * time.Second},
|
||||
}
|
||||
})
|
||||
return embedClient
|
||||
}
|
||||
|
||||
type semanticEmbedTask struct {
|
||||
namespace string
|
||||
session string
|
||||
texts []string
|
||||
roles []string
|
||||
source string
|
||||
}
|
||||
|
||||
type semanticEmbedQueue struct {
|
||||
ch chan semanticEmbedTask
|
||||
fs *memory.FileStore
|
||||
}
|
||||
|
||||
func agenticTodoEnabled() bool {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_TODO_ENABLED")); v != "" {
|
||||
if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func agenticTodoMaxChars() int {
|
||||
v := strings.TrimSpace(os.Getenv("CLIPROXY_TODO_MAX_CHARS"))
|
||||
if v == "" {
|
||||
return 4000
|
||||
}
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return 4000
|
||||
}
|
||||
if n < 512 {
|
||||
return 512
|
||||
}
|
||||
if n > 20_000 {
|
||||
return 20_000
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func agenticMemoryMaxAgeDays() int {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_MAX_AGE_DAYS")); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func agenticMemoryMaxSessions() int {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_MAX_SESSIONS")); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func agenticMemoryMaxBytesPerSession() int64 {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_MAX_BYTES_PER_SESSION")); v != "" {
|
||||
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func agenticSemanticMaxNamespaces() int {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_NAMESPACES")); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func agenticSemanticMaxBytesPerNamespace() int64 {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_BYTES_PER_NAMESPACE")); v != "" {
|
||||
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func agenticMemoryMaxWritesPerMin() int {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_MEMORY_MAX_WRITES_PER_MIN")); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 120
|
||||
}
|
||||
|
||||
func agenticSemanticMaxWritesPerMin() int {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_MAX_WRITES_PER_MIN")); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 120
|
||||
}
|
||||
|
||||
func agenticSemanticQueryMaxChars() int {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_QUERY_MAX_CHARS")); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 512
|
||||
}
|
||||
|
||||
func agenticAnchorAppendOnly() bool {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_ANCHOR_APPEND_ONLY")); v != "" {
|
||||
if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") {
|
||||
return false
|
||||
}
|
||||
if strings.EqualFold(v, "1") || strings.EqualFold(v, "true") || strings.EqualFold(v, "on") || strings.EqualFold(v, "yes") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func agenticAnchorSummaryMaxChars() int {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_ANCHOR_SUMMARY_MAX_CHARS")); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 14_000
|
||||
}
|
||||
|
||||
func agenticCompressionThreshold() float64 {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_COMPRESSION_THRESHOLD")); v != "" {
|
||||
if f, err := strconv.ParseFloat(v, 64); err == nil && f > 0 && f < 1 {
|
||||
return f
|
||||
}
|
||||
}
|
||||
return 0.85
|
||||
}
|
||||
|
||||
func agenticMinKeepMessages() int {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_MIN_KEEP_MESSAGES")); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 25
|
||||
}
|
||||
|
||||
func agenticLLMSummaryEnabled() bool {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_LLM_SUMMARY_ENABLED")); v != "" {
|
||||
return strings.EqualFold(v, "1") || strings.EqualFold(v, "true") || strings.EqualFold(v, "on")
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func getModelContextWindow(model string) int {
|
||||
info := registry.GetGlobalRegistry().GetModelInfo(model, "")
|
||||
if info != nil && info.ContextLength > 0 {
|
||||
return info.ContextLength
|
||||
}
|
||||
lowerModel := strings.ToLower(model)
|
||||
switch {
|
||||
case strings.Contains(lowerModel, "claude-3.5"), strings.Contains(lowerModel, "claude-3-5"):
|
||||
return 200000
|
||||
case strings.Contains(lowerModel, "claude-3"):
|
||||
return 200000
|
||||
case strings.Contains(lowerModel, "claude"):
|
||||
return 100000
|
||||
case strings.Contains(lowerModel, "gpt-4-turbo"), strings.Contains(lowerModel, "gpt-4o"):
|
||||
return 128000
|
||||
case strings.Contains(lowerModel, "gpt-4"):
|
||||
return 8192
|
||||
case strings.Contains(lowerModel, "gpt-3.5"):
|
||||
return 16384
|
||||
case strings.Contains(lowerModel, "gemini"):
|
||||
return 1000000
|
||||
case strings.Contains(lowerModel, "o1"), strings.Contains(lowerModel, "o3"):
|
||||
return 200000
|
||||
default:
|
||||
return 100000
|
||||
}
|
||||
}
|
||||
|
||||
func agenticTokenAwareEnabled() bool {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_TOKEN_AWARE_ENABLED")); v != "" {
|
||||
return strings.EqualFold(v, "1") || strings.EqualFold(v, "true") || strings.EqualFold(v, "on")
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func agenticReserveTokens() int {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_RESERVE_TOKENS")); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 8192
|
||||
}
|
||||
|
||||
func agenticScaffoldEnabled() bool {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SCAFFOLD_ENABLED")); v != "" {
|
||||
if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func agenticScaffoldAppendOnly() bool {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SCAFFOLD_APPEND_ONLY")); v != "" {
|
||||
if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func allowMemoryWrite(session string) bool {
|
||||
limit := agenticMemoryMaxWritesPerMin()
|
||||
if limit <= 0 || session == "" {
|
||||
return true
|
||||
}
|
||||
limiter := getSessionLimiter(memoryLimiters, session)
|
||||
rate := float64(limit) / 60.0
|
||||
burst := float64(limit) / 6.0
|
||||
if burst < 5 {
|
||||
burst = 5
|
||||
}
|
||||
return limiter.Allow(rate, burst)
|
||||
}
|
||||
|
||||
func allowSemanticWrite(session string) bool {
|
||||
limit := agenticSemanticMaxWritesPerMin()
|
||||
if limit <= 0 || session == "" {
|
||||
return true
|
||||
}
|
||||
limiter := getSessionLimiter(semanticLimiters, session)
|
||||
rate := float64(limit) / 60.0
|
||||
burst := float64(limit) / 6.0
|
||||
if burst < 5 {
|
||||
burst = 5
|
||||
}
|
||||
return limiter.Allow(rate, burst)
|
||||
}
|
||||
|
||||
func getSessionLimiter(store map[string]*rateLimiter, session string) *rateLimiter {
|
||||
limiterMu.Lock()
|
||||
defer limiterMu.Unlock()
|
||||
if limiter, ok := store[session]; ok {
|
||||
return limiter
|
||||
}
|
||||
limiter := &rateLimiter{}
|
||||
store[session] = limiter
|
||||
for key, lim := range store {
|
||||
if lim == nil {
|
||||
delete(store, key)
|
||||
continue
|
||||
}
|
||||
lim.mu.Lock()
|
||||
last := lim.last
|
||||
lim.mu.Unlock()
|
||||
if !last.IsZero() && time.Since(last) > 15*time.Minute {
|
||||
delete(store, key)
|
||||
}
|
||||
}
|
||||
return limiter
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user