mirror of
https://github.com/Finesssee/ProxyPilot.git
synced 2026-11-19 02:17:46 +00:00
feat(middleware): add token-aware context compression
Implement proactive compression based on token count rather than bytes: - Uses tiktoken for accurate token estimation - Compresses at 75% threshold before hitting limits - Reserves tokens for output generation 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.5
parent
57d5b4b1b1
commit
25dd8b2087
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/router-for-me/CLIProxyAPI/v6/internal/embeddings"
|
||||
"github.com/router-for-me/CLIProxyAPI/v6/internal/memory"
|
||||
"github.com/router-for-me/CLIProxyAPI/v6/internal/registry"
|
||||
"github.com/router-for-me/CLIProxyAPI/v6/internal/runtime/executor"
|
||||
"github.com/router-for-me/CLIProxyAPI/v6/internal/util"
|
||||
"github.com/tidwall/gjson"
|
||||
"github.com/tidwall/sjson"
|
||||
@@ -134,10 +135,45 @@ func agenticMemoryStore() memory.Store {
|
||||
}
|
||||
}
|
||||
memStore = memory.NewFileStore(base)
|
||||
|
||||
// Initialize summarizer with default config (uses regex fallback until executor is set)
|
||||
if agenticLLMSummaryEnabled() {
|
||||
if fs, ok := memStore.(*memory.FileStore); ok {
|
||||
config := memory.DefaultSummarizerConfig()
|
||||
// Start with NoOp executor - will fall back to regex
|
||||
// Call SetSummarizerExecutor() later with a real executor to enable LLM summarization
|
||||
summarizer := memory.NewSummarizer(config, memory.NewNoOpSummarizerExecutor())
|
||||
fs.SetSummarizer(summarizer)
|
||||
}
|
||||
}
|
||||
})
|
||||
return memStore
|
||||
}
|
||||
|
||||
// SetSummarizerExecutor configures the LLM executor for context summarization.
|
||||
// Call this from server initialization with the auth manager to enable full LLM summarization.
|
||||
// Example: middleware.SetSummarizerExecutor(memory.NewPipelineSummarizerExecutor(authManager, providers))
|
||||
func SetSummarizerExecutor(executor memory.SummarizerExecutor) {
|
||||
store := agenticMemoryStore()
|
||||
if store == nil {
|
||||
return
|
||||
}
|
||||
fs, ok := store.(*memory.FileStore)
|
||||
if !ok || fs == nil {
|
||||
return
|
||||
}
|
||||
summarizer := fs.GetSummarizer()
|
||||
if summarizer == nil {
|
||||
config := memory.DefaultSummarizerConfig()
|
||||
summarizer = memory.NewSummarizer(config, executor)
|
||||
fs.SetSummarizer(summarizer)
|
||||
} else {
|
||||
// Replace the existing summarizer with one that has the real executor
|
||||
config := memory.DefaultSummarizerConfig()
|
||||
fs.SetSummarizer(memory.NewSummarizer(config, executor))
|
||||
}
|
||||
}
|
||||
|
||||
func agenticSemanticEnabled() bool {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_SEMANTIC_ENABLED")); v != "" {
|
||||
if strings.EqualFold(v, "0") || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") || strings.EqualFold(v, "no") {
|
||||
@@ -400,6 +436,176 @@ func agenticAnchorSummaryMaxChars() int {
|
||||
return 14_000
|
||||
}
|
||||
|
||||
// agenticCompressionThreshold returns the context usage ratio at which to trigger LLM compression.
|
||||
// Default: 0.75 (75% of context window)
|
||||
func agenticCompressionThreshold() float64 {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_COMPRESSION_THRESHOLD")); v != "" {
|
||||
if f, err := strconv.ParseFloat(v, 64); err == nil && f > 0 && f < 1 {
|
||||
return f
|
||||
}
|
||||
}
|
||||
return 0.75
|
||||
}
|
||||
|
||||
// agenticLLMSummaryEnabled returns whether LLM-based summarization is enabled.
|
||||
// Default: true
|
||||
func agenticLLMSummaryEnabled() bool {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_LLM_SUMMARY_ENABLED")); v != "" {
|
||||
return strings.EqualFold(v, "1") || strings.EqualFold(v, "true") || strings.EqualFold(v, "on")
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// getModelContextWindow returns the context window size for a model.
|
||||
func getModelContextWindow(model string) int {
|
||||
// Try registry first
|
||||
info := registry.GetGlobalRegistry().GetModelInfo(model)
|
||||
if info != nil && info.ContextLength > 0 {
|
||||
return info.ContextLength
|
||||
}
|
||||
// Fallback based on model name patterns
|
||||
lowerModel := strings.ToLower(model)
|
||||
switch {
|
||||
case strings.Contains(lowerModel, "claude-3.5"), strings.Contains(lowerModel, "claude-3-5"):
|
||||
return 200000
|
||||
case strings.Contains(lowerModel, "claude-3"):
|
||||
return 200000
|
||||
case strings.Contains(lowerModel, "claude"):
|
||||
return 100000
|
||||
case strings.Contains(lowerModel, "gpt-4-turbo"), strings.Contains(lowerModel, "gpt-4o"):
|
||||
return 128000
|
||||
case strings.Contains(lowerModel, "gpt-4"):
|
||||
return 8192
|
||||
case strings.Contains(lowerModel, "gpt-3.5"):
|
||||
return 16384
|
||||
case strings.Contains(lowerModel, "gemini"):
|
||||
return 1000000
|
||||
case strings.Contains(lowerModel, "o1"), strings.Contains(lowerModel, "o3"):
|
||||
return 200000
|
||||
default:
|
||||
return 100000
|
||||
}
|
||||
}
|
||||
|
||||
// agenticTokenAwareEnabled returns whether token-aware compression is enabled.
|
||||
// Default: true
|
||||
func agenticTokenAwareEnabled() bool {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_TOKEN_AWARE_ENABLED")); v != "" {
|
||||
return strings.EqualFold(v, "1") || strings.EqualFold(v, "true") || strings.EqualFold(v, "on")
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// agenticReserveTokens returns the number of tokens to reserve for model output.
|
||||
// Default: 8192 (reasonable output buffer)
|
||||
func agenticReserveTokens() int {
|
||||
if v := strings.TrimSpace(os.Getenv("CLIPROXY_RESERVE_TOKENS")); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 8192
|
||||
}
|
||||
|
||||
// tokenAwareCompressionResult contains the result of token-aware analysis
|
||||
type tokenAwareCompressionResult struct {
|
||||
ShouldTrim bool // Whether trimming is needed
|
||||
CurrentTokens int64 // Estimated current token count
|
||||
ContextWindow int // Model's context window
|
||||
TargetTokens int64 // Target token count after trimming
|
||||
TargetMaxBytes int // Approximate bytes to achieve target tokens
|
||||
Model string // The model name
|
||||
}
|
||||
|
||||
// analyzeTokenBudget checks if the request exceeds the token threshold and calculates trimming targets.
|
||||
// Returns whether trimming is needed and the target byte limit.
|
||||
func analyzeTokenBudget(body []byte) *tokenAwareCompressionResult {
|
||||
result := &tokenAwareCompressionResult{
|
||||
ShouldTrim: false,
|
||||
TargetMaxBytes: agenticMaxBodyBytes(), // Default fallback
|
||||
}
|
||||
|
||||
if !agenticTokenAwareEnabled() {
|
||||
return result
|
||||
}
|
||||
|
||||
model := gjson.GetBytes(body, "model").String()
|
||||
if model == "" {
|
||||
return result
|
||||
}
|
||||
result.Model = model
|
||||
|
||||
// Get context window for this model
|
||||
contextWindow := getModelContextWindow(model)
|
||||
result.ContextWindow = contextWindow
|
||||
|
||||
// Estimate current token count
|
||||
currentTokens, err := executor.EstimateRequestTokens(model, body)
|
||||
if err != nil {
|
||||
// Fallback to byte-based heuristic: ~4 chars per token
|
||||
currentTokens = int64(len(body) / 4)
|
||||
}
|
||||
result.CurrentTokens = currentTokens
|
||||
|
||||
// Reserve tokens for output
|
||||
reserveTokens := agenticReserveTokens()
|
||||
availableContext := contextWindow - reserveTokens
|
||||
if availableContext < 1000 {
|
||||
availableContext = contextWindow / 2 // At least half for input
|
||||
}
|
||||
|
||||
// Check against threshold
|
||||
threshold := agenticCompressionThreshold()
|
||||
maxInputTokens := int64(float64(availableContext) * threshold)
|
||||
|
||||
if currentTokens <= maxInputTokens {
|
||||
// Under threshold, no trimming needed
|
||||
result.ShouldTrim = false
|
||||
return result
|
||||
}
|
||||
|
||||
// Calculate target tokens (aim for 70% of threshold to give buffer)
|
||||
targetRatio := threshold * 0.9 // Slightly under threshold
|
||||
targetTokens := int64(float64(availableContext) * targetRatio)
|
||||
result.TargetTokens = targetTokens
|
||||
result.ShouldTrim = true
|
||||
|
||||
// Convert target tokens to approximate byte limit
|
||||
// Use ~4 bytes per token as heuristic, but be conservative
|
||||
targetBytes := int(targetTokens * 4)
|
||||
|
||||
// Clamp to reasonable bounds
|
||||
minBytes := 32 * 1024 // 32KB minimum
|
||||
if targetBytes < minBytes {
|
||||
targetBytes = minBytes
|
||||
}
|
||||
|
||||
// Don't exceed the configured max
|
||||
maxBytes := agenticMaxBodyBytes()
|
||||
if targetBytes > maxBytes {
|
||||
targetBytes = maxBytes
|
||||
}
|
||||
|
||||
result.TargetMaxBytes = targetBytes
|
||||
return result
|
||||
}
|
||||
|
||||
// getTokenAwareMaxBytes returns the maximum body size based on token analysis.
|
||||
// This is more accurate than the byte-based heuristic in agenticMaxBodyBytesForModel.
|
||||
func getTokenAwareMaxBytes(body []byte) int {
|
||||
if !agenticTokenAwareEnabled() {
|
||||
return agenticMaxBodyBytesForModel(body)
|
||||
}
|
||||
|
||||
result := analyzeTokenBudget(body)
|
||||
if result.ShouldTrim {
|
||||
return result.TargetMaxBytes
|
||||
}
|
||||
|
||||
// Not over threshold, but still apply model-specific limits
|
||||
return agenticMaxBodyBytesForModel(body)
|
||||
}
|
||||
|
||||
func allowMemoryWrite(session string) bool {
|
||||
limit := agenticMemoryMaxWritesPerMin()
|
||||
if limit <= 0 || session == "" {
|
||||
@@ -523,7 +729,31 @@ func CodexPromptBudgetMiddlewareWithRootDir(rootDir string) gin.HandlerFunc {
|
||||
}
|
||||
|
||||
originalLen := len(body)
|
||||
maxBytes := agenticMaxBodyBytesForModel(body)
|
||||
|
||||
// Token-aware compression: analyze token budget before byte-based check
|
||||
tokenAnalysis := analyzeTokenBudget(body)
|
||||
maxBytes := tokenAnalysis.TargetMaxBytes
|
||||
|
||||
// If token analysis didn't trigger, fall back to byte-based model limit
|
||||
if !tokenAnalysis.ShouldTrim {
|
||||
maxBytes = agenticMaxBodyBytesForModel(body)
|
||||
}
|
||||
|
||||
// Add diagnostic headers for localhost debugging
|
||||
if c != nil {
|
||||
ip := c.ClientIP()
|
||||
if ip == "127.0.0.1" || ip == "::1" {
|
||||
if tokenAnalysis.Model != "" {
|
||||
c.Header("X-ProxyPilot-Model", tokenAnalysis.Model)
|
||||
c.Header("X-ProxyPilot-Context-Window", strconv.Itoa(tokenAnalysis.ContextWindow))
|
||||
c.Header("X-ProxyPilot-Current-Tokens", strconv.FormatInt(tokenAnalysis.CurrentTokens, 10))
|
||||
if tokenAnalysis.ShouldTrim {
|
||||
c.Header("X-ProxyPilot-Token-Triggered", "true")
|
||||
c.Header("X-ProxyPilot-Target-Tokens", strconv.FormatInt(tokenAnalysis.TargetTokens, 10))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Session-scoped state (pinned + anchor + TODO + spec) is injected as append-only
|
||||
// scaffolding when enabled. This preserves prompt-cache friendliness.
|
||||
@@ -533,7 +763,9 @@ func CodexPromptBudgetMiddlewareWithRootDir(rootDir string) gin.HandlerFunc {
|
||||
originalLen = len(body)
|
||||
}
|
||||
|
||||
if originalLen <= maxBytes {
|
||||
// Proactive compression: trim if over token threshold OR over byte limit
|
||||
needsTrim := tokenAnalysis.ShouldTrim || originalLen > maxBytes
|
||||
if !needsTrim {
|
||||
req.Body = io.NopCloser(bytes.NewReader(body))
|
||||
req.ContentLength = int64(originalLen)
|
||||
req.Header.Set("Content-Length", strconv.Itoa(originalLen))
|
||||
@@ -1349,7 +1581,13 @@ func agenticStoreAndInjectMemory(c *gin.Context, req *http.Request, session stri
|
||||
_ = fs.WritePinned(session, pinned, 8000)
|
||||
}
|
||||
if len(res.Dropped) > 0 {
|
||||
_ = agenticUpdateAnchoredSummary(fs, session, res.Dropped, pinned, res.Query)
|
||||
if agenticLLMSummaryEnabled() {
|
||||
model := gjson.GetBytes(res.Body, "model").String()
|
||||
ctx := c.Request.Context()
|
||||
_ = agenticUpdateAnchoredSummaryWithLLM(ctx, model, fs, session, res.Dropped, pinned, res.Query)
|
||||
} else {
|
||||
_ = agenticUpdateAnchoredSummary(fs, session, res.Dropped, pinned, res.Query)
|
||||
}
|
||||
}
|
||||
if agenticSemanticEnabled() && len(res.Dropped) > 0 && !fs.IsSemanticDisabled(session) {
|
||||
ns := semanticNamespace(req, res.Body, session)
|
||||
@@ -1410,6 +1648,28 @@ func agenticUpdateAnchoredSummary(fs *memory.FileStore, session string, dropped
|
||||
return fs.WriteSummary(session, next, agenticAnchorSummaryMaxChars())
|
||||
}
|
||||
|
||||
// agenticUpdateAnchoredSummaryWithLLM updates the anchored summary using LLM.
|
||||
func agenticUpdateAnchoredSummaryWithLLM(ctx context.Context, model string, fs *memory.FileStore, session string, dropped []memory.Event, pinned string, latestIntent string) error {
|
||||
if fs == nil || session == "" {
|
||||
return nil
|
||||
}
|
||||
summarizer := fs.GetSummarizer()
|
||||
if summarizer == nil {
|
||||
// Fall back to regex-based
|
||||
return agenticUpdateAnchoredSummary(fs, session, dropped, pinned, latestIntent)
|
||||
}
|
||||
|
||||
prev := fs.ReadSummary(session, agenticAnchorSummaryMaxChars())
|
||||
next := memory.BuildAnchoredSummaryWithLLM(ctx, model, prev, dropped, latestIntent, summarizer)
|
||||
if strings.TrimSpace(next) == "" {
|
||||
return nil
|
||||
}
|
||||
if agenticAnchorAppendOnly() {
|
||||
return fs.SetAnchorSummary(session, next, agenticAnchorSummaryMaxChars())
|
||||
}
|
||||
return fs.WriteSummary(session, next, agenticAnchorSummaryMaxChars())
|
||||
}
|
||||
|
||||
func agenticMaybePruneMemory() {
|
||||
maxAge := agenticMemoryMaxAgeDays()
|
||||
maxSessions := agenticMemoryMaxSessions()
|
||||
|
||||
Reference in New Issue
Block a user