mirror of
https://github.com/Finesssee/ProxyPilot.git
synced 2026-11-19 02:17:46 +00:00
572 lines
15 KiB
Go
572 lines
15 KiB
Go
package cache
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/gob"
|
|
"encoding/hex"
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/router-for-me/CLIProxyAPI/v6/internal/api/middleware"
|
|
log "github.com/sirupsen/logrus"
|
|
)
|
|
|
|
// DefaultEvictionInterval is the default interval for periodic cache eviction.
|
|
const DefaultEvictionInterval = 1 * time.Minute
|
|
|
|
// ResponseCacheConfig defines configuration for the response cache.
|
|
type ResponseCacheConfig struct {
|
|
// Enabled controls whether response caching is active.
|
|
Enabled bool `yaml:"enabled" json:"enabled"`
|
|
// MaxSize is the maximum number of cached responses.
|
|
MaxSize int `yaml:"max-size" json:"max-size"`
|
|
// MaxBytes is the maximum total size (bytes) of cached responses.
|
|
// 0 disables size-based eviction.
|
|
MaxBytes int64 `yaml:"max-bytes" json:"max-bytes"`
|
|
// TTL is how long responses are cached (e.g., "5m", "1h").
|
|
TTL time.Duration `yaml:"ttl" json:"ttl"`
|
|
// ExcludeModels is a list of model patterns to exclude from caching.
|
|
ExcludeModels []string `yaml:"exclude-models" json:"exclude-models"`
|
|
// PersistFile is the optional file path to persist cache across restarts.
|
|
PersistFile string `yaml:"persist-file" json:"persist-file"`
|
|
}
|
|
|
|
// DefaultResponseCacheConfig returns sensible defaults.
|
|
func DefaultResponseCacheConfig() ResponseCacheConfig {
|
|
return ResponseCacheConfig{
|
|
Enabled: false, // Opt-in by default
|
|
MaxSize: 1000,
|
|
TTL: 5 * time.Minute,
|
|
}
|
|
}
|
|
|
|
// CachedResponse stores a cached API response with metadata.
|
|
type CachedResponse struct {
|
|
// Response is the cached response body.
|
|
Response []byte
|
|
// ContentType is the response content type.
|
|
ContentType string
|
|
// StatusCode is the HTTP status code.
|
|
StatusCode int
|
|
// Model is the model that generated this response.
|
|
Model string
|
|
// CreatedAt is when the response was cached.
|
|
CreatedAt time.Time
|
|
// HitCount tracks how many times this cache entry was hit.
|
|
HitCount int
|
|
// SizeBytes tracks the approximate size of this entry.
|
|
SizeBytes int64
|
|
}
|
|
|
|
// ResponseCache provides LRU-based caching for API responses.
|
|
// This caches complete responses at the proxy layer to reduce upstream API calls.
|
|
type ResponseCache struct {
|
|
mu sync.RWMutex
|
|
entries map[string]*CachedResponse
|
|
order []string // LRU order tracking
|
|
config ResponseCacheConfig
|
|
stats ResponseCacheStats
|
|
totalBytes int64
|
|
disabled bool // runtime toggle
|
|
}
|
|
|
|
// ResponseCacheStats tracks cache performance metrics.
|
|
type ResponseCacheStats struct {
|
|
Hits int64
|
|
Misses int64
|
|
Evictions int64
|
|
Size int
|
|
TotalSaved int64 // Approximate tokens saved
|
|
}
|
|
|
|
// NewResponseCache creates a new response cache with the given config.
|
|
func NewResponseCache(cfg ResponseCacheConfig) *ResponseCache {
|
|
return &ResponseCache{
|
|
entries: make(map[string]*CachedResponse),
|
|
order: make([]string, 0, cfg.MaxSize),
|
|
config: cfg,
|
|
disabled: !cfg.Enabled,
|
|
}
|
|
}
|
|
|
|
// defaultResponseCache is the package-level cache instance.
|
|
var (
|
|
defaultResponseCache *ResponseCache
|
|
defaultResponseCacheOnce sync.Once
|
|
)
|
|
|
|
// GetDefaultResponseCache returns the package-level response cache.
|
|
func GetDefaultResponseCache() *ResponseCache {
|
|
defaultResponseCacheOnce.Do(func() {
|
|
defaultResponseCache = NewResponseCache(DefaultResponseCacheConfig())
|
|
})
|
|
return defaultResponseCache
|
|
}
|
|
|
|
// InitDefaultResponseCache initializes the default cache with custom config.
|
|
func InitDefaultResponseCache(cfg ResponseCacheConfig) {
|
|
defaultResponseCacheOnce.Do(func() {
|
|
defaultResponseCache = NewResponseCache(cfg)
|
|
})
|
|
// If already initialized, update config
|
|
if defaultResponseCache != nil {
|
|
defaultResponseCache.UpdateConfig(cfg)
|
|
}
|
|
}
|
|
|
|
// generateKey creates a cache key from request parameters.
|
|
// Key is based on: model + messages hash + relevant parameters.
|
|
func (rc *ResponseCache) generateKey(model string, payload []byte) string {
|
|
h := sha256.New()
|
|
h.Write([]byte(model))
|
|
h.Write(payload)
|
|
return hex.EncodeToString(h.Sum(nil))[:32]
|
|
}
|
|
|
|
// Get retrieves a cached response.
|
|
// Returns nil if not found, expired, or cache is disabled.
|
|
func (rc *ResponseCache) Get(model string, payload []byte) *CachedResponse {
|
|
if rc.disabled || !rc.config.Enabled {
|
|
return nil
|
|
}
|
|
|
|
key := rc.generateKey(model, payload)
|
|
|
|
rc.mu.RLock()
|
|
entry, exists := rc.entries[key]
|
|
rc.mu.RUnlock()
|
|
|
|
if !exists {
|
|
rc.mu.Lock()
|
|
rc.stats.Misses++
|
|
rc.mu.Unlock()
|
|
middleware.RecordResponseCacheMiss()
|
|
return nil
|
|
}
|
|
|
|
// Check TTL
|
|
if time.Since(entry.CreatedAt) > rc.config.TTL {
|
|
rc.mu.Lock()
|
|
rc.removeEntryLocked(key)
|
|
rc.stats.Evictions++
|
|
rc.stats.Misses++
|
|
middleware.SetResponseCacheSize(len(rc.entries))
|
|
rc.mu.Unlock()
|
|
middleware.RecordResponseCacheMiss()
|
|
return nil
|
|
}
|
|
|
|
rc.mu.Lock()
|
|
entry.HitCount++
|
|
rc.stats.Hits++
|
|
rc.moveToEnd(key)
|
|
rc.mu.Unlock()
|
|
middleware.RecordResponseCacheHit()
|
|
|
|
log.Debugf("response cache HIT for model %s (hits: %d)", model, entry.HitCount)
|
|
return entry
|
|
}
|
|
|
|
// Set stores a response in the cache.
|
|
func (rc *ResponseCache) Set(model string, payload []byte, response []byte, contentType string, statusCode int) {
|
|
if rc.disabled || !rc.config.Enabled {
|
|
return
|
|
}
|
|
|
|
// Only cache successful responses
|
|
if statusCode < 200 || statusCode >= 300 {
|
|
return
|
|
}
|
|
|
|
// Don't cache empty responses
|
|
if len(response) == 0 {
|
|
return
|
|
}
|
|
|
|
// Check if model is excluded
|
|
if rc.isModelExcluded(model) {
|
|
return
|
|
}
|
|
|
|
key := rc.generateKey(model, payload)
|
|
|
|
rc.mu.Lock()
|
|
defer rc.mu.Unlock()
|
|
|
|
entrySize := estimateResponseEntrySize(model, response, contentType)
|
|
if rc.config.MaxBytes > 0 && entrySize > rc.config.MaxBytes {
|
|
return
|
|
}
|
|
|
|
// Evict if at capacity
|
|
for (rc.config.MaxSize > 0 && len(rc.entries) >= rc.config.MaxSize && len(rc.order) > 0) ||
|
|
(rc.config.MaxBytes > 0 && rc.totalBytes+entrySize > rc.config.MaxBytes && len(rc.order) > 0) {
|
|
rc.removeEntryLocked(rc.order[0])
|
|
rc.stats.Evictions++
|
|
}
|
|
|
|
// Store new entry
|
|
if existing := rc.entries[key]; existing != nil {
|
|
rc.totalBytes -= existing.SizeBytes
|
|
}
|
|
rc.entries[key] = &CachedResponse{
|
|
Response: response,
|
|
ContentType: contentType,
|
|
StatusCode: statusCode,
|
|
Model: model,
|
|
CreatedAt: time.Now(),
|
|
HitCount: 0,
|
|
SizeBytes: entrySize,
|
|
}
|
|
rc.order = append(rc.order, key)
|
|
rc.totalBytes += entrySize
|
|
rc.stats.Size = len(rc.entries)
|
|
middleware.SetResponseCacheSize(len(rc.entries))
|
|
|
|
log.Debugf("response cache SET for model %s (size: %d/%d)", model, len(rc.entries), rc.config.MaxSize)
|
|
}
|
|
|
|
// isModelExcluded checks if a model should be excluded from caching.
|
|
func (rc *ResponseCache) isModelExcluded(model string) bool {
|
|
for _, pattern := range rc.config.ExcludeModels {
|
|
if matchPattern(pattern, model) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// matchPattern performs simple wildcard matching.
|
|
func matchPattern(pattern, model string) bool {
|
|
if pattern == "*" {
|
|
return true
|
|
}
|
|
if pattern == model {
|
|
return true
|
|
}
|
|
// Simple prefix/suffix matching
|
|
if len(pattern) > 1 && pattern[0] == '*' {
|
|
return len(model) >= len(pattern)-1 && model[len(model)-(len(pattern)-1):] == pattern[1:]
|
|
}
|
|
if len(pattern) > 1 && pattern[len(pattern)-1] == '*' {
|
|
return len(model) >= len(pattern)-1 && model[:len(pattern)-1] == pattern[:len(pattern)-1]
|
|
}
|
|
return false
|
|
}
|
|
|
|
// moveToEnd moves a key to the end of the LRU order.
|
|
func (rc *ResponseCache) moveToEnd(key string) {
|
|
for i, k := range rc.order {
|
|
if k == key {
|
|
rc.order = append(rc.order[:i], rc.order[i+1:]...)
|
|
rc.order = append(rc.order, key)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// removeFromOrder removes a key from the LRU order.
|
|
func (rc *ResponseCache) removeFromOrder(key string) {
|
|
for i, k := range rc.order {
|
|
if k == key {
|
|
rc.order = append(rc.order[:i], rc.order[i+1:]...)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func (rc *ResponseCache) removeEntryLocked(key string) {
|
|
entry := rc.entries[key]
|
|
if entry != nil {
|
|
rc.totalBytes -= entry.SizeBytes
|
|
}
|
|
delete(rc.entries, key)
|
|
rc.removeFromOrder(key)
|
|
}
|
|
|
|
// Clear removes all entries from the cache.
|
|
func (rc *ResponseCache) Clear() {
|
|
rc.mu.Lock()
|
|
defer rc.mu.Unlock()
|
|
rc.entries = make(map[string]*CachedResponse)
|
|
rc.order = make([]string, 0, rc.config.MaxSize)
|
|
rc.totalBytes = 0
|
|
rc.stats.Size = 0
|
|
middleware.SetResponseCacheSize(0)
|
|
}
|
|
|
|
// GetStats returns current cache statistics.
|
|
func (rc *ResponseCache) GetStats() ResponseCacheStats {
|
|
rc.mu.RLock()
|
|
defer rc.mu.RUnlock()
|
|
rc.stats.Size = len(rc.entries)
|
|
return rc.stats
|
|
}
|
|
|
|
// SetEnabled enables or disables the cache at runtime.
|
|
func (rc *ResponseCache) SetEnabled(enabled bool) {
|
|
rc.mu.Lock()
|
|
defer rc.mu.Unlock()
|
|
rc.disabled = !enabled
|
|
rc.config.Enabled = enabled
|
|
if !enabled {
|
|
log.Info("response cache disabled")
|
|
} else {
|
|
log.Info("response cache enabled")
|
|
}
|
|
}
|
|
|
|
// IsEnabled returns whether the cache is enabled.
|
|
func (rc *ResponseCache) IsEnabled() bool {
|
|
rc.mu.RLock()
|
|
defer rc.mu.RUnlock()
|
|
return rc.config.Enabled && !rc.disabled
|
|
}
|
|
|
|
// UpdateConfig updates the cache configuration.
|
|
func (rc *ResponseCache) UpdateConfig(cfg ResponseCacheConfig) {
|
|
rc.mu.Lock()
|
|
defer rc.mu.Unlock()
|
|
rc.config = cfg
|
|
rc.disabled = !cfg.Enabled
|
|
// Evict if new max size is smaller
|
|
for (cfg.MaxSize > 0 && len(rc.entries) > cfg.MaxSize && len(rc.order) > 0) ||
|
|
(cfg.MaxBytes > 0 && rc.totalBytes > cfg.MaxBytes && len(rc.order) > 0) {
|
|
rc.removeEntryLocked(rc.order[0])
|
|
rc.stats.Evictions++
|
|
}
|
|
rc.stats.Size = len(rc.entries)
|
|
}
|
|
|
|
// EvictExpired removes all expired entries from the cache.
|
|
func (rc *ResponseCache) EvictExpired() int {
|
|
rc.mu.Lock()
|
|
defer rc.mu.Unlock()
|
|
|
|
evicted := 0
|
|
now := time.Now()
|
|
newOrder := make([]string, 0, len(rc.order))
|
|
|
|
for _, key := range rc.order {
|
|
entry, exists := rc.entries[key]
|
|
if !exists {
|
|
continue
|
|
}
|
|
if now.Sub(entry.CreatedAt) > rc.config.TTL {
|
|
rc.totalBytes -= entry.SizeBytes
|
|
delete(rc.entries, key)
|
|
evicted++
|
|
rc.stats.Evictions++
|
|
} else {
|
|
newOrder = append(newOrder, key)
|
|
}
|
|
}
|
|
|
|
rc.order = newOrder
|
|
rc.stats.Size = len(rc.entries)
|
|
middleware.SetResponseCacheSize(len(rc.entries))
|
|
return evicted
|
|
}
|
|
|
|
// StartPeriodicEviction starts a background goroutine that periodically evicts expired entries.
|
|
// The goroutine stops when the context is cancelled.
|
|
func (rc *ResponseCache) StartPeriodicEviction(ctx context.Context, interval time.Duration) {
|
|
if interval <= 0 {
|
|
interval = DefaultEvictionInterval
|
|
}
|
|
go func() {
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
if evicted := rc.EvictExpired(); evicted > 0 {
|
|
log.Debugf("response cache: evicted %d expired entries", evicted)
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
// TryGetCached checks the default response cache for a cached response.
|
|
// Returns (response, contentType, found). If not found, returns (nil, "", false).
|
|
func TryGetCached(model string, payload []byte) ([]byte, string, bool) {
|
|
cache := GetDefaultResponseCache()
|
|
if cache == nil || !cache.IsEnabled() {
|
|
return nil, "", false
|
|
}
|
|
entry := cache.Get(model, payload)
|
|
if entry == nil {
|
|
return nil, "", false
|
|
}
|
|
return entry.Response, entry.ContentType, true
|
|
}
|
|
|
|
// StoreCached stores a response in the default response cache.
|
|
func StoreCached(model string, payload []byte, response []byte, contentType string, statusCode int) {
|
|
cache := GetDefaultResponseCache()
|
|
if cache == nil || !cache.IsEnabled() {
|
|
return
|
|
}
|
|
cache.Set(model, payload, response, contentType, statusCode)
|
|
}
|
|
|
|
// responseCachePersist is the serializable form of the cache for gob encoding.
|
|
type responseCachePersist struct {
|
|
Entries map[string]*CachedResponse
|
|
Order []string
|
|
Stats ResponseCacheStats
|
|
}
|
|
|
|
// SaveToFile persists the cache to a file using gob encoding.
|
|
// If path is empty, this is a no-op. Creates parent directories if needed.
|
|
func (rc *ResponseCache) SaveToFile(path string) error {
|
|
if path == "" {
|
|
return nil
|
|
}
|
|
|
|
rc.mu.RLock()
|
|
data := responseCachePersist{
|
|
Entries: make(map[string]*CachedResponse, len(rc.entries)),
|
|
Order: append([]string(nil), rc.order...),
|
|
Stats: rc.stats,
|
|
}
|
|
for k, v := range rc.entries {
|
|
data.Entries[k] = v
|
|
}
|
|
rc.mu.RUnlock()
|
|
|
|
// Ensure parent directory exists
|
|
dir := filepath.Dir(path)
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Write to temp file then rename for atomicity
|
|
tmpPath := path + ".tmp"
|
|
f, err := os.Create(tmpPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
enc := gob.NewEncoder(f)
|
|
if err := enc.Encode(data); err != nil {
|
|
_ = f.Close()
|
|
_ = os.Remove(tmpPath)
|
|
return err
|
|
}
|
|
if err := f.Close(); err != nil {
|
|
_ = os.Remove(tmpPath)
|
|
return err
|
|
}
|
|
|
|
if err := os.Rename(tmpPath, path); err != nil {
|
|
_ = os.Remove(tmpPath)
|
|
return err
|
|
}
|
|
|
|
log.Debugf("response cache saved to %s (%d entries)", path, len(data.Entries))
|
|
return nil
|
|
}
|
|
|
|
// LoadFromFile loads the cache from a file using gob decoding.
|
|
// If path is empty or the file doesn't exist, this is a no-op.
|
|
// Expired entries are skipped during load.
|
|
func (rc *ResponseCache) LoadFromFile(path string) error {
|
|
if path == "" {
|
|
return nil
|
|
}
|
|
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil // No cache file yet, not an error
|
|
}
|
|
return err
|
|
}
|
|
defer func() { _ = f.Close() }()
|
|
|
|
var data responseCachePersist
|
|
dec := gob.NewDecoder(f)
|
|
if err := dec.Decode(&data); err != nil {
|
|
return err
|
|
}
|
|
|
|
now := time.Now()
|
|
rc.mu.Lock()
|
|
defer rc.mu.Unlock()
|
|
|
|
// Load entries, skipping expired ones
|
|
loaded := 0
|
|
for _, key := range data.Order {
|
|
entry, exists := data.Entries[key]
|
|
if !exists || entry == nil {
|
|
continue
|
|
}
|
|
// Skip expired entries
|
|
if now.Sub(entry.CreatedAt) > rc.config.TTL {
|
|
continue
|
|
}
|
|
// Skip if over capacity
|
|
entry.SizeBytes = ensureResponseEntrySize(entry)
|
|
if (rc.config.MaxSize > 0 && len(rc.entries) >= rc.config.MaxSize) ||
|
|
(rc.config.MaxBytes > 0 && rc.totalBytes+entry.SizeBytes > rc.config.MaxBytes) {
|
|
break
|
|
}
|
|
rc.entries[key] = entry
|
|
rc.order = append(rc.order, key)
|
|
rc.totalBytes += entry.SizeBytes
|
|
loaded++
|
|
}
|
|
rc.stats.Size = len(rc.entries)
|
|
|
|
log.Infof("response cache loaded from %s (%d entries)", path, loaded)
|
|
return nil
|
|
}
|
|
|
|
func estimateResponseEntrySize(model string, response []byte, contentType string) int64 {
|
|
return int64(len(response) + len(model) + len(contentType))
|
|
}
|
|
|
|
func ensureResponseEntrySize(entry *CachedResponse) int64 {
|
|
if entry == nil {
|
|
return 0
|
|
}
|
|
if entry.SizeBytes > 0 {
|
|
return entry.SizeBytes
|
|
}
|
|
entry.SizeBytes = estimateResponseEntrySize(entry.Model, entry.Response, entry.ContentType)
|
|
return entry.SizeBytes
|
|
}
|
|
|
|
// StartPeriodicPersistence starts a background goroutine that periodically saves the cache.
|
|
// The goroutine stops when the context is cancelled.
|
|
func (rc *ResponseCache) StartPeriodicPersistence(ctx context.Context, path string, interval time.Duration) {
|
|
if path == "" || interval <= 0 {
|
|
return
|
|
}
|
|
go func() {
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
// Final save on shutdown
|
|
if err := rc.SaveToFile(path); err != nil {
|
|
log.Warnf("failed to save response cache on shutdown: %v", err)
|
|
}
|
|
return
|
|
case <-ticker.C:
|
|
if err := rc.SaveToFile(path); err != nil {
|
|
log.Warnf("failed to save response cache: %v", err)
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
}
|