Files
ProxyPilot/internal/cache/prompt_cache.go
T

717 lines
19 KiB
Go

package cache
import (
"context"
"crypto/sha256"
"encoding/gob"
"encoding/hex"
"encoding/json"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/router-for-me/CLIProxyAPI/v6/internal/api/middleware"
log "github.com/sirupsen/logrus"
)
// PromptCacheConfig defines configuration for the prompt cache.
type PromptCacheConfig struct {
// Enabled controls whether prompt caching is active.
Enabled bool `yaml:"enabled" json:"enabled"`
// MaxSize is the maximum number of cached prompts.
MaxSize int `yaml:"max-size" json:"max-size"`
// MaxBytes is the maximum total size (bytes) of cached prompts.
// 0 disables size-based eviction.
MaxBytes int64 `yaml:"max-bytes" json:"max-bytes"`
// TTL is how long prompts are cached.
TTL time.Duration `yaml:"ttl" json:"ttl"`
// PersistFile is the optional file path to persist cache across restarts.
PersistFile string `yaml:"persist-file" json:"persist-file"`
}
// DefaultPromptCacheConfig returns sensible defaults.
func DefaultPromptCacheConfig() PromptCacheConfig {
return PromptCacheConfig{
Enabled: false, // Opt-in by default
MaxSize: 500,
TTL: 30 * time.Minute,
}
}
// CachedPrompt stores a cached system prompt with metadata.
type CachedPrompt struct {
// Hash is the SHA256 hash of the prompt content.
Hash string
// Prompt is the full system prompt text.
Prompt string
// TokenEstimate is an estimated token count for this prompt.
TokenEstimate int
// CreatedAt is when the prompt was cached.
CreatedAt time.Time
// HitCount tracks how many times this cache entry was hit.
HitCount int64
// LastHit is the last time this entry was accessed.
LastHit time.Time
// Provider tracks which provider(s) this prompt was used with.
Providers map[string]int
// SizeBytes tracks the approximate size of this entry.
SizeBytes int64
}
// PromptCache provides LRU-based caching for system prompts.
// This tracks repeated system prompts to enable synthetic caching for
// providers that don't support native prompt caching.
type PromptCache struct {
mu sync.RWMutex
entries map[string]*CachedPrompt
order []string // LRU order tracking
config PromptCacheConfig
stats PromptCacheStats
totalBytes int64
disabled bool // runtime toggle
}
// PromptCacheStats tracks cache performance metrics.
type PromptCacheStats struct {
// Hits is the number of cache hits.
Hits int64 `json:"hits"`
// Misses is the number of cache misses.
Misses int64 `json:"misses"`
// Evictions is the number of evicted entries.
Evictions int64 `json:"evictions"`
// Size is the current number of cached entries.
Size int `json:"size"`
// UniquePrompts is the total number of unique prompts seen.
UniquePrompts int64 `json:"unique_prompts"`
// TotalRequests is the total number of cache operations.
TotalRequests int64 `json:"total_requests"`
// EstimatedTokensSaved estimates tokens saved from cache hits.
EstimatedTokensSaved int64 `json:"estimated_tokens_saved"`
// TopProviders tracks hit counts by provider.
TopProviders map[string]int64 `json:"top_providers,omitempty"`
}
// NewPromptCache creates a new prompt cache with the given config.
func NewPromptCache(cfg PromptCacheConfig) *PromptCache {
return &PromptCache{
entries: make(map[string]*CachedPrompt),
order: make([]string, 0, cfg.MaxSize),
config: cfg,
disabled: !cfg.Enabled,
stats: PromptCacheStats{
TopProviders: make(map[string]int64),
},
}
}
// defaultPromptCache is the package-level cache instance.
var (
defaultPromptCache *PromptCache
defaultPromptCacheOnce sync.Once
)
// GetDefaultPromptCache returns the package-level prompt cache.
func GetDefaultPromptCache() *PromptCache {
defaultPromptCacheOnce.Do(func() {
defaultPromptCache = NewPromptCache(DefaultPromptCacheConfig())
})
return defaultPromptCache
}
// InitDefaultPromptCache initializes the default cache with custom config.
func InitDefaultPromptCache(cfg PromptCacheConfig) {
defaultPromptCacheOnce.Do(func() {
defaultPromptCache = NewPromptCache(cfg)
})
// If already initialized, update config
if defaultPromptCache != nil {
defaultPromptCache.UpdateConfig(cfg)
}
}
// HashPrompt generates a hash for a system prompt.
func HashPrompt(prompt string) string {
h := sha256.New()
h.Write([]byte(prompt))
return hex.EncodeToString(h.Sum(nil))[:32]
}
// estimateTokens provides a rough token estimate based on character count.
// Uses ~4 characters per token as a rough estimate.
func estimateTokens(prompt string) int {
return (len(prompt) + 3) / 4
}
// CacheSystemPrompt caches a system prompt and returns its hash.
// If the prompt already exists, it increments the hit count.
// Returns (hash, isNewPrompt).
func (pc *PromptCache) CacheSystemPrompt(prompt string, provider string) (string, bool) {
if pc.disabled || !pc.config.Enabled || prompt == "" {
return HashPrompt(prompt), false
}
hash := HashPrompt(prompt)
pc.mu.Lock()
defer pc.mu.Unlock()
pc.stats.TotalRequests++
entry, exists := pc.entries[hash]
if exists {
// Check TTL
if time.Since(entry.CreatedAt) > pc.config.TTL {
pc.removeEntryLocked(hash)
pc.stats.Evictions++
exists = false
}
}
if exists {
entry.HitCount++
entry.LastHit = time.Now()
if provider != "" {
if entry.Providers == nil {
entry.Providers = make(map[string]int)
}
entry.Providers[provider]++
pc.stats.TopProviders[provider]++
}
pc.stats.Hits++
pc.stats.EstimatedTokensSaved += int64(entry.TokenEstimate)
pc.moveToEndLocked(hash)
middleware.RecordPromptCacheHit()
middleware.RecordPromptCacheTokensSaved(entry.TokenEstimate)
log.Debugf("prompt cache HIT hash=%s provider=%s hits=%d", hash[:8], provider, entry.HitCount)
return hash, false
}
pc.stats.Misses++
pc.stats.UniquePrompts++
middleware.RecordPromptCacheMiss()
// Evict if at capacity
entrySize := estimatePromptEntrySize(prompt, hash)
if pc.config.MaxBytes > 0 && entrySize > pc.config.MaxBytes {
return hash, false
}
for (pc.config.MaxSize > 0 && len(pc.entries) >= pc.config.MaxSize && len(pc.order) > 0) ||
(pc.config.MaxBytes > 0 && pc.totalBytes+entrySize > pc.config.MaxBytes && len(pc.order) > 0) {
pc.removeEntryLocked(pc.order[0])
pc.stats.Evictions++
}
// Store new entry
tokenEst := estimateTokens(prompt)
providers := make(map[string]int)
if provider != "" {
providers[provider] = 1
pc.stats.TopProviders[provider]++
}
pc.entries[hash] = &CachedPrompt{
Hash: hash,
Prompt: prompt,
TokenEstimate: tokenEst,
CreatedAt: time.Now(),
HitCount: 0,
LastHit: time.Now(),
Providers: providers,
SizeBytes: entrySize,
}
pc.order = append(pc.order, hash)
pc.totalBytes += entrySize
pc.stats.Size = len(pc.entries)
middleware.SetPromptCacheSize(len(pc.entries))
log.Debugf("prompt cache SET hash=%s provider=%s tokens~%d size=%d/%d",
hash[:8], provider, tokenEst, len(pc.entries), pc.config.MaxSize)
return hash, true
}
// GetCachedPrompt retrieves a cached prompt by its hash.
// Returns nil if not found or expired.
func (pc *PromptCache) GetCachedPrompt(hash string) *CachedPrompt {
if pc.disabled || !pc.config.Enabled {
return nil
}
pc.mu.RLock()
entry, exists := pc.entries[hash]
pc.mu.RUnlock()
if !exists {
return nil
}
// Check TTL
if time.Since(entry.CreatedAt) > pc.config.TTL {
pc.mu.Lock()
pc.removeEntryLocked(hash)
pc.stats.Evictions++
pc.mu.Unlock()
return nil
}
return entry
}
// LookupByPrompt checks if a prompt is cached and returns hit info.
// Returns (hash, hitCount, found).
func (pc *PromptCache) LookupByPrompt(prompt string) (string, int64, bool) {
if pc.disabled || !pc.config.Enabled || prompt == "" {
return "", 0, false
}
hash := HashPrompt(prompt)
pc.mu.RLock()
entry, exists := pc.entries[hash]
pc.mu.RUnlock()
if !exists {
return hash, 0, false
}
// Check TTL
if time.Since(entry.CreatedAt) > pc.config.TTL {
return hash, 0, false
}
return hash, entry.HitCount, true
}
// moveToEndLocked moves a key to the end of the LRU order.
// Caller must hold the lock.
func (pc *PromptCache) moveToEndLocked(key string) {
for i, k := range pc.order {
if k == key {
pc.order = append(pc.order[:i], pc.order[i+1:]...)
pc.order = append(pc.order, key)
return
}
}
}
// removeFromOrderLocked removes a key from the LRU order.
// Caller must hold the lock.
func (pc *PromptCache) removeFromOrderLocked(key string) {
for i, k := range pc.order {
if k == key {
pc.order = append(pc.order[:i], pc.order[i+1:]...)
return
}
}
}
func (pc *PromptCache) removeEntryLocked(key string) {
entry := pc.entries[key]
if entry != nil {
pc.totalBytes -= entry.SizeBytes
}
delete(pc.entries, key)
pc.removeFromOrderLocked(key)
}
// Clear removes all entries from the cache.
func (pc *PromptCache) Clear() {
pc.mu.Lock()
defer pc.mu.Unlock()
pc.entries = make(map[string]*CachedPrompt)
pc.order = make([]string, 0, pc.config.MaxSize)
pc.totalBytes = 0
pc.stats.Size = 0
middleware.SetPromptCacheSize(0)
}
// GetStats returns current cache statistics.
func (pc *PromptCache) GetStats() PromptCacheStats {
pc.mu.RLock()
defer pc.mu.RUnlock()
stats := pc.stats
stats.Size = len(pc.entries)
// Copy provider map to avoid race
if len(pc.stats.TopProviders) > 0 {
stats.TopProviders = make(map[string]int64, len(pc.stats.TopProviders))
for k, v := range pc.stats.TopProviders {
stats.TopProviders[k] = v
}
}
return stats
}
// SetEnabled enables or disables the cache at runtime.
func (pc *PromptCache) SetEnabled(enabled bool) {
pc.mu.Lock()
defer pc.mu.Unlock()
pc.disabled = !enabled
pc.config.Enabled = enabled
if !enabled {
log.Info("prompt cache disabled")
} else {
log.Info("prompt cache enabled")
}
}
// IsEnabled returns whether the cache is enabled.
func (pc *PromptCache) IsEnabled() bool {
pc.mu.RLock()
defer pc.mu.RUnlock()
return pc.config.Enabled && !pc.disabled
}
// UpdateConfig updates the cache configuration.
func (pc *PromptCache) UpdateConfig(cfg PromptCacheConfig) {
pc.mu.Lock()
defer pc.mu.Unlock()
pc.config = cfg
pc.disabled = !cfg.Enabled
// Evict if new max size is smaller
for (cfg.MaxSize > 0 && len(pc.entries) > cfg.MaxSize && len(pc.order) > 0) ||
(cfg.MaxBytes > 0 && pc.totalBytes > cfg.MaxBytes && len(pc.order) > 0) {
pc.removeEntryLocked(pc.order[0])
pc.stats.Evictions++
}
pc.stats.Size = len(pc.entries)
}
// EvictExpired removes all expired entries from the cache.
func (pc *PromptCache) EvictExpired() int {
pc.mu.Lock()
defer pc.mu.Unlock()
evicted := 0
now := time.Now()
newOrder := make([]string, 0, len(pc.order))
for _, key := range pc.order {
entry, exists := pc.entries[key]
if !exists {
continue
}
if now.Sub(entry.CreatedAt) > pc.config.TTL {
pc.totalBytes -= entry.SizeBytes
delete(pc.entries, key)
evicted++
pc.stats.Evictions++
} else {
newOrder = append(newOrder, key)
}
}
pc.order = newOrder
pc.stats.Size = len(pc.entries)
middleware.SetPromptCacheSize(len(pc.entries))
return evicted
}
// GetTopPrompts returns the most frequently hit prompts.
func (pc *PromptCache) GetTopPrompts(limit int) []*CachedPrompt {
pc.mu.RLock()
defer pc.mu.RUnlock()
if limit <= 0 {
limit = 10
}
// Collect all entries
entries := make([]*CachedPrompt, 0, len(pc.entries))
for _, entry := range pc.entries {
entries = append(entries, entry)
}
// Sort by hit count (simple selection for small lists)
for i := 0; i < len(entries) && i < limit; i++ {
maxIdx := i
for j := i + 1; j < len(entries); j++ {
if entries[j].HitCount > entries[maxIdx].HitCount {
maxIdx = j
}
}
entries[i], entries[maxIdx] = entries[maxIdx], entries[i]
}
if len(entries) > limit {
entries = entries[:limit]
}
return entries
}
// TrackSystemPrompt is a convenience function for the default cache.
func TrackSystemPrompt(prompt string, provider string) (string, bool) {
cache := GetDefaultPromptCache()
if cache == nil || !cache.IsEnabled() {
return HashPrompt(prompt), false
}
return cache.CacheSystemPrompt(prompt, provider)
}
// IsPromptCached checks if a prompt is already cached in the default cache.
func IsPromptCached(prompt string) (string, int64, bool) {
cache := GetDefaultPromptCache()
if cache == nil || !cache.IsEnabled() {
return HashPrompt(prompt), 0, false
}
return cache.LookupByPrompt(prompt)
}
// promptCachePersist is the serializable form of the cache for gob encoding.
type promptCachePersist struct {
Entries map[string]*CachedPrompt
Order []string
Stats PromptCacheStats
}
// SaveToFile persists the cache to a file using gob encoding.
// If path is empty, this is a no-op. Creates parent directories if needed.
func (pc *PromptCache) SaveToFile(path string) error {
if path == "" {
return nil
}
pc.mu.RLock()
data := promptCachePersist{
Entries: make(map[string]*CachedPrompt, len(pc.entries)),
Order: append([]string(nil), pc.order...),
Stats: pc.stats,
}
// Deep copy stats.TopProviders to avoid race
if len(pc.stats.TopProviders) > 0 {
data.Stats.TopProviders = make(map[string]int64, len(pc.stats.TopProviders))
for k, v := range pc.stats.TopProviders {
data.Stats.TopProviders[k] = v
}
}
for k, v := range pc.entries {
data.Entries[k] = v
}
pc.mu.RUnlock()
// Ensure parent directory exists
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0o755); err != nil {
return err
}
// Write to temp file then rename for atomicity
tmpPath := path + ".tmp"
f, err := os.Create(tmpPath)
if err != nil {
return err
}
enc := gob.NewEncoder(f)
if err := enc.Encode(data); err != nil {
_ = f.Close()
_ = os.Remove(tmpPath)
return err
}
if err := f.Close(); err != nil {
_ = os.Remove(tmpPath)
return err
}
if err := os.Rename(tmpPath, path); err != nil {
_ = os.Remove(tmpPath)
return err
}
log.Debugf("prompt cache saved to %s (%d entries)", path, len(data.Entries))
return nil
}
// LoadFromFile loads the cache from a file using gob decoding.
// If path is empty or the file doesn't exist, this is a no-op.
// Expired entries are skipped during load.
func (pc *PromptCache) LoadFromFile(path string) error {
if path == "" {
return nil
}
f, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return nil // No cache file yet, not an error
}
return err
}
defer func() { _ = f.Close() }()
var data promptCachePersist
dec := gob.NewDecoder(f)
if err := dec.Decode(&data); err != nil {
return err
}
now := time.Now()
pc.mu.Lock()
defer pc.mu.Unlock()
// Load entries, skipping expired ones
loaded := 0
for _, key := range data.Order {
entry, exists := data.Entries[key]
if !exists || entry == nil {
continue
}
// Skip expired entries
if now.Sub(entry.CreatedAt) > pc.config.TTL {
continue
}
// Skip if over capacity
entry.SizeBytes = ensurePromptEntrySize(entry)
if (pc.config.MaxSize > 0 && len(pc.entries) >= pc.config.MaxSize) ||
(pc.config.MaxBytes > 0 && pc.totalBytes+entry.SizeBytes > pc.config.MaxBytes) {
break
}
pc.entries[key] = entry
pc.order = append(pc.order, key)
pc.totalBytes += entry.SizeBytes
loaded++
}
pc.stats.Size = len(pc.entries)
// Restore stats from persisted data
pc.stats.Hits = data.Stats.Hits
pc.stats.Misses = data.Stats.Misses
pc.stats.Evictions = data.Stats.Evictions
pc.stats.UniquePrompts = data.Stats.UniquePrompts
pc.stats.TotalRequests = data.Stats.TotalRequests
pc.stats.EstimatedTokensSaved = data.Stats.EstimatedTokensSaved
if len(data.Stats.TopProviders) > 0 {
pc.stats.TopProviders = make(map[string]int64, len(data.Stats.TopProviders))
for k, v := range data.Stats.TopProviders {
pc.stats.TopProviders[k] = v
}
}
log.Infof("prompt cache loaded from %s (%d entries)", path, loaded)
return nil
}
func estimatePromptEntrySize(prompt, hash string) int64 {
return int64(len(prompt) + len(hash))
}
func ensurePromptEntrySize(entry *CachedPrompt) int64 {
if entry == nil {
return 0
}
if entry.SizeBytes > 0 {
return entry.SizeBytes
}
entry.SizeBytes = estimatePromptEntrySize(entry.Prompt, entry.Hash)
return entry.SizeBytes
}
// StartPeriodicPersistence starts a background goroutine that periodically saves the cache.
// The goroutine stops when the context is cancelled.
func (pc *PromptCache) StartPeriodicPersistence(ctx context.Context, path string, interval time.Duration) {
if path == "" || interval <= 0 {
return
}
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
// Final save on shutdown
if err := pc.SaveToFile(path); err != nil {
log.Warnf("failed to save prompt cache on shutdown: %v", err)
}
return
case <-ticker.C:
if err := pc.SaveToFile(path); err != nil {
log.Warnf("failed to save prompt cache: %v", err)
}
}
}
}()
}
// WarmPromptEntry represents a prompt to pre-load into the cache.
type WarmPromptEntry struct {
// Prompt is the system prompt text to cache.
Prompt string `json:"prompt"`
// Provider is the provider name (optional).
Provider string `json:"provider"`
}
// WarmCacheResult tracks the result of a cache warming operation.
type WarmCacheResult struct {
// Total is the number of prompts processed.
Total int `json:"total"`
// Added is the number of new prompts added to cache.
Added int `json:"added"`
// Skipped is the number of prompts already in cache.
Skipped int `json:"skipped"`
// Errors is the number of prompts that failed.
Errors int `json:"errors"`
}
// WarmCache pre-populates the cache with the given prompts.
// This is useful for loading known expensive system prompts on startup.
func (pc *PromptCache) WarmCache(prompts []WarmPromptEntry) WarmCacheResult {
result := WarmCacheResult{Total: len(prompts)}
for _, entry := range prompts {
prompt := strings.TrimSpace(entry.Prompt)
if prompt == "" {
result.Errors++
continue
}
// CacheSystemPrompt returns (hash, isNew)
_, isNew := pc.CacheSystemPrompt(prompt, entry.Provider)
if isNew {
result.Added++
} else {
result.Skipped++
}
}
log.Infof("prompt cache warmed: added=%d skipped=%d errors=%d total=%d",
result.Added, result.Skipped, result.Errors, result.Total)
return result
}
// LoadWarmFile reads prompts from a JSON file and warms the cache.
// Expected format: [{"prompt": "...", "provider": "claude"}, ...]
// Returns the warming result or an error if the file cannot be read.
func (pc *PromptCache) LoadWarmFile(path string) (WarmCacheResult, error) {
if path == "" {
return WarmCacheResult{}, nil
}
data, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
log.Debugf("warm file not found: %s", path)
return WarmCacheResult{}, nil
}
return WarmCacheResult{}, err
}
var prompts []WarmPromptEntry
if err := json.Unmarshal(data, &prompts); err != nil {
return WarmCacheResult{}, err
}
result := pc.WarmCache(prompts)
log.Infof("prompt cache warmed from file %s: added=%d skipped=%d",
path, result.Added, result.Skipped)
return result, nil
}
// WarmDefaultCache warms the default cache from a file.
func WarmDefaultCache(path string) (WarmCacheResult, error) {
cache := GetDefaultPromptCache()
if cache == nil {
return WarmCacheResult{}, nil
}
return cache.LoadWarmFile(path)
}