Merge FreeRide feature and doc updates into security_shield_v2

2026-04-19 22:19:07 +02:00 · 2026-04-19 22:19:07 +02:00 · 383132dbe6
commit 383132dbe6
parent 1f9ecedf3b 9d9b075ea7
23 changed files with 401 additions and 91 deletions
--- a/README.md
+++ b/README.md
@ -96,8 +96,8 @@
 👁️ **Vision pipeline**: Send images and files directly to the Agent — automatic base64 encoding for multimodal LLMs.

 🧠 **Smart routing**: Rule-based model routing — simple queries go to lightweight models, saving API costs.
-
 🧬 **FreeRide**: Intelligent model rotation using OpenRouter's free pool — never pay for basic LLM traffic again. [Learn more](docs/guides/freeride.md).
+❄️ **Persistent Cooldowns**: Remembers rate-limited models across restarts via `cooldowns.json`, ensuring instant failover and zero "cooldown amnesia".

 _*Recent builds may use 10-20MB due to rapid PR merges. Resource optimization is planned. Boot speed comparison based on 0.8GHz single-core benchmarks (see table below)._

@ -624,7 +624,7 @@ For detailed guides beyond this README:
 | [Scheduled Tasks and Cron Jobs](docs/reference/cron.md) | Cron schedule types, deliver modes, command gates, job storage |
 | [Providers & Models](docs/guides/providers.md) | 30+ LLM providers, model routing, model_list configuration |
 | [Spawn & Async Tasks](docs/guides/spawn-tasks.md) | Quick tasks, long tasks with spawn, async sub-agent orchestration |
-| [FreeRide](docs/guides/freeride.md) | Dynamic free model rotation and K3s secret management |
+| [FreeRide](docs/freeride.md) | Dynamic free model rotation and K3s secret management |
 | [Hooks](docs/architecture/hooks/README.md) | Event-driven hook system: observers, interceptors, approval hooks |
 | [Steering](docs/architecture/steering.md) | Inject messages into a running agent loop between tool calls |
 | [SubTurn](docs/architecture/subturn.md) | Subagent coordination, concurrency control, lifecycle |
--- a/cmd/picoclaw/internal/freeride/command.go
+++ b/cmd/picoclaw/internal/freeride/command.go
@ -0,0 +1,84 @@
+package freeride
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/sipeed/picoclaw/cmd/picoclaw/internal"
+	"github.com/sipeed/picoclaw/pkg/tools"
+)
+
+// NewFreerideCommand returns a new cobra.Command for managing OpenRouter free models.
+func NewFreerideCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "freeride",
+		Short: "Manage OpenRouter free models and fallbacks",
+		Long:  "FreeRide automatically discovers and configures OpenRouter's best free models as fallbacks for your PicoClaw agent.",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return cmd.Help()
+		},
+	}
+
+	cmd.AddCommand(
+		newListCommand(),
+		newAutoCommand(),
+		newStatusCommand(),
+	)
+
+	return cmd
+}
+
+func newListCommand() *cobra.Command {
+	var limit int
+	cmd := &cobra.Command{
+		Use:   "list",
+		Short: "List available free models from OpenRouter",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			t := tools.NewFreeRideTool(internal.GetConfigPath(), nil)
+			result := t.Execute(context.Background(), map[string]any{
+				"command": "list",
+				"limit":   float64(limit),
+			})
+			fmt.Println(result.ForLLM)
+			return nil
+		},
+	}
+	cmd.Flags().IntVarP(&limit, "limit", "l", 10, "Number of models to list")
+	return cmd
+}
+
+func newAutoCommand() *cobra.Command {
+	var limit int
+	cmd := &cobra.Command{
+		Use:   "auto",
+		Short: "Automatically configure best free models as fallbacks",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			t := tools.NewFreeRideTool(internal.GetConfigPath(), nil)
+			result := t.Execute(context.Background(), map[string]any{
+				"command": "auto",
+				"limit":   float64(limit),
+			})
+			fmt.Println(result.ForLLM)
+			return nil
+		},
+	}
+	cmd.Flags().IntVarP(&limit, "limit", "l", 5, "Number of fallbacks to configure")
+	return cmd
+}
+
+func newStatusCommand() *cobra.Command {
+	return &cobra.Command{
+		Use:   "status",
+		Short: "Check current FreeRide configuration",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			t := tools.NewFreeRideTool(internal.GetConfigPath(), nil)
+			result := t.Execute(context.Background(), map[string]any{
+				"command": "status",
+			})
+			fmt.Println(result.ForLLM)
+			return nil
+		},
+	}
+}
--- a/cmd/picoclaw/main.go
+++ b/cmd/picoclaw/main.go
@ -18,6 +18,7 @@ import (
 	"github.com/sipeed/picoclaw/cmd/picoclaw/internal/auth"
 	"github.com/sipeed/picoclaw/cmd/picoclaw/internal/cliui"
 	"github.com/sipeed/picoclaw/cmd/picoclaw/internal/cron"
+	"github.com/sipeed/picoclaw/cmd/picoclaw/internal/freeride"
 	"github.com/sipeed/picoclaw/cmd/picoclaw/internal/gateway"
 	"github.com/sipeed/picoclaw/cmd/picoclaw/internal/migrate"
 	"github.com/sipeed/picoclaw/cmd/picoclaw/internal/model"
@ -84,6 +85,7 @@ picoclaw --no-color status`,
 		onboard.NewOnboardCommand(),
 		agent.NewAgentCommand(),
 		auth.NewAuthCommand(),
+		freeride.NewFreerideCommand(),
 		gateway.NewGatewayCommand(),
 		status.NewStatusCommand(),
 		cron.NewCronCommand(),
--- a/cmd/picoclaw/main_test.go
+++ b/cmd/picoclaw/main_test.go
@ -40,6 +40,7 @@ func TestNewPicoclawCommand(t *testing.T) {
 		"agent",
 		"auth",
 		"cron",
+		"freeride",
 		"gateway",
 		"migrate",
 		"model",
--- a/docs/guides/freeride.md
+++ b/docs/guides/freeride.md
@ -12,23 +12,16 @@ FreeRide is a dynamic model rotation and failover system for PicoClaw that lever

 ## Configuration

-FreeRide is implemented as a native PicoClaw tool.
+FreeRide is implemented as a native PicoClaw tool. For production environments (especially in the **main branch**), ensure you follow the [Security Configuration](../security/security_configuration.md) to manage your API keys safely.

 ### 1. Enable the Tool
-Ensure the `freeride` tool is enabled and whitelisted in your `config.json`:
-
+Ensure the `skills` tool is enabled in your `config.json` (FreeRide is bundled with the skills system):
+ 
 ```json
 {
  "tools": {
-    "whitelist": ["freeride", ...],
-    "whitelist_enabled": true,
-    "security_policy": {
-      "enabled": true,
-      "config": {
-        "allowed_tools": {
-          "freeride": true
-        }
-      }
+    "skills": {
+      "enabled": true
    }
  }
 }
@ -106,11 +99,20 @@ env:
        key: openrouter-api-key
 ```

+## Cooldown Persistence & Timing ❄️
+ 
+To prevent the agent from "hanging" or retrying known-failed models, PicoClaw uses a two-pronged approach:
+ 
+### 1. Zero-Amnesia Persistence
+Model failures (e.g., 429 Rate Limits) are saved to `~/.picoclaw/cooldowns.json`. This ensures that if you restart the agent, it **remembers** which models were saturated and skips them instantly. You no longer have to wait through a series of timeouts every time you restart.
+ 
+### 2. Aggressive 30s Timeout
+The default request timeout for LLM calls is **30 seconds**. If a free model is stalled or unresponsive, the agent will move to the next fallback in your pool much faster than the standard HTTP default.
+ 
 ## Troubleshooting

 - **404 Errors**: Ensure the model is still available on OpenRouter using `freeride list`. If it's gone, run `freeride auto` to refresh your fallback pool.
- **429 Rate Limiting**: This is common with free models. PicoClaw will automatically try the next model in your `model_fallbacks` list.
- **Security Blocks**: Ensure `freeride` is added to your `security_policy` allowed tools map.
+- **429 Rate Limiting**: This is common with free models. PicoClaw will automatically try the next model in your `model_fallbacks` list and persist the cooldown to `cooldowns.json`.

 ---

--- a/docs/reference/rate-limiting.md
+++ b/docs/reference/rate-limiting.md
@ -25,13 +25,20 @@ AgentLoop.callLLM()
 The rate limiter runs **after** the cooldown check and **before** the provider call, so:
 - Candidates already in cooldown are skipped entirely (no token consumed)
 - Candidates that are available get throttled to the configured RPM
-
-The same check applies in `ExecuteImage`.
-
+ 
+## Cooldown Persistence ❄️
+ 
+While the Rate Limiter is **proactive**, the Cooldown Tracker is **reactive** (handled *after* a 429 is actually received). 
+ 
+To ensure stability across restarts, the Cooldown Tracker persists its state to disk:
+- **Location**: `~/.picoclaw/cooldowns.json` (or sibling to your workspace)
+- **Behavior**: If the agent is restarted, it loads the failure history and continues to enforce cooldowns. This prevents "initialization hangs" where a new agent process tries a long list of models that are already known to be rate-limited.
+- **Timing**: Default LLM request timeout is now **30 seconds** to ensure faster failover.
+ 
 ### Thread safety
-
+ 
 `RateLimiterRegistry` is safe for concurrent use. The per-limiter token bucket uses a fine-grained mutex so concurrent goroutines each acquire their own token independently.
-
+ 
 ## Configuration

 Set `rpm` on any model in `model_list`:
--- a/docs/security/security_configuration.md
+++ b/docs/security/security_configuration.md
@ -541,6 +541,19 @@ go test ./pkg/config -run TestSecurityConfig
 - Remember: Models, Brave, Tavily, Perplexity MUST use `api_keys` (array format)
 - GLMSearch and BaiduSearch MUST use `api_key` (single string format)

+## FreeRide & Dynamic Failover Security
+
+When using the **FreeRide** dynamic model failover system, PicoClaw dynamically adds models to your `model_list`. To ensure these models stay secure:
+
+1. **Protocol-Level Security**: You can define a single entry in `.security.yml` that matches a specific provider/protocol. For example, to provide an identity for ALL OpenRouter models added by FreeRide:
+   ```yaml
+   model_list:
+     openrouter:
+       api_keys: ["sk-or-v1-your-global-key"]
+   ```
+2. **Environment Variable Fallback**: FreeRide is configured to look for `OPENROUTER_API_KEY` in the environment. In production (`main` branch), it is recommended to use K3s Secrets or `SecureString` to inject this.
+3. **Sensitive Filtering**: All models added by FreeRide are subject to the same `sensitive_data_filtering.md` rules as your primary models.
+
 ### Load Balancing/Failover Issues

 - Verify all API keys in the `api_keys` array are valid
--- a/examples/freeride-config.json
+++ b/examples/freeride-config.json
@ -0,0 +1,37 @@
+{
+  "agents": {
+    "defaults": {
+      "model_name": "google/gemma-3-27b-it:free",
+      "model_fallbacks": [
+        "google/gemma-3-27b-it:free",
+        "nvidia/nemotron-4-340b-instruct:free",
+        "qwen/qwen-2.5-72b-instruct:free",
+        "mistralai/mistral-small-24b-it-v1:free"
+      ],
+      "max_tokens": 4096,
+      "max_tool_iterations": 10
+    }
+  },
+  "model_list": [
+    {
+      "model_name": "google/gemma-3-27b-it:free",
+      "model": "google/gemma-3-27b-it:free",
+      "protocol": "openrouter"
+    },
+    {
+      "model_name": "nvidia/nemotron-4-340b-instruct:free",
+      "model": "nvidia/nemotron-4-340b-instruct:free",
+      "protocol": "openrouter"
+    },
+    {
+      "model_name": "qwen/qwen-2.5-72b-instruct:free",
+      "model": "qwen/qwen-2.5-72b-instruct:free",
+      "protocol": "openrouter"
+    },
+    {
+      "model_name": "mistralai/mistral-small-24b-it-v1:free",
+      "model": "mistralai/mistral-small-24b-it-v1:free",
+      "protocol": "openrouter"
+    }
+  ]
+}
--- a/pkg/agent/loop.go
+++ b/pkg/agent/loop.go
@ -64,6 +64,7 @@ type AgentLoop struct {
 	turnSeq        atomic.Uint64
 	activeRequests sync.WaitGroup
 	configPath     string
+	cooldownPath   string

 	reloadFunc func() error

@ -386,7 +387,7 @@ func (al *AgentLoop) ReloadProviderAndConfig(
 			newRL.RegisterCandidates(agent.LightCandidates)
 		}
 	}
-	al.fallback = providers.NewFallbackChain(providers.NewCooldownTracker(), newRL)
+	al.fallback = providers.NewFallbackChain(providers.NewCooldownTracker(al.cooldownPath), newRL)

 	al.mu.Unlock()

@ -521,16 +522,17 @@ func (al *AgentLoop) runAgentLoop(
 		}
 	}

-	if opts.SendResponse && result.finalContent != "" {
+	finalContent := result.finalContent
+	if usedFallback, fallbackModel := ts.GetFallbackInfo(); usedFallback {
+		finalContent += fmt.Sprintf("\n\n🦞 _(FreeRide: %s)_", fallbackModel)
+	}
+
+	if opts.SendResponse && finalContent != "" {
 		agentID, sessionKey, scope := outboundTurnMetadata(
 			agent.ID,
 			opts.Dispatch.SessionKey,
 			opts.Dispatch.SessionScope,
 		)
-		finalContent := result.finalContent
-		if usedFallback, fallbackModel := ts.GetFallbackInfo(); usedFallback {
-			finalContent += fmt.Sprintf("\n\n🦞 _(FreeRide: %s)_", fallbackModel)
-		}
 		al.bus.PublishOutbound(ctx, bus.OutboundMessage{
 			Context: outboundContextFromInbound(
 				opts.Dispatch.InboundContext,
@ -545,7 +547,7 @@ func (al *AgentLoop) runAgentLoop(
 		})
 	}

-	return result.finalContent, nil
+	return finalContent, nil
 }

 // selectCandidates returns the model candidates and resolved model name to use
--- a/pkg/agent/loop_init.go
+++ b/pkg/agent/loop_init.go
@ -5,6 +5,7 @@ package agent
 import (
 	"context"
 	"fmt"
+	"path/filepath"
 	"time"

 	"github.com/sipeed/picoclaw/pkg/audio/tts"
@ -28,7 +29,8 @@ func NewAgentLoop(
 	registry := NewAgentRegistry(cfg, provider)

 	// Set up shared fallback chain with rate limiting.
-	cooldown := providers.NewCooldownTracker()
+	cooldownPath := filepath.Join(filepath.Dir(filepath.Clean(registry.GetDefaultAgent().Workspace)), "cooldowns.json")
+	cooldown := providers.NewCooldownTracker(cooldownPath)
 	rl := providers.NewRateLimiterRegistry()
 	// Register rate limiters for all agents' candidates so that RPM limits
 	// configured in ModelConfig are enforced before each LLM call.
@ -56,16 +58,17 @@ func NewAgentLoop(
 	}

 	al := &AgentLoop{
-		bus:         msgBus,
-		cfg:         cfg,
-		configPath:  configPath,
-		registry:    registry,
-		state:       stateManager,
-		eventBus:    eventBus,
-		fallback:    fallbackChain,
-		cmdRegistry: commands.NewRegistry(commands.BuiltinDefinitions()),
-		steering:    newSteeringQueue(parseSteeringMode(cfg.Agents.Defaults.SteeringMode)),
-		workerSem:   make(chan struct{}, workerPoolSize),
+		bus:          msgBus,
+		cfg:          cfg,
+		configPath:   configPath,
+		cooldownPath: cooldownPath,
+		registry:     registry,
+		state:        stateManager,
+		eventBus:     eventBus,
+		fallback:     fallbackChain,
+		cmdRegistry:  commands.NewRegistry(commands.BuiltinDefinitions()),
+		steering:     newSteeringQueue(parseSteeringMode(cfg.Agents.Defaults.SteeringMode)),
+		workerSem:    make(chan struct{}, workerPoolSize),
 	}
 	al.providerFactory = providers.CreateProviderFromConfig
 	al.hooks = NewHookManager(eventBus)
--- a/pkg/agent/loop_test.go
+++ b/pkg/agent/loop_test.go
@ -2646,8 +2646,8 @@ func TestProcessMessage_FallbackUsesPerCandidateProvider(t *testing.T) {
 		Content:  "hi",
 	})

-	if resp != "fallback reply" {
-		t.Fatalf("response = %q, want %q (fallback provider)", resp, "fallback reply")
+	if !strings.HasPrefix(resp, "fallback reply") || !strings.Contains(resp, "🦞") {
+		t.Fatalf("response = %q, want it to contain %q and 🦞 (fallback provider)", resp, "fallback reply")
 	}
 	if primaryCalls == 0 {
 		t.Fatal("primary server was never called; expected at least one attempt")
@ -2723,8 +2723,8 @@ func TestProcessMessage_FallbackUsesActiveProviderWhenCandidateNotRegistered(t *
 		Content:  "hi",
 	})

-	if resp != "active provider reply" {
-		t.Fatalf("response = %q, want %q", resp, "active provider reply")
+	if !strings.HasPrefix(resp, "active provider reply") || !strings.Contains(resp, "🦞") {
+		t.Fatalf("response = %q, want it to contain %q and 🦞", resp, "active provider reply")
 	}
 	if callCount < 2 {
 		t.Fatalf("primary server calls = %d, want >= 2 (one 429 + one success via activeProvider)", callCount)
--- a/pkg/agent/loop_turn.go
+++ b/pkg/agent/loop_turn.go
@ -387,7 +387,11 @@ turnLoop:
 							fbResult.Provider, fbResult.Model, len(fbResult.Attempts)+1),
 						map[string]any{"agent_id": ts.agent.ID, "iteration": iteration},
 					)
-					ts.SetFallbackInfo(true, fbResult.Model)
+					displayName := fbResult.Model
+					if strings.HasPrefix(fbResult.IdentityKey, "model_name:") {
+						displayName = strings.TrimPrefix(fbResult.IdentityKey, "model_name:")
+					}
+					ts.SetFallbackInfo(true, displayName)
 				}
 				return fbResult.Response, nil
 			}
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@ -530,8 +530,9 @@ type VoiceConfig struct {
 // Default protocol is "openai" if no prefix is specified.
 type ModelConfig struct {
 	// Required fields
-	ModelName string `json:"model_name"` // User-facing alias for the model
-	Model     string `json:"model"`      // Protocol/model-identifier (e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4.6")
+	ModelName string `json:"model_name"`         // User-facing alias for the model
+	Model     string `json:"model"`              // Protocol/model-identifier (e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4.6")
+	Protocol  string `json:"protocol,omitempty"` // Explicit protocol (e.g., "openai", "openrouter", "anthropic")

 	// HTTP-based providers
 	APIBase   string   `json:"api_base,omitempty"`  // API endpoint URL
--- a/pkg/providers/common/common.go
+++ b/pkg/providers/common/common.go
@ -36,7 +36,7 @@ type (
 	ReasoningDetail        = protocoltypes.ReasoningDetail
 )

-const DefaultRequestTimeout = 120 * time.Second
+const DefaultRequestTimeout = 30 * time.Second

 // NewHTTPClient creates an *http.Client with an optional proxy and the default timeout.
 func NewHTTPClient(proxy string) *http.Client {
--- a/pkg/providers/cooldown.go
+++ b/pkg/providers/cooldown.go
@ -1,7 +1,10 @@
 package providers

 import (
+	"encoding/json"
 	"math"
+	"os"
+	"path/filepath"
 	"sync"
 	"time"
 )
@ -16,6 +19,7 @@ type CooldownTracker struct {
 	mu            sync.RWMutex
 	entries       map[string]*cooldownEntry
 	failureWindow time.Duration
+	storagePath   string
 	nowFunc       func() time.Time // for testing
 }

@ -29,20 +33,23 @@ type cooldownEntry struct {
 }

 // NewCooldownTracker creates a tracker with default 24h failure window.
-func NewCooldownTracker() *CooldownTracker {
-	return &CooldownTracker{
+func NewCooldownTracker(storagePath string) *CooldownTracker {
+	ct := &CooldownTracker{
 		entries:       make(map[string]*cooldownEntry),
 		failureWindow: defaultFailureWindow,
+		storagePath:   storagePath,
 		nowFunc:       time.Now,
 	}
+	if storagePath != "" {
+		ct.Load()
+	}
+	return ct
 }

 // MarkFailure records a failure for a provider and sets appropriate cooldown.
 // Resets error counts if last failure was more than failureWindow ago.
 func (ct *CooldownTracker) MarkFailure(provider string, reason FailoverReason) {
 	ct.mu.Lock()
-	defer ct.mu.Unlock()
-
 	now := ct.nowFunc()
 	entry := ct.getOrCreate(provider)

@ -53,6 +60,9 @@ func (ct *CooldownTracker) MarkFailure(provider string, reason FailoverReason) {
 	}

 	entry.ErrorCount++
+	if entry.FailureCounts == nil {
+		entry.FailureCounts = make(map[FailoverReason]int)
+	}
 	entry.FailureCounts[reason]++
 	entry.LastFailure = now

@ -63,15 +73,20 @@ func (ct *CooldownTracker) MarkFailure(provider string, reason FailoverReason) {
 	} else {
 		entry.CooldownEnd = now.Add(calculateStandardCooldown(entry.ErrorCount))
 	}
+
+	// Capture state for saving outside the lock
+	toSave := ct.copyEntriesLocked()
+	ct.mu.Unlock()
+
+	ct.persist(toSave)
 }

 // MarkSuccess resets all counters and cooldowns for a provider.
 func (ct *CooldownTracker) MarkSuccess(provider string) {
 	ct.mu.Lock()
-	defer ct.mu.Unlock()
-
 	entry := ct.entries[provider]
 	if entry == nil {
+		ct.mu.Unlock()
 		return
 	}

@ -80,6 +95,12 @@ func (ct *CooldownTracker) MarkSuccess(provider string) {
 	entry.CooldownEnd = time.Time{}
 	entry.DisabledUntil = time.Time{}
 	entry.DisabledReason = ""
+
+	// Capture state for saving outside the lock
+	toSave := ct.copyEntriesLocked()
+	ct.mu.Unlock()
+
+	ct.persist(toSave)
 }

 // IsAvailable returns true if the provider is not in cooldown or disabled.
@ -162,6 +183,93 @@ func (ct *CooldownTracker) FailureCount(provider string, reason FailoverReason)
 	return entry.FailureCounts[reason]
 }

+// Load reads cooldown state from disk.
+func (ct *CooldownTracker) Load() {
+	if ct.storagePath == "" {
+		return
+	}
+
+	data, err := os.ReadFile(ct.storagePath)
+	if err != nil {
+		return // ignore missing file
+	}
+
+	var loaded map[string]*cooldownEntry
+	if err := json.Unmarshal(data, &loaded); err != nil {
+		return
+	}
+
+	ct.mu.Lock()
+	defer ct.mu.Unlock()
+
+	now := ct.nowFunc()
+	for k, v := range loaded {
+		// Only load entries that aren't fully expired yet
+		if (!v.CooldownEnd.IsZero() && now.Before(v.CooldownEnd)) ||
+			(!v.DisabledUntil.IsZero() && now.Before(v.DisabledUntil)) ||
+			(!v.LastFailure.IsZero() && now.Sub(v.LastFailure) < ct.failureWindow) {
+			if v.FailureCounts == nil {
+				v.FailureCounts = make(map[FailoverReason]int)
+			}
+			ct.entries[k] = v
+		}
+	}
+}
+
+// Save writes cooldown state to disk.
+func (ct *CooldownTracker) Save() {
+	ct.mu.RLock()
+	toSave := ct.copyEntriesLocked()
+	ct.mu.RUnlock()
+
+	ct.persist(toSave)
+}
+
+func (ct *CooldownTracker) copyEntriesLocked() map[string]*cooldownEntry {
+	toSave := make(map[string]*cooldownEntry)
+	now := ct.nowFunc()
+	for k, v := range ct.entries {
+		if v == nil {
+			continue
+		}
+		// Only save entries that are still relevant
+		if (!v.CooldownEnd.IsZero() && now.Before(v.CooldownEnd)) ||
+			(!v.DisabledUntil.IsZero() && now.Before(v.DisabledUntil)) ||
+			(!v.LastFailure.IsZero() && now.Sub(v.LastFailure) < ct.failureWindow) {
+
+			// Deep copy the entry to avoid data races when serializing outside the lock
+			copy := *v
+			if v.FailureCounts != nil {
+				copy.FailureCounts = make(map[FailoverReason]int)
+				for r, c := range v.FailureCounts {
+					copy.FailureCounts[r] = c
+				}
+			}
+			toSave[k] = &copy
+		}
+	}
+	return toSave
+}
+
+func (ct *CooldownTracker) persist(toSave map[string]*cooldownEntry) {
+	if ct.storagePath == "" {
+		return
+	}
+
+	if len(toSave) == 0 {
+		os.Remove(ct.storagePath) // cleanup if empty
+		return
+	}
+
+	data, err := json.MarshalIndent(toSave, "", "  ")
+	if err != nil {
+		return
+	}
+
+	os.MkdirAll(filepath.Dir(ct.storagePath), 0755)
+	os.WriteFile(ct.storagePath, data, 0644)
+}
+
 func (ct *CooldownTracker) getOrCreate(provider string) *cooldownEntry {
 	entry := ct.entries[provider]
 	if entry == nil {
--- a/pkg/providers/cooldown_test.go
+++ b/pkg/providers/cooldown_test.go
@ -8,13 +8,13 @@ import (

 func newTestTracker(now time.Time) (*CooldownTracker, *time.Time) {
 	current := now
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	ct.nowFunc = func() time.Time { return current }
 	return ct, &current
 }

 func TestCooldown_InitiallyAvailable(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	if !ct.IsAvailable("openai") {
 		t.Error("new provider should be available")
 	}
@ -110,7 +110,7 @@ func TestCooldown_BillingCap(t *testing.T) {
 }

 func TestCooldown_SuccessReset(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")

 	ct.MarkFailure("openai", FailoverRateLimit)
 	ct.MarkFailure("openai", FailoverBilling)
@ -157,7 +157,7 @@ func TestCooldown_FailureWindowReset(t *testing.T) {
 }

 func TestCooldown_PerReasonTracking(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")

 	ct.MarkFailure("openai", FailoverRateLimit)
 	ct.MarkFailure("openai", FailoverRateLimit)
@ -218,7 +218,7 @@ func TestCooldown_CooldownRemaining(t *testing.T) {
 }

 func TestCooldown_SuccessOnUnknownProvider(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	// Should not panic
 	ct.MarkSuccess("nonexistent")
 	if !ct.IsAvailable("nonexistent") {
@ -227,7 +227,7 @@ func TestCooldown_SuccessOnUnknownProvider(t *testing.T) {
 }

 func TestCooldown_ConcurrentAccess(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	var wg sync.WaitGroup

 	for range 100 {
@ -251,7 +251,7 @@ func TestCooldown_ConcurrentAccess(t *testing.T) {
 }

 func TestCooldown_MultipleProviders(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")

 	ct.MarkFailure("openai", FailoverRateLimit)
 	ct.MarkFailure("anthropic", FailoverBilling)
--- a/pkg/providers/factory_provider.go
+++ b/pkg/providers/factory_provider.go
@ -109,6 +109,9 @@ func ResolveAPIBase(cfg *config.ModelConfig) string {
 		return strings.TrimRight(apiBase, "/")
 	}
 	protocol, _ := ExtractProtocol(cfg.Model)
+	if cfg.Protocol != "" {
+		protocol = cfg.Protocol
+	}
 	return strings.TrimRight(getDefaultAPIBase(protocol), "/")
 }

@ -128,6 +131,10 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err
 	}

 	protocol, modelID := ExtractProtocol(cfg.Model)
+	if cfg.Protocol != "" {
+		protocol = cfg.Protocol
+		modelID = cfg.Model
+	}

 	userAgent := cfg.UserAgent
 	if userAgent == "" {
@ -224,13 +231,13 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err
 		"qwen-us", "dashscope-us", "mistral", "avian", "longcat", "modelscope", "novita",
 		"coding-plan", "alibaba-coding", "qwen-coding", "mimo":
 		// All other OpenAI-compatible HTTP providers
-		if cfg.APIKey() == "" && cfg.APIBase == "" && !isEmptyAPIKeyAllowed(protocol) {
-			return nil, "", fmt.Errorf("api_key or api_base is required for HTTP-based protocol %q", protocol)
-		}
 		apiBase := cfg.APIBase
 		if apiBase == "" {
 			apiBase = getDefaultAPIBase(protocol)
 		}
+		if cfg.APIKey() == "" && apiBase == "" && !isEmptyAPIKeyAllowed(protocol) {
+			return nil, "", fmt.Errorf("api_key or api_base is required for HTTP-based protocol %q", protocol)
+		}
 		return NewHTTPProviderWithMaxTokensFieldAndRequestTimeout(
 			cfg.APIKey(),
 			apiBase,
--- a/pkg/providers/fallback.go
+++ b/pkg/providers/fallback.go
@ -32,10 +32,11 @@ func (c FallbackCandidate) StableKey() string {

 // FallbackResult contains the successful response and metadata about all attempts.
 type FallbackResult struct {
-	Response *LLMResponse
-	Provider string
-	Model    string
-	Attempts []FallbackAttempt
+	Response    *LLMResponse
+	Provider    string
+	Model       string
+	IdentityKey string
+	Attempts    []FallbackAttempt
 }

 // FallbackAttempt records one attempt in the fallback chain.
@ -187,6 +188,7 @@ func (fc *FallbackChain) Execute(
 			result.Response = resp
 			result.Provider = candidate.Provider
 			result.Model = candidate.Model
+			result.IdentityKey = candidate.IdentityKey
 			return result, nil
 		}

@ -305,6 +307,7 @@ func (fc *FallbackChain) ExecuteImage(
 			result.Response = resp
 			result.Provider = candidate.Provider
 			result.Model = candidate.Model
+			result.IdentityKey = candidate.IdentityKey
 			return result, nil
 		}

--- a/pkg/providers/fallback_multikey_test.go
+++ b/pkg/providers/fallback_multikey_test.go
@ -24,7 +24,7 @@ func TestMultiKeyFailover(t *testing.T) {
 	}

 	// Create fallback chain
-	cooldown := NewCooldownTracker()
+	cooldown := NewCooldownTracker("")
 	chain := NewFallbackChain(cooldown, nil)

 	// Mock run function: first call fails with 429, second succeeds
@ -81,7 +81,7 @@ func TestMultiKeyFailoverAllFail(t *testing.T) {

 	candidates := ResolveCandidates(cfg, "zhipu")

-	cooldown := NewCooldownTracker()
+	cooldown := NewCooldownTracker("")
 	chain := NewFallbackChain(cooldown, nil)

 	// Mock run function: all calls fail with rate limit
@ -126,7 +126,7 @@ func TestMultiKeyFailoverCooldown(t *testing.T) {

 	candidates := ResolveCandidates(cfg, "zhipu")

-	cooldown := NewCooldownTracker()
+	cooldown := NewCooldownTracker("")
 	chain := NewFallbackChain(cooldown, nil)

 	// Put the first model in cooldown (using ModelKey now, not just provider)
@ -182,7 +182,7 @@ func TestMultiKeyFailoverWithFormatError(t *testing.T) {

 	candidates := ResolveCandidates(cfg, "zhipu")

-	cooldown := NewCooldownTracker()
+	cooldown := NewCooldownTracker("")
 	chain := NewFallbackChain(cooldown, nil)

 	// Mock run function: first call fails with format error (bad request)
@ -262,7 +262,7 @@ func TestMultiKeyWithModelFallback(t *testing.T) {
 		)
 	}

-	cooldown := NewCooldownTracker()
+	cooldown := NewCooldownTracker("")
 	chain := NewFallbackChain(cooldown, nil)

 	// Mock run function: first two fail, third succeeds (model fallback)
@ -336,7 +336,7 @@ func TestMultiKeyFailoverMixedErrors(t *testing.T) {

 	candidates := ResolveCandidates(cfg, "zhipu")

-	cooldown := NewCooldownTracker()
+	cooldown := NewCooldownTracker("")
 	chain := NewFallbackChain(cooldown, nil)

 	// Mock run function: different errors for each key
--- a/pkg/providers/fallback_test.go
+++ b/pkg/providers/fallback_test.go
@ -18,7 +18,7 @@ func successRun(content string) func(ctx context.Context, provider, model string
 }

 func TestFallback_SingleCandidate_Success(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	fc := NewFallbackChain(ct, nil)

 	candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4")}
@ -35,7 +35,7 @@ func TestFallback_SingleCandidate_Success(t *testing.T) {
 }

 func TestFallback_SecondCandidateSuccess(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	fc := NewFallbackChain(ct, nil)

 	candidates := []FallbackCandidate{
@ -68,7 +68,7 @@ func TestFallback_SecondCandidateSuccess(t *testing.T) {
 }

 func TestFallback_AllFail(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	fc := NewFallbackChain(ct, nil)

 	candidates := []FallbackCandidate{
@ -95,7 +95,7 @@ func TestFallback_AllFail(t *testing.T) {
 }

 func TestFallback_ContextCanceled(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	fc := NewFallbackChain(ct, nil)

 	ctx, cancel := context.WithCancel(context.Background())
@ -122,7 +122,7 @@ func TestFallback_ContextCanceled(t *testing.T) {
 }

 func TestFallback_NonRetriableError(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	fc := NewFallbackChain(ct, nil)

 	candidates := []FallbackCandidate{
@ -192,7 +192,7 @@ func TestFallback_CooldownSkip(t *testing.T) {
 }

 func TestFallback_AllInCooldown(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	fc := NewFallbackChain(ct, nil)

 	// Put all models in cooldown (using ModelKey now)
@ -220,7 +220,7 @@ func TestFallback_AllInCooldown(t *testing.T) {
 }

 func TestFallback_NoCandidates(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	fc := NewFallbackChain(ct, nil)

 	_, err := fc.Execute(context.Background(), nil, successRun("ok"))
@ -231,7 +231,7 @@ func TestFallback_NoCandidates(t *testing.T) {

 func TestFallback_EmptyFallbacks(t *testing.T) {
 	// Single primary, no fallbacks: should work like direct call
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	fc := NewFallbackChain(ct, nil)

 	candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4")}
@ -245,7 +245,7 @@ func TestFallback_EmptyFallbacks(t *testing.T) {
 }

 func TestFallback_UnclassifiedError(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	fc := NewFallbackChain(ct, nil)

 	candidates := []FallbackCandidate{
@ -278,7 +278,7 @@ func assertFallbackErrorFallsBack(
 ) {
 	t.Helper()

-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	fc := NewFallbackChain(ct, nil)

 	candidates := []FallbackCandidate{
@ -338,7 +338,7 @@ func TestFallback_TimeoutErrorFallsBack(t *testing.T) {
 }

 func TestFallback_SuccessResetsCooldown(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	fc := NewFallbackChain(ct, nil)

 	candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4")}
@ -375,7 +375,7 @@ func assertLocalRateLimitSkipsToHealthyFallback(
 ) {
 	t.Helper()

-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	rl := NewRateLimiterRegistry()
 	rl.Register(primaryKey, 1)
 	if err := rl.Wait(context.Background(), primaryKey); err != nil {
@ -432,7 +432,7 @@ func TestFallback_LocalRateLimitSkipsToHealthyFallback(t *testing.T) {
 // --- Image Fallback Tests ---

 func TestImageFallback_Success(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	fc := NewFallbackChain(ct, nil)

 	candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4o")}
@ -446,7 +446,7 @@ func TestImageFallback_Success(t *testing.T) {
 }

 func TestImageFallback_DimensionError(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	fc := NewFallbackChain(ct, nil)

 	candidates := []FallbackCandidate{
@ -470,7 +470,7 @@ func TestImageFallback_DimensionError(t *testing.T) {
 }

 func TestImageFallback_SizeError(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	fc := NewFallbackChain(ct, nil)

 	candidates := []FallbackCandidate{
@ -494,7 +494,7 @@ func TestImageFallback_SizeError(t *testing.T) {
 }

 func TestImageFallback_RetryOnOtherErrors(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	fc := NewFallbackChain(ct, nil)

 	candidates := []FallbackCandidate{
@ -540,7 +540,7 @@ func TestImageFallback_LocalRateLimitSkipsToHealthyFallback(t *testing.T) {
 }

 func TestImageFallback_NoCandidates(t *testing.T) {
-	ct := NewCooldownTracker()
+	ct := NewCooldownTracker("")
 	fc := NewFallbackChain(ct, nil)

 	_, err := fc.ExecuteImage(context.Background(), nil, successRun("ok"))
--- a/pkg/seahorse/.omc/state/last-tool-error.json
+++ b/pkg/seahorse/.omc/state/last-tool-error.json
@ -0,0 +1,7 @@
+{
+  "tool_name": "Bash",
+  "tool_input_preview": "{\"command\":\"cd /home/yliu/repos/picoclaw && make lint 2>&1\",\"timeout\":120000}",
+  "error": "Exit code 2\npkg/agent/context_seahorse_test.go:1027:1: File is not properly formatted (gci)\n\t\t\tEarliestAt:    &now,\n^\n1 issues:\n* gci: 1\nmake: *** [Makefile:264: lint] Error 1",
+  "timestamp": "2026-04-04T02:38:32.067Z",
+  "retry_count": 6
+}
--- a/pkg/tools/freeride.go
+++ b/pkg/tools/freeride.go
@ -218,7 +218,8 @@ func (t *FreeRideTool) handleAuto(ctx context.Context, limit int) *ToolResult {
 		if !modelExists(cfgObj, modelName) {
 			mc := &config.ModelConfig{
 				ModelName: modelName,
-				Model:     "openrouter/" + m.ID,
+				Model:     m.ID,
+				Protocol:  "openrouter",
 				Enabled:   true,
 			}
 			mc.SetAPIKey("env://OPENROUTER_API_KEY")
--- a/scratch/check_paths.go
+++ b/scratch/check_paths.go
@ -0,0 +1,28 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/sipeed/picoclaw/pkg/agent"
+	"github.com/sipeed/picoclaw/pkg/config"
+)
+
+func main() {
+	cfg, err := config.LoadConfig(os.ExpandEnv("$HOME/.picoclaw/config.json"))
+	if err != nil {
+		fmt.Printf("Error: %v\n", err)
+		return
+	}
+	registry := agent.NewAgentRegistry(cfg, nil)
+	defaultAgent := registry.GetDefaultAgent()
+	if defaultAgent == nil {
+		fmt.Println("No default agent")
+		return
+	}
+
+	cooldownPath := filepath.Join(filepath.Dir(filepath.Clean(defaultAgent.Workspace)), "cooldowns.json")
+	fmt.Printf("Workspace: %s\n", defaultAgent.Workspace)
+	fmt.Printf("Cooldown Path: %s\n", cooldownPath)
+}