Merge FreeRide feature and doc updates into security_shield_v2

This commit is contained in:
stevef 2026-04-19 22:19:07 +02:00
commit 383132dbe6
23 changed files with 401 additions and 91 deletions

View file

@ -96,8 +96,8 @@
👁️ **Vision pipeline**: Send images and files directly to the Agent — automatic base64 encoding for multimodal LLMs. 👁️ **Vision pipeline**: Send images and files directly to the Agent — automatic base64 encoding for multimodal LLMs.
🧠 **Smart routing**: Rule-based model routing — simple queries go to lightweight models, saving API costs. 🧠 **Smart routing**: Rule-based model routing — simple queries go to lightweight models, saving API costs.
🧬 **FreeRide**: Intelligent model rotation using OpenRouter's free pool — never pay for basic LLM traffic again. [Learn more](docs/guides/freeride.md). 🧬 **FreeRide**: Intelligent model rotation using OpenRouter's free pool — never pay for basic LLM traffic again. [Learn more](docs/guides/freeride.md).
❄️ **Persistent Cooldowns**: Remembers rate-limited models across restarts via `cooldowns.json`, ensuring instant failover and zero "cooldown amnesia".
_*Recent builds may use 10-20MB due to rapid PR merges. Resource optimization is planned. Boot speed comparison based on 0.8GHz single-core benchmarks (see table below)._ _*Recent builds may use 10-20MB due to rapid PR merges. Resource optimization is planned. Boot speed comparison based on 0.8GHz single-core benchmarks (see table below)._
@ -624,7 +624,7 @@ For detailed guides beyond this README:
| [Scheduled Tasks and Cron Jobs](docs/reference/cron.md) | Cron schedule types, deliver modes, command gates, job storage | | [Scheduled Tasks and Cron Jobs](docs/reference/cron.md) | Cron schedule types, deliver modes, command gates, job storage |
| [Providers & Models](docs/guides/providers.md) | 30+ LLM providers, model routing, model_list configuration | | [Providers & Models](docs/guides/providers.md) | 30+ LLM providers, model routing, model_list configuration |
| [Spawn & Async Tasks](docs/guides/spawn-tasks.md) | Quick tasks, long tasks with spawn, async sub-agent orchestration | | [Spawn & Async Tasks](docs/guides/spawn-tasks.md) | Quick tasks, long tasks with spawn, async sub-agent orchestration |
| [FreeRide](docs/guides/freeride.md) | Dynamic free model rotation and K3s secret management | | [FreeRide](docs/freeride.md) | Dynamic free model rotation and K3s secret management |
| [Hooks](docs/architecture/hooks/README.md) | Event-driven hook system: observers, interceptors, approval hooks | | [Hooks](docs/architecture/hooks/README.md) | Event-driven hook system: observers, interceptors, approval hooks |
| [Steering](docs/architecture/steering.md) | Inject messages into a running agent loop between tool calls | | [Steering](docs/architecture/steering.md) | Inject messages into a running agent loop between tool calls |
| [SubTurn](docs/architecture/subturn.md) | Subagent coordination, concurrency control, lifecycle | | [SubTurn](docs/architecture/subturn.md) | Subagent coordination, concurrency control, lifecycle |

View file

@ -0,0 +1,84 @@
package freeride
import (
"context"
"fmt"
"github.com/spf13/cobra"
"github.com/sipeed/picoclaw/cmd/picoclaw/internal"
"github.com/sipeed/picoclaw/pkg/tools"
)
// NewFreerideCommand returns a new cobra.Command for managing OpenRouter free models.
func NewFreerideCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "freeride",
Short: "Manage OpenRouter free models and fallbacks",
Long: "FreeRide automatically discovers and configures OpenRouter's best free models as fallbacks for your PicoClaw agent.",
RunE: func(cmd *cobra.Command, args []string) error {
return cmd.Help()
},
}
cmd.AddCommand(
newListCommand(),
newAutoCommand(),
newStatusCommand(),
)
return cmd
}
func newListCommand() *cobra.Command {
var limit int
cmd := &cobra.Command{
Use: "list",
Short: "List available free models from OpenRouter",
RunE: func(cmd *cobra.Command, args []string) error {
t := tools.NewFreeRideTool(internal.GetConfigPath(), nil)
result := t.Execute(context.Background(), map[string]any{
"command": "list",
"limit": float64(limit),
})
fmt.Println(result.ForLLM)
return nil
},
}
cmd.Flags().IntVarP(&limit, "limit", "l", 10, "Number of models to list")
return cmd
}
func newAutoCommand() *cobra.Command {
var limit int
cmd := &cobra.Command{
Use: "auto",
Short: "Automatically configure best free models as fallbacks",
RunE: func(cmd *cobra.Command, args []string) error {
t := tools.NewFreeRideTool(internal.GetConfigPath(), nil)
result := t.Execute(context.Background(), map[string]any{
"command": "auto",
"limit": float64(limit),
})
fmt.Println(result.ForLLM)
return nil
},
}
cmd.Flags().IntVarP(&limit, "limit", "l", 5, "Number of fallbacks to configure")
return cmd
}
func newStatusCommand() *cobra.Command {
return &cobra.Command{
Use: "status",
Short: "Check current FreeRide configuration",
RunE: func(cmd *cobra.Command, args []string) error {
t := tools.NewFreeRideTool(internal.GetConfigPath(), nil)
result := t.Execute(context.Background(), map[string]any{
"command": "status",
})
fmt.Println(result.ForLLM)
return nil
},
}
}

View file

@ -18,6 +18,7 @@ import (
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/auth" "github.com/sipeed/picoclaw/cmd/picoclaw/internal/auth"
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/cliui" "github.com/sipeed/picoclaw/cmd/picoclaw/internal/cliui"
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/cron" "github.com/sipeed/picoclaw/cmd/picoclaw/internal/cron"
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/freeride"
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/gateway" "github.com/sipeed/picoclaw/cmd/picoclaw/internal/gateway"
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/migrate" "github.com/sipeed/picoclaw/cmd/picoclaw/internal/migrate"
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/model" "github.com/sipeed/picoclaw/cmd/picoclaw/internal/model"
@ -84,6 +85,7 @@ picoclaw --no-color status`,
onboard.NewOnboardCommand(), onboard.NewOnboardCommand(),
agent.NewAgentCommand(), agent.NewAgentCommand(),
auth.NewAuthCommand(), auth.NewAuthCommand(),
freeride.NewFreerideCommand(),
gateway.NewGatewayCommand(), gateway.NewGatewayCommand(),
status.NewStatusCommand(), status.NewStatusCommand(),
cron.NewCronCommand(), cron.NewCronCommand(),

View file

@ -40,6 +40,7 @@ func TestNewPicoclawCommand(t *testing.T) {
"agent", "agent",
"auth", "auth",
"cron", "cron",
"freeride",
"gateway", "gateway",
"migrate", "migrate",
"model", "model",

View file

@ -12,23 +12,16 @@ FreeRide is a dynamic model rotation and failover system for PicoClaw that lever
## Configuration ## Configuration
FreeRide is implemented as a native PicoClaw tool. FreeRide is implemented as a native PicoClaw tool. For production environments (especially in the **main branch**), ensure you follow the [Security Configuration](../security/security_configuration.md) to manage your API keys safely.
### 1. Enable the Tool ### 1. Enable the Tool
Ensure the `freeride` tool is enabled and whitelisted in your `config.json`: Ensure the `skills` tool is enabled in your `config.json` (FreeRide is bundled with the skills system):
```json ```json
{ {
"tools": { "tools": {
"whitelist": ["freeride", ...], "skills": {
"whitelist_enabled": true, "enabled": true
"security_policy": {
"enabled": true,
"config": {
"allowed_tools": {
"freeride": true
}
}
} }
} }
} }
@ -106,11 +99,20 @@ env:
key: openrouter-api-key key: openrouter-api-key
``` ```
## Cooldown Persistence & Timing ❄️
To prevent the agent from "hanging" or retrying known-failed models, PicoClaw uses a two-pronged approach:
### 1. Zero-Amnesia Persistence
Model failures (e.g., 429 Rate Limits) are saved to `~/.picoclaw/cooldowns.json`. This ensures that if you restart the agent, it **remembers** which models were saturated and skips them instantly. You no longer have to wait through a series of timeouts every time you restart.
### 2. Aggressive 30s Timeout
The default request timeout for LLM calls is **30 seconds**. If a free model is stalled or unresponsive, the agent will move to the next fallback in your pool much faster than the standard HTTP default.
## Troubleshooting ## Troubleshooting
- **404 Errors**: Ensure the model is still available on OpenRouter using `freeride list`. If it's gone, run `freeride auto` to refresh your fallback pool. - **404 Errors**: Ensure the model is still available on OpenRouter using `freeride list`. If it's gone, run `freeride auto` to refresh your fallback pool.
- **429 Rate Limiting**: This is common with free models. PicoClaw will automatically try the next model in your `model_fallbacks` list. - **429 Rate Limiting**: This is common with free models. PicoClaw will automatically try the next model in your `model_fallbacks` list and persist the cooldown to `cooldowns.json`.
- **Security Blocks**: Ensure `freeride` is added to your `security_policy` allowed tools map.
--- ---

View file

@ -25,13 +25,20 @@ AgentLoop.callLLM()
The rate limiter runs **after** the cooldown check and **before** the provider call, so: The rate limiter runs **after** the cooldown check and **before** the provider call, so:
- Candidates already in cooldown are skipped entirely (no token consumed) - Candidates already in cooldown are skipped entirely (no token consumed)
- Candidates that are available get throttled to the configured RPM - Candidates that are available get throttled to the configured RPM
The same check applies in `ExecuteImage`. ## Cooldown Persistence ❄️
While the Rate Limiter is **proactive**, the Cooldown Tracker is **reactive** (handled *after* a 429 is actually received).
To ensure stability across restarts, the Cooldown Tracker persists its state to disk:
- **Location**: `~/.picoclaw/cooldowns.json` (or sibling to your workspace)
- **Behavior**: If the agent is restarted, it loads the failure history and continues to enforce cooldowns. This prevents "initialization hangs" where a new agent process tries a long list of models that are already known to be rate-limited.
- **Timing**: Default LLM request timeout is now **30 seconds** to ensure faster failover.
### Thread safety ### Thread safety
`RateLimiterRegistry` is safe for concurrent use. The per-limiter token bucket uses a fine-grained mutex so concurrent goroutines each acquire their own token independently. `RateLimiterRegistry` is safe for concurrent use. The per-limiter token bucket uses a fine-grained mutex so concurrent goroutines each acquire their own token independently.
## Configuration ## Configuration
Set `rpm` on any model in `model_list`: Set `rpm` on any model in `model_list`:

View file

@ -541,6 +541,19 @@ go test ./pkg/config -run TestSecurityConfig
- Remember: Models, Brave, Tavily, Perplexity MUST use `api_keys` (array format) - Remember: Models, Brave, Tavily, Perplexity MUST use `api_keys` (array format)
- GLMSearch and BaiduSearch MUST use `api_key` (single string format) - GLMSearch and BaiduSearch MUST use `api_key` (single string format)
## FreeRide & Dynamic Failover Security
When using the **FreeRide** dynamic model failover system, PicoClaw dynamically adds models to your `model_list`. To ensure these models stay secure:
1. **Protocol-Level Security**: You can define a single entry in `.security.yml` that matches a specific provider/protocol. For example, to provide an identity for ALL OpenRouter models added by FreeRide:
```yaml
model_list:
openrouter:
api_keys: ["sk-or-v1-your-global-key"]
```
2. **Environment Variable Fallback**: FreeRide is configured to look for `OPENROUTER_API_KEY` in the environment. In production (`main` branch), it is recommended to use K3s Secrets or `SecureString` to inject this.
3. **Sensitive Filtering**: All models added by FreeRide are subject to the same `sensitive_data_filtering.md` rules as your primary models.
### Load Balancing/Failover Issues ### Load Balancing/Failover Issues
- Verify all API keys in the `api_keys` array are valid - Verify all API keys in the `api_keys` array are valid

View file

@ -0,0 +1,37 @@
{
"agents": {
"defaults": {
"model_name": "google/gemma-3-27b-it:free",
"model_fallbacks": [
"google/gemma-3-27b-it:free",
"nvidia/nemotron-4-340b-instruct:free",
"qwen/qwen-2.5-72b-instruct:free",
"mistralai/mistral-small-24b-it-v1:free"
],
"max_tokens": 4096,
"max_tool_iterations": 10
}
},
"model_list": [
{
"model_name": "google/gemma-3-27b-it:free",
"model": "google/gemma-3-27b-it:free",
"protocol": "openrouter"
},
{
"model_name": "nvidia/nemotron-4-340b-instruct:free",
"model": "nvidia/nemotron-4-340b-instruct:free",
"protocol": "openrouter"
},
{
"model_name": "qwen/qwen-2.5-72b-instruct:free",
"model": "qwen/qwen-2.5-72b-instruct:free",
"protocol": "openrouter"
},
{
"model_name": "mistralai/mistral-small-24b-it-v1:free",
"model": "mistralai/mistral-small-24b-it-v1:free",
"protocol": "openrouter"
}
]
}

View file

@ -64,6 +64,7 @@ type AgentLoop struct {
turnSeq atomic.Uint64 turnSeq atomic.Uint64
activeRequests sync.WaitGroup activeRequests sync.WaitGroup
configPath string configPath string
cooldownPath string
reloadFunc func() error reloadFunc func() error
@ -386,7 +387,7 @@ func (al *AgentLoop) ReloadProviderAndConfig(
newRL.RegisterCandidates(agent.LightCandidates) newRL.RegisterCandidates(agent.LightCandidates)
} }
} }
al.fallback = providers.NewFallbackChain(providers.NewCooldownTracker(), newRL) al.fallback = providers.NewFallbackChain(providers.NewCooldownTracker(al.cooldownPath), newRL)
al.mu.Unlock() al.mu.Unlock()
@ -521,16 +522,17 @@ func (al *AgentLoop) runAgentLoop(
} }
} }
if opts.SendResponse && result.finalContent != "" { finalContent := result.finalContent
if usedFallback, fallbackModel := ts.GetFallbackInfo(); usedFallback {
finalContent += fmt.Sprintf("\n\n🦞 _(FreeRide: %s)_", fallbackModel)
}
if opts.SendResponse && finalContent != "" {
agentID, sessionKey, scope := outboundTurnMetadata( agentID, sessionKey, scope := outboundTurnMetadata(
agent.ID, agent.ID,
opts.Dispatch.SessionKey, opts.Dispatch.SessionKey,
opts.Dispatch.SessionScope, opts.Dispatch.SessionScope,
) )
finalContent := result.finalContent
if usedFallback, fallbackModel := ts.GetFallbackInfo(); usedFallback {
finalContent += fmt.Sprintf("\n\n🦞 _(FreeRide: %s)_", fallbackModel)
}
al.bus.PublishOutbound(ctx, bus.OutboundMessage{ al.bus.PublishOutbound(ctx, bus.OutboundMessage{
Context: outboundContextFromInbound( Context: outboundContextFromInbound(
opts.Dispatch.InboundContext, opts.Dispatch.InboundContext,
@ -545,7 +547,7 @@ func (al *AgentLoop) runAgentLoop(
}) })
} }
return result.finalContent, nil return finalContent, nil
} }
// selectCandidates returns the model candidates and resolved model name to use // selectCandidates returns the model candidates and resolved model name to use

View file

@ -5,6 +5,7 @@ package agent
import ( import (
"context" "context"
"fmt" "fmt"
"path/filepath"
"time" "time"
"github.com/sipeed/picoclaw/pkg/audio/tts" "github.com/sipeed/picoclaw/pkg/audio/tts"
@ -28,7 +29,8 @@ func NewAgentLoop(
registry := NewAgentRegistry(cfg, provider) registry := NewAgentRegistry(cfg, provider)
// Set up shared fallback chain with rate limiting. // Set up shared fallback chain with rate limiting.
cooldown := providers.NewCooldownTracker() cooldownPath := filepath.Join(filepath.Dir(filepath.Clean(registry.GetDefaultAgent().Workspace)), "cooldowns.json")
cooldown := providers.NewCooldownTracker(cooldownPath)
rl := providers.NewRateLimiterRegistry() rl := providers.NewRateLimiterRegistry()
// Register rate limiters for all agents' candidates so that RPM limits // Register rate limiters for all agents' candidates so that RPM limits
// configured in ModelConfig are enforced before each LLM call. // configured in ModelConfig are enforced before each LLM call.
@ -56,16 +58,17 @@ func NewAgentLoop(
} }
al := &AgentLoop{ al := &AgentLoop{
bus: msgBus, bus: msgBus,
cfg: cfg, cfg: cfg,
configPath: configPath, configPath: configPath,
registry: registry, cooldownPath: cooldownPath,
state: stateManager, registry: registry,
eventBus: eventBus, state: stateManager,
fallback: fallbackChain, eventBus: eventBus,
cmdRegistry: commands.NewRegistry(commands.BuiltinDefinitions()), fallback: fallbackChain,
steering: newSteeringQueue(parseSteeringMode(cfg.Agents.Defaults.SteeringMode)), cmdRegistry: commands.NewRegistry(commands.BuiltinDefinitions()),
workerSem: make(chan struct{}, workerPoolSize), steering: newSteeringQueue(parseSteeringMode(cfg.Agents.Defaults.SteeringMode)),
workerSem: make(chan struct{}, workerPoolSize),
} }
al.providerFactory = providers.CreateProviderFromConfig al.providerFactory = providers.CreateProviderFromConfig
al.hooks = NewHookManager(eventBus) al.hooks = NewHookManager(eventBus)

View file

@ -2646,8 +2646,8 @@ func TestProcessMessage_FallbackUsesPerCandidateProvider(t *testing.T) {
Content: "hi", Content: "hi",
}) })
if resp != "fallback reply" { if !strings.HasPrefix(resp, "fallback reply") || !strings.Contains(resp, "🦞") {
t.Fatalf("response = %q, want %q (fallback provider)", resp, "fallback reply") t.Fatalf("response = %q, want it to contain %q and 🦞 (fallback provider)", resp, "fallback reply")
} }
if primaryCalls == 0 { if primaryCalls == 0 {
t.Fatal("primary server was never called; expected at least one attempt") t.Fatal("primary server was never called; expected at least one attempt")
@ -2723,8 +2723,8 @@ func TestProcessMessage_FallbackUsesActiveProviderWhenCandidateNotRegistered(t *
Content: "hi", Content: "hi",
}) })
if resp != "active provider reply" { if !strings.HasPrefix(resp, "active provider reply") || !strings.Contains(resp, "🦞") {
t.Fatalf("response = %q, want %q", resp, "active provider reply") t.Fatalf("response = %q, want it to contain %q and 🦞", resp, "active provider reply")
} }
if callCount < 2 { if callCount < 2 {
t.Fatalf("primary server calls = %d, want >= 2 (one 429 + one success via activeProvider)", callCount) t.Fatalf("primary server calls = %d, want >= 2 (one 429 + one success via activeProvider)", callCount)

View file

@ -387,7 +387,11 @@ turnLoop:
fbResult.Provider, fbResult.Model, len(fbResult.Attempts)+1), fbResult.Provider, fbResult.Model, len(fbResult.Attempts)+1),
map[string]any{"agent_id": ts.agent.ID, "iteration": iteration}, map[string]any{"agent_id": ts.agent.ID, "iteration": iteration},
) )
ts.SetFallbackInfo(true, fbResult.Model) displayName := fbResult.Model
if strings.HasPrefix(fbResult.IdentityKey, "model_name:") {
displayName = strings.TrimPrefix(fbResult.IdentityKey, "model_name:")
}
ts.SetFallbackInfo(true, displayName)
} }
return fbResult.Response, nil return fbResult.Response, nil
} }

View file

@ -530,8 +530,9 @@ type VoiceConfig struct {
// Default protocol is "openai" if no prefix is specified. // Default protocol is "openai" if no prefix is specified.
type ModelConfig struct { type ModelConfig struct {
// Required fields // Required fields
ModelName string `json:"model_name"` // User-facing alias for the model ModelName string `json:"model_name"` // User-facing alias for the model
Model string `json:"model"` // Protocol/model-identifier (e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4.6") Model string `json:"model"` // Protocol/model-identifier (e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4.6")
Protocol string `json:"protocol,omitempty"` // Explicit protocol (e.g., "openai", "openrouter", "anthropic")
// HTTP-based providers // HTTP-based providers
APIBase string `json:"api_base,omitempty"` // API endpoint URL APIBase string `json:"api_base,omitempty"` // API endpoint URL

View file

@ -36,7 +36,7 @@ type (
ReasoningDetail = protocoltypes.ReasoningDetail ReasoningDetail = protocoltypes.ReasoningDetail
) )
const DefaultRequestTimeout = 120 * time.Second const DefaultRequestTimeout = 30 * time.Second
// NewHTTPClient creates an *http.Client with an optional proxy and the default timeout. // NewHTTPClient creates an *http.Client with an optional proxy and the default timeout.
func NewHTTPClient(proxy string) *http.Client { func NewHTTPClient(proxy string) *http.Client {

View file

@ -1,7 +1,10 @@
package providers package providers
import ( import (
"encoding/json"
"math" "math"
"os"
"path/filepath"
"sync" "sync"
"time" "time"
) )
@ -16,6 +19,7 @@ type CooldownTracker struct {
mu sync.RWMutex mu sync.RWMutex
entries map[string]*cooldownEntry entries map[string]*cooldownEntry
failureWindow time.Duration failureWindow time.Duration
storagePath string
nowFunc func() time.Time // for testing nowFunc func() time.Time // for testing
} }
@ -29,20 +33,23 @@ type cooldownEntry struct {
} }
// NewCooldownTracker creates a tracker with default 24h failure window. // NewCooldownTracker creates a tracker with default 24h failure window.
func NewCooldownTracker() *CooldownTracker { func NewCooldownTracker(storagePath string) *CooldownTracker {
return &CooldownTracker{ ct := &CooldownTracker{
entries: make(map[string]*cooldownEntry), entries: make(map[string]*cooldownEntry),
failureWindow: defaultFailureWindow, failureWindow: defaultFailureWindow,
storagePath: storagePath,
nowFunc: time.Now, nowFunc: time.Now,
} }
if storagePath != "" {
ct.Load()
}
return ct
} }
// MarkFailure records a failure for a provider and sets appropriate cooldown. // MarkFailure records a failure for a provider and sets appropriate cooldown.
// Resets error counts if last failure was more than failureWindow ago. // Resets error counts if last failure was more than failureWindow ago.
func (ct *CooldownTracker) MarkFailure(provider string, reason FailoverReason) { func (ct *CooldownTracker) MarkFailure(provider string, reason FailoverReason) {
ct.mu.Lock() ct.mu.Lock()
defer ct.mu.Unlock()
now := ct.nowFunc() now := ct.nowFunc()
entry := ct.getOrCreate(provider) entry := ct.getOrCreate(provider)
@ -53,6 +60,9 @@ func (ct *CooldownTracker) MarkFailure(provider string, reason FailoverReason) {
} }
entry.ErrorCount++ entry.ErrorCount++
if entry.FailureCounts == nil {
entry.FailureCounts = make(map[FailoverReason]int)
}
entry.FailureCounts[reason]++ entry.FailureCounts[reason]++
entry.LastFailure = now entry.LastFailure = now
@ -63,15 +73,20 @@ func (ct *CooldownTracker) MarkFailure(provider string, reason FailoverReason) {
} else { } else {
entry.CooldownEnd = now.Add(calculateStandardCooldown(entry.ErrorCount)) entry.CooldownEnd = now.Add(calculateStandardCooldown(entry.ErrorCount))
} }
// Capture state for saving outside the lock
toSave := ct.copyEntriesLocked()
ct.mu.Unlock()
ct.persist(toSave)
} }
// MarkSuccess resets all counters and cooldowns for a provider. // MarkSuccess resets all counters and cooldowns for a provider.
func (ct *CooldownTracker) MarkSuccess(provider string) { func (ct *CooldownTracker) MarkSuccess(provider string) {
ct.mu.Lock() ct.mu.Lock()
defer ct.mu.Unlock()
entry := ct.entries[provider] entry := ct.entries[provider]
if entry == nil { if entry == nil {
ct.mu.Unlock()
return return
} }
@ -80,6 +95,12 @@ func (ct *CooldownTracker) MarkSuccess(provider string) {
entry.CooldownEnd = time.Time{} entry.CooldownEnd = time.Time{}
entry.DisabledUntil = time.Time{} entry.DisabledUntil = time.Time{}
entry.DisabledReason = "" entry.DisabledReason = ""
// Capture state for saving outside the lock
toSave := ct.copyEntriesLocked()
ct.mu.Unlock()
ct.persist(toSave)
} }
// IsAvailable returns true if the provider is not in cooldown or disabled. // IsAvailable returns true if the provider is not in cooldown or disabled.
@ -162,6 +183,93 @@ func (ct *CooldownTracker) FailureCount(provider string, reason FailoverReason)
return entry.FailureCounts[reason] return entry.FailureCounts[reason]
} }
// Load reads cooldown state from disk.
func (ct *CooldownTracker) Load() {
if ct.storagePath == "" {
return
}
data, err := os.ReadFile(ct.storagePath)
if err != nil {
return // ignore missing file
}
var loaded map[string]*cooldownEntry
if err := json.Unmarshal(data, &loaded); err != nil {
return
}
ct.mu.Lock()
defer ct.mu.Unlock()
now := ct.nowFunc()
for k, v := range loaded {
// Only load entries that aren't fully expired yet
if (!v.CooldownEnd.IsZero() && now.Before(v.CooldownEnd)) ||
(!v.DisabledUntil.IsZero() && now.Before(v.DisabledUntil)) ||
(!v.LastFailure.IsZero() && now.Sub(v.LastFailure) < ct.failureWindow) {
if v.FailureCounts == nil {
v.FailureCounts = make(map[FailoverReason]int)
}
ct.entries[k] = v
}
}
}
// Save writes cooldown state to disk.
func (ct *CooldownTracker) Save() {
ct.mu.RLock()
toSave := ct.copyEntriesLocked()
ct.mu.RUnlock()
ct.persist(toSave)
}
func (ct *CooldownTracker) copyEntriesLocked() map[string]*cooldownEntry {
toSave := make(map[string]*cooldownEntry)
now := ct.nowFunc()
for k, v := range ct.entries {
if v == nil {
continue
}
// Only save entries that are still relevant
if (!v.CooldownEnd.IsZero() && now.Before(v.CooldownEnd)) ||
(!v.DisabledUntil.IsZero() && now.Before(v.DisabledUntil)) ||
(!v.LastFailure.IsZero() && now.Sub(v.LastFailure) < ct.failureWindow) {
// Deep copy the entry to avoid data races when serializing outside the lock
copy := *v
if v.FailureCounts != nil {
copy.FailureCounts = make(map[FailoverReason]int)
for r, c := range v.FailureCounts {
copy.FailureCounts[r] = c
}
}
toSave[k] = &copy
}
}
return toSave
}
func (ct *CooldownTracker) persist(toSave map[string]*cooldownEntry) {
if ct.storagePath == "" {
return
}
if len(toSave) == 0 {
os.Remove(ct.storagePath) // cleanup if empty
return
}
data, err := json.MarshalIndent(toSave, "", " ")
if err != nil {
return
}
os.MkdirAll(filepath.Dir(ct.storagePath), 0755)
os.WriteFile(ct.storagePath, data, 0644)
}
func (ct *CooldownTracker) getOrCreate(provider string) *cooldownEntry { func (ct *CooldownTracker) getOrCreate(provider string) *cooldownEntry {
entry := ct.entries[provider] entry := ct.entries[provider]
if entry == nil { if entry == nil {

View file

@ -8,13 +8,13 @@ import (
func newTestTracker(now time.Time) (*CooldownTracker, *time.Time) { func newTestTracker(now time.Time) (*CooldownTracker, *time.Time) {
current := now current := now
ct := NewCooldownTracker() ct := NewCooldownTracker("")
ct.nowFunc = func() time.Time { return current } ct.nowFunc = func() time.Time { return current }
return ct, &current return ct, &current
} }
func TestCooldown_InitiallyAvailable(t *testing.T) { func TestCooldown_InitiallyAvailable(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
if !ct.IsAvailable("openai") { if !ct.IsAvailable("openai") {
t.Error("new provider should be available") t.Error("new provider should be available")
} }
@ -110,7 +110,7 @@ func TestCooldown_BillingCap(t *testing.T) {
} }
func TestCooldown_SuccessReset(t *testing.T) { func TestCooldown_SuccessReset(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
ct.MarkFailure("openai", FailoverRateLimit) ct.MarkFailure("openai", FailoverRateLimit)
ct.MarkFailure("openai", FailoverBilling) ct.MarkFailure("openai", FailoverBilling)
@ -157,7 +157,7 @@ func TestCooldown_FailureWindowReset(t *testing.T) {
} }
func TestCooldown_PerReasonTracking(t *testing.T) { func TestCooldown_PerReasonTracking(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
ct.MarkFailure("openai", FailoverRateLimit) ct.MarkFailure("openai", FailoverRateLimit)
ct.MarkFailure("openai", FailoverRateLimit) ct.MarkFailure("openai", FailoverRateLimit)
@ -218,7 +218,7 @@ func TestCooldown_CooldownRemaining(t *testing.T) {
} }
func TestCooldown_SuccessOnUnknownProvider(t *testing.T) { func TestCooldown_SuccessOnUnknownProvider(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
// Should not panic // Should not panic
ct.MarkSuccess("nonexistent") ct.MarkSuccess("nonexistent")
if !ct.IsAvailable("nonexistent") { if !ct.IsAvailable("nonexistent") {
@ -227,7 +227,7 @@ func TestCooldown_SuccessOnUnknownProvider(t *testing.T) {
} }
func TestCooldown_ConcurrentAccess(t *testing.T) { func TestCooldown_ConcurrentAccess(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
var wg sync.WaitGroup var wg sync.WaitGroup
for range 100 { for range 100 {
@ -251,7 +251,7 @@ func TestCooldown_ConcurrentAccess(t *testing.T) {
} }
func TestCooldown_MultipleProviders(t *testing.T) { func TestCooldown_MultipleProviders(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
ct.MarkFailure("openai", FailoverRateLimit) ct.MarkFailure("openai", FailoverRateLimit)
ct.MarkFailure("anthropic", FailoverBilling) ct.MarkFailure("anthropic", FailoverBilling)

View file

@ -109,6 +109,9 @@ func ResolveAPIBase(cfg *config.ModelConfig) string {
return strings.TrimRight(apiBase, "/") return strings.TrimRight(apiBase, "/")
} }
protocol, _ := ExtractProtocol(cfg.Model) protocol, _ := ExtractProtocol(cfg.Model)
if cfg.Protocol != "" {
protocol = cfg.Protocol
}
return strings.TrimRight(getDefaultAPIBase(protocol), "/") return strings.TrimRight(getDefaultAPIBase(protocol), "/")
} }
@ -128,6 +131,10 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err
} }
protocol, modelID := ExtractProtocol(cfg.Model) protocol, modelID := ExtractProtocol(cfg.Model)
if cfg.Protocol != "" {
protocol = cfg.Protocol
modelID = cfg.Model
}
userAgent := cfg.UserAgent userAgent := cfg.UserAgent
if userAgent == "" { if userAgent == "" {
@ -224,13 +231,13 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err
"qwen-us", "dashscope-us", "mistral", "avian", "longcat", "modelscope", "novita", "qwen-us", "dashscope-us", "mistral", "avian", "longcat", "modelscope", "novita",
"coding-plan", "alibaba-coding", "qwen-coding", "mimo": "coding-plan", "alibaba-coding", "qwen-coding", "mimo":
// All other OpenAI-compatible HTTP providers // All other OpenAI-compatible HTTP providers
if cfg.APIKey() == "" && cfg.APIBase == "" && !isEmptyAPIKeyAllowed(protocol) {
return nil, "", fmt.Errorf("api_key or api_base is required for HTTP-based protocol %q", protocol)
}
apiBase := cfg.APIBase apiBase := cfg.APIBase
if apiBase == "" { if apiBase == "" {
apiBase = getDefaultAPIBase(protocol) apiBase = getDefaultAPIBase(protocol)
} }
if cfg.APIKey() == "" && apiBase == "" && !isEmptyAPIKeyAllowed(protocol) {
return nil, "", fmt.Errorf("api_key or api_base is required for HTTP-based protocol %q", protocol)
}
return NewHTTPProviderWithMaxTokensFieldAndRequestTimeout( return NewHTTPProviderWithMaxTokensFieldAndRequestTimeout(
cfg.APIKey(), cfg.APIKey(),
apiBase, apiBase,

View file

@ -32,10 +32,11 @@ func (c FallbackCandidate) StableKey() string {
// FallbackResult contains the successful response and metadata about all attempts. // FallbackResult contains the successful response and metadata about all attempts.
type FallbackResult struct { type FallbackResult struct {
Response *LLMResponse Response *LLMResponse
Provider string Provider string
Model string Model string
Attempts []FallbackAttempt IdentityKey string
Attempts []FallbackAttempt
} }
// FallbackAttempt records one attempt in the fallback chain. // FallbackAttempt records one attempt in the fallback chain.
@ -187,6 +188,7 @@ func (fc *FallbackChain) Execute(
result.Response = resp result.Response = resp
result.Provider = candidate.Provider result.Provider = candidate.Provider
result.Model = candidate.Model result.Model = candidate.Model
result.IdentityKey = candidate.IdentityKey
return result, nil return result, nil
} }
@ -305,6 +307,7 @@ func (fc *FallbackChain) ExecuteImage(
result.Response = resp result.Response = resp
result.Provider = candidate.Provider result.Provider = candidate.Provider
result.Model = candidate.Model result.Model = candidate.Model
result.IdentityKey = candidate.IdentityKey
return result, nil return result, nil
} }

View file

@ -24,7 +24,7 @@ func TestMultiKeyFailover(t *testing.T) {
} }
// Create fallback chain // Create fallback chain
cooldown := NewCooldownTracker() cooldown := NewCooldownTracker("")
chain := NewFallbackChain(cooldown, nil) chain := NewFallbackChain(cooldown, nil)
// Mock run function: first call fails with 429, second succeeds // Mock run function: first call fails with 429, second succeeds
@ -81,7 +81,7 @@ func TestMultiKeyFailoverAllFail(t *testing.T) {
candidates := ResolveCandidates(cfg, "zhipu") candidates := ResolveCandidates(cfg, "zhipu")
cooldown := NewCooldownTracker() cooldown := NewCooldownTracker("")
chain := NewFallbackChain(cooldown, nil) chain := NewFallbackChain(cooldown, nil)
// Mock run function: all calls fail with rate limit // Mock run function: all calls fail with rate limit
@ -126,7 +126,7 @@ func TestMultiKeyFailoverCooldown(t *testing.T) {
candidates := ResolveCandidates(cfg, "zhipu") candidates := ResolveCandidates(cfg, "zhipu")
cooldown := NewCooldownTracker() cooldown := NewCooldownTracker("")
chain := NewFallbackChain(cooldown, nil) chain := NewFallbackChain(cooldown, nil)
// Put the first model in cooldown (using ModelKey now, not just provider) // Put the first model in cooldown (using ModelKey now, not just provider)
@ -182,7 +182,7 @@ func TestMultiKeyFailoverWithFormatError(t *testing.T) {
candidates := ResolveCandidates(cfg, "zhipu") candidates := ResolveCandidates(cfg, "zhipu")
cooldown := NewCooldownTracker() cooldown := NewCooldownTracker("")
chain := NewFallbackChain(cooldown, nil) chain := NewFallbackChain(cooldown, nil)
// Mock run function: first call fails with format error (bad request) // Mock run function: first call fails with format error (bad request)
@ -262,7 +262,7 @@ func TestMultiKeyWithModelFallback(t *testing.T) {
) )
} }
cooldown := NewCooldownTracker() cooldown := NewCooldownTracker("")
chain := NewFallbackChain(cooldown, nil) chain := NewFallbackChain(cooldown, nil)
// Mock run function: first two fail, third succeeds (model fallback) // Mock run function: first two fail, third succeeds (model fallback)
@ -336,7 +336,7 @@ func TestMultiKeyFailoverMixedErrors(t *testing.T) {
candidates := ResolveCandidates(cfg, "zhipu") candidates := ResolveCandidates(cfg, "zhipu")
cooldown := NewCooldownTracker() cooldown := NewCooldownTracker("")
chain := NewFallbackChain(cooldown, nil) chain := NewFallbackChain(cooldown, nil)
// Mock run function: different errors for each key // Mock run function: different errors for each key

View file

@ -18,7 +18,7 @@ func successRun(content string) func(ctx context.Context, provider, model string
} }
func TestFallback_SingleCandidate_Success(t *testing.T) { func TestFallback_SingleCandidate_Success(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
fc := NewFallbackChain(ct, nil) fc := NewFallbackChain(ct, nil)
candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4")} candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4")}
@ -35,7 +35,7 @@ func TestFallback_SingleCandidate_Success(t *testing.T) {
} }
func TestFallback_SecondCandidateSuccess(t *testing.T) { func TestFallback_SecondCandidateSuccess(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
fc := NewFallbackChain(ct, nil) fc := NewFallbackChain(ct, nil)
candidates := []FallbackCandidate{ candidates := []FallbackCandidate{
@ -68,7 +68,7 @@ func TestFallback_SecondCandidateSuccess(t *testing.T) {
} }
func TestFallback_AllFail(t *testing.T) { func TestFallback_AllFail(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
fc := NewFallbackChain(ct, nil) fc := NewFallbackChain(ct, nil)
candidates := []FallbackCandidate{ candidates := []FallbackCandidate{
@ -95,7 +95,7 @@ func TestFallback_AllFail(t *testing.T) {
} }
func TestFallback_ContextCanceled(t *testing.T) { func TestFallback_ContextCanceled(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
fc := NewFallbackChain(ct, nil) fc := NewFallbackChain(ct, nil)
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
@ -122,7 +122,7 @@ func TestFallback_ContextCanceled(t *testing.T) {
} }
func TestFallback_NonRetriableError(t *testing.T) { func TestFallback_NonRetriableError(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
fc := NewFallbackChain(ct, nil) fc := NewFallbackChain(ct, nil)
candidates := []FallbackCandidate{ candidates := []FallbackCandidate{
@ -192,7 +192,7 @@ func TestFallback_CooldownSkip(t *testing.T) {
} }
func TestFallback_AllInCooldown(t *testing.T) { func TestFallback_AllInCooldown(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
fc := NewFallbackChain(ct, nil) fc := NewFallbackChain(ct, nil)
// Put all models in cooldown (using ModelKey now) // Put all models in cooldown (using ModelKey now)
@ -220,7 +220,7 @@ func TestFallback_AllInCooldown(t *testing.T) {
} }
func TestFallback_NoCandidates(t *testing.T) { func TestFallback_NoCandidates(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
fc := NewFallbackChain(ct, nil) fc := NewFallbackChain(ct, nil)
_, err := fc.Execute(context.Background(), nil, successRun("ok")) _, err := fc.Execute(context.Background(), nil, successRun("ok"))
@ -231,7 +231,7 @@ func TestFallback_NoCandidates(t *testing.T) {
func TestFallback_EmptyFallbacks(t *testing.T) { func TestFallback_EmptyFallbacks(t *testing.T) {
// Single primary, no fallbacks: should work like direct call // Single primary, no fallbacks: should work like direct call
ct := NewCooldownTracker() ct := NewCooldownTracker("")
fc := NewFallbackChain(ct, nil) fc := NewFallbackChain(ct, nil)
candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4")} candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4")}
@ -245,7 +245,7 @@ func TestFallback_EmptyFallbacks(t *testing.T) {
} }
func TestFallback_UnclassifiedError(t *testing.T) { func TestFallback_UnclassifiedError(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
fc := NewFallbackChain(ct, nil) fc := NewFallbackChain(ct, nil)
candidates := []FallbackCandidate{ candidates := []FallbackCandidate{
@ -278,7 +278,7 @@ func assertFallbackErrorFallsBack(
) { ) {
t.Helper() t.Helper()
ct := NewCooldownTracker() ct := NewCooldownTracker("")
fc := NewFallbackChain(ct, nil) fc := NewFallbackChain(ct, nil)
candidates := []FallbackCandidate{ candidates := []FallbackCandidate{
@ -338,7 +338,7 @@ func TestFallback_TimeoutErrorFallsBack(t *testing.T) {
} }
func TestFallback_SuccessResetsCooldown(t *testing.T) { func TestFallback_SuccessResetsCooldown(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
fc := NewFallbackChain(ct, nil) fc := NewFallbackChain(ct, nil)
candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4")} candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4")}
@ -375,7 +375,7 @@ func assertLocalRateLimitSkipsToHealthyFallback(
) { ) {
t.Helper() t.Helper()
ct := NewCooldownTracker() ct := NewCooldownTracker("")
rl := NewRateLimiterRegistry() rl := NewRateLimiterRegistry()
rl.Register(primaryKey, 1) rl.Register(primaryKey, 1)
if err := rl.Wait(context.Background(), primaryKey); err != nil { if err := rl.Wait(context.Background(), primaryKey); err != nil {
@ -432,7 +432,7 @@ func TestFallback_LocalRateLimitSkipsToHealthyFallback(t *testing.T) {
// --- Image Fallback Tests --- // --- Image Fallback Tests ---
func TestImageFallback_Success(t *testing.T) { func TestImageFallback_Success(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
fc := NewFallbackChain(ct, nil) fc := NewFallbackChain(ct, nil)
candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4o")} candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4o")}
@ -446,7 +446,7 @@ func TestImageFallback_Success(t *testing.T) {
} }
func TestImageFallback_DimensionError(t *testing.T) { func TestImageFallback_DimensionError(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
fc := NewFallbackChain(ct, nil) fc := NewFallbackChain(ct, nil)
candidates := []FallbackCandidate{ candidates := []FallbackCandidate{
@ -470,7 +470,7 @@ func TestImageFallback_DimensionError(t *testing.T) {
} }
func TestImageFallback_SizeError(t *testing.T) { func TestImageFallback_SizeError(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
fc := NewFallbackChain(ct, nil) fc := NewFallbackChain(ct, nil)
candidates := []FallbackCandidate{ candidates := []FallbackCandidate{
@ -494,7 +494,7 @@ func TestImageFallback_SizeError(t *testing.T) {
} }
func TestImageFallback_RetryOnOtherErrors(t *testing.T) { func TestImageFallback_RetryOnOtherErrors(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
fc := NewFallbackChain(ct, nil) fc := NewFallbackChain(ct, nil)
candidates := []FallbackCandidate{ candidates := []FallbackCandidate{
@ -540,7 +540,7 @@ func TestImageFallback_LocalRateLimitSkipsToHealthyFallback(t *testing.T) {
} }
func TestImageFallback_NoCandidates(t *testing.T) { func TestImageFallback_NoCandidates(t *testing.T) {
ct := NewCooldownTracker() ct := NewCooldownTracker("")
fc := NewFallbackChain(ct, nil) fc := NewFallbackChain(ct, nil)
_, err := fc.ExecuteImage(context.Background(), nil, successRun("ok")) _, err := fc.ExecuteImage(context.Background(), nil, successRun("ok"))

View file

@ -0,0 +1,7 @@
{
"tool_name": "Bash",
"tool_input_preview": "{\"command\":\"cd /home/yliu/repos/picoclaw && make lint 2>&1\",\"timeout\":120000}",
"error": "Exit code 2\npkg/agent/context_seahorse_test.go:1027:1: File is not properly formatted (gci)\n\t\t\tEarliestAt: &now,\n^\n1 issues:\n* gci: 1\nmake: *** [Makefile:264: lint] Error 1",
"timestamp": "2026-04-04T02:38:32.067Z",
"retry_count": 6
}

View file

@ -218,7 +218,8 @@ func (t *FreeRideTool) handleAuto(ctx context.Context, limit int) *ToolResult {
if !modelExists(cfgObj, modelName) { if !modelExists(cfgObj, modelName) {
mc := &config.ModelConfig{ mc := &config.ModelConfig{
ModelName: modelName, ModelName: modelName,
Model: "openrouter/" + m.ID, Model: m.ID,
Protocol: "openrouter",
Enabled: true, Enabled: true,
} }
mc.SetAPIKey("env://OPENROUTER_API_KEY") mc.SetAPIKey("env://OPENROUTER_API_KEY")

28
scratch/check_paths.go Normal file
View file

@ -0,0 +1,28 @@
package main
import (
"fmt"
"os"
"path/filepath"
"github.com/sipeed/picoclaw/pkg/agent"
"github.com/sipeed/picoclaw/pkg/config"
)
func main() {
cfg, err := config.LoadConfig(os.ExpandEnv("$HOME/.picoclaw/config.json"))
if err != nil {
fmt.Printf("Error: %v\n", err)
return
}
registry := agent.NewAgentRegistry(cfg, nil)
defaultAgent := registry.GetDefaultAgent()
if defaultAgent == nil {
fmt.Println("No default agent")
return
}
cooldownPath := filepath.Join(filepath.Dir(filepath.Clean(defaultAgent.Workspace)), "cooldowns.json")
fmt.Printf("Workspace: %s\n", defaultAgent.Workspace)
fmt.Printf("Cooldown Path: %s\n", cooldownPath)
}