diff --git a/README.md b/README.md index 22b7b7049..f7b2806bf 100644 --- a/README.md +++ b/README.md @@ -96,8 +96,8 @@ šŸ‘ļø **Vision pipeline**: Send images and files directly to the Agent ā€” automatic base64 encoding for multimodal LLMs. šŸ§  **Smart routing**: Rule-based model routing ā€” simple queries go to lightweight models, saving API costs. - šŸ§¬ **FreeRide**: Intelligent model rotation using OpenRouter's free pool ā€” never pay for basic LLM traffic again. [Learn more](docs/guides/freeride.md). +ā„ļø **Persistent Cooldowns**: Remembers rate-limited models across restarts via `cooldowns.json`, ensuring instant failover and zero "cooldown amnesia". _*Recent builds may use 10-20MB due to rapid PR merges. Resource optimization is planned. Boot speed comparison based on 0.8GHz single-core benchmarks (see table below)._ @@ -624,7 +624,7 @@ For detailed guides beyond this README: | [Scheduled Tasks and Cron Jobs](docs/reference/cron.md) | Cron schedule types, deliver modes, command gates, job storage | | [Providers & Models](docs/guides/providers.md) | 30+ LLM providers, model routing, model_list configuration | | [Spawn & Async Tasks](docs/guides/spawn-tasks.md) | Quick tasks, long tasks with spawn, async sub-agent orchestration | -| [FreeRide](docs/guides/freeride.md) | Dynamic free model rotation and K3s secret management | +| [FreeRide](docs/freeride.md) | Dynamic free model rotation and K3s secret management | | [Hooks](docs/architecture/hooks/README.md) | Event-driven hook system: observers, interceptors, approval hooks | | [Steering](docs/architecture/steering.md) | Inject messages into a running agent loop between tool calls | | [SubTurn](docs/architecture/subturn.md) | Subagent coordination, concurrency control, lifecycle | diff --git a/cmd/picoclaw/internal/freeride/command.go b/cmd/picoclaw/internal/freeride/command.go new file mode 100644 index 000000000..2c2bf9112 --- /dev/null +++ b/cmd/picoclaw/internal/freeride/command.go @@ -0,0 +1,84 @@ +package freeride + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + + "github.com/sipeed/picoclaw/cmd/picoclaw/internal" + "github.com/sipeed/picoclaw/pkg/tools" +) + +// NewFreerideCommand returns a new cobra.Command for managing OpenRouter free models. +func NewFreerideCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "freeride", + Short: "Manage OpenRouter free models and fallbacks", + Long: "FreeRide automatically discovers and configures OpenRouter's best free models as fallbacks for your PicoClaw agent.", + RunE: func(cmd *cobra.Command, args []string) error { + return cmd.Help() + }, + } + + cmd.AddCommand( + newListCommand(), + newAutoCommand(), + newStatusCommand(), + ) + + return cmd +} + +func newListCommand() *cobra.Command { + var limit int + cmd := &cobra.Command{ + Use: "list", + Short: "List available free models from OpenRouter", + RunE: func(cmd *cobra.Command, args []string) error { + t := tools.NewFreeRideTool(internal.GetConfigPath(), nil) + result := t.Execute(context.Background(), map[string]any{ + "command": "list", + "limit": float64(limit), + }) + fmt.Println(result.ForLLM) + return nil + }, + } + cmd.Flags().IntVarP(&limit, "limit", "l", 10, "Number of models to list") + return cmd +} + +func newAutoCommand() *cobra.Command { + var limit int + cmd := &cobra.Command{ + Use: "auto", + Short: "Automatically configure best free models as fallbacks", + RunE: func(cmd *cobra.Command, args []string) error { + t := tools.NewFreeRideTool(internal.GetConfigPath(), nil) + result := t.Execute(context.Background(), map[string]any{ + "command": "auto", + "limit": float64(limit), + }) + fmt.Println(result.ForLLM) + return nil + }, + } + cmd.Flags().IntVarP(&limit, "limit", "l", 5, "Number of fallbacks to configure") + return cmd +} + +func newStatusCommand() *cobra.Command { + return &cobra.Command{ + Use: "status", + Short: "Check current FreeRide configuration", + RunE: func(cmd *cobra.Command, args []string) error { + t := tools.NewFreeRideTool(internal.GetConfigPath(), nil) + result := t.Execute(context.Background(), map[string]any{ + "command": "status", + }) + fmt.Println(result.ForLLM) + return nil + }, + } +} diff --git a/cmd/picoclaw/main.go b/cmd/picoclaw/main.go index 0867203a6..0858f2fe2 100644 --- a/cmd/picoclaw/main.go +++ b/cmd/picoclaw/main.go @@ -18,6 +18,7 @@ import ( "github.com/sipeed/picoclaw/cmd/picoclaw/internal/auth" "github.com/sipeed/picoclaw/cmd/picoclaw/internal/cliui" "github.com/sipeed/picoclaw/cmd/picoclaw/internal/cron" + "github.com/sipeed/picoclaw/cmd/picoclaw/internal/freeride" "github.com/sipeed/picoclaw/cmd/picoclaw/internal/gateway" "github.com/sipeed/picoclaw/cmd/picoclaw/internal/migrate" "github.com/sipeed/picoclaw/cmd/picoclaw/internal/model" @@ -84,6 +85,7 @@ picoclaw --no-color status`, onboard.NewOnboardCommand(), agent.NewAgentCommand(), auth.NewAuthCommand(), + freeride.NewFreerideCommand(), gateway.NewGatewayCommand(), status.NewStatusCommand(), cron.NewCronCommand(), diff --git a/cmd/picoclaw/main_test.go b/cmd/picoclaw/main_test.go index 309e60ba9..c6d53d146 100644 --- a/cmd/picoclaw/main_test.go +++ b/cmd/picoclaw/main_test.go @@ -40,6 +40,7 @@ func TestNewPicoclawCommand(t *testing.T) { "agent", "auth", "cron", + "freeride", "gateway", "migrate", "model", diff --git a/docs/guides/freeride.md b/docs/guides/freeride.md index 98b7af927..af0ce39f4 100644 --- a/docs/guides/freeride.md +++ b/docs/guides/freeride.md @@ -12,23 +12,16 @@ FreeRide is a dynamic model rotation and failover system for PicoClaw that lever ## Configuration -FreeRide is implemented as a native PicoClaw tool. +FreeRide is implemented as a native PicoClaw tool. For production environments (especially in the **main branch**), ensure you follow the [Security Configuration](../security/security_configuration.md) to manage your API keys safely. ### 1. Enable the Tool -Ensure the `freeride` tool is enabled and whitelisted in your `config.json`: - +Ensure the `skills` tool is enabled in your `config.json` (FreeRide is bundled with the skills system): + ```json { "tools": { - "whitelist": ["freeride", ...], - "whitelist_enabled": true, - "security_policy": { - "enabled": true, - "config": { - "allowed_tools": { - "freeride": true - } - } + "skills": { + "enabled": true } } } @@ -106,11 +99,20 @@ env: key: openrouter-api-key ``` +## Cooldown Persistence & Timing ā„ļø + +To prevent the agent from "hanging" or retrying known-failed models, PicoClaw uses a two-pronged approach: + +### 1. Zero-Amnesia Persistence +Model failures (e.g., 429 Rate Limits) are saved to `~/.picoclaw/cooldowns.json`. This ensures that if you restart the agent, it **remembers** which models were saturated and skips them instantly. You no longer have to wait through a series of timeouts every time you restart. + +### 2. Aggressive 30s Timeout +The default request timeout for LLM calls is **30 seconds**. If a free model is stalled or unresponsive, the agent will move to the next fallback in your pool much faster than the standard HTTP default. + ## Troubleshooting - **404 Errors**: Ensure the model is still available on OpenRouter using `freeride list`. If it's gone, run `freeride auto` to refresh your fallback pool. -- **429 Rate Limiting**: This is common with free models. PicoClaw will automatically try the next model in your `model_fallbacks` list. -- **Security Blocks**: Ensure `freeride` is added to your `security_policy` allowed tools map. +- **429 Rate Limiting**: This is common with free models. PicoClaw will automatically try the next model in your `model_fallbacks` list and persist the cooldown to `cooldowns.json`. --- diff --git a/docs/reference/rate-limiting.md b/docs/reference/rate-limiting.md index b54c757f8..5b550f823 100644 --- a/docs/reference/rate-limiting.md +++ b/docs/reference/rate-limiting.md @@ -25,13 +25,20 @@ AgentLoop.callLLM() The rate limiter runs **after** the cooldown check and **before** the provider call, so: - Candidates already in cooldown are skipped entirely (no token consumed) - Candidates that are available get throttled to the configured RPM - -The same check applies in `ExecuteImage`. - + +## Cooldown Persistence ā„ļø + +While the Rate Limiter is **proactive**, the Cooldown Tracker is **reactive** (handled *after* a 429 is actually received). + +To ensure stability across restarts, the Cooldown Tracker persists its state to disk: +- **Location**: `~/.picoclaw/cooldowns.json` (or sibling to your workspace) +- **Behavior**: If the agent is restarted, it loads the failure history and continues to enforce cooldowns. This prevents "initialization hangs" where a new agent process tries a long list of models that are already known to be rate-limited. +- **Timing**: Default LLM request timeout is now **30 seconds** to ensure faster failover. + ### Thread safety - + `RateLimiterRegistry` is safe for concurrent use. The per-limiter token bucket uses a fine-grained mutex so concurrent goroutines each acquire their own token independently. - + ## Configuration Set `rpm` on any model in `model_list`: diff --git a/docs/security/security_configuration.md b/docs/security/security_configuration.md index 065eb1e76..94b4ece3f 100644 --- a/docs/security/security_configuration.md +++ b/docs/security/security_configuration.md @@ -541,6 +541,19 @@ go test ./pkg/config -run TestSecurityConfig - Remember: Models, Brave, Tavily, Perplexity MUST use `api_keys` (array format) - GLMSearch and BaiduSearch MUST use `api_key` (single string format) +## FreeRide & Dynamic Failover Security + +When using the **FreeRide** dynamic model failover system, PicoClaw dynamically adds models to your `model_list`. To ensure these models stay secure: + +1. **Protocol-Level Security**: You can define a single entry in `.security.yml` that matches a specific provider/protocol. For example, to provide an identity for ALL OpenRouter models added by FreeRide: + ```yaml + model_list: + openrouter: + api_keys: ["sk-or-v1-your-global-key"] + ``` +2. **Environment Variable Fallback**: FreeRide is configured to look for `OPENROUTER_API_KEY` in the environment. In production (`main` branch), it is recommended to use K3s Secrets or `SecureString` to inject this. +3. **Sensitive Filtering**: All models added by FreeRide are subject to the same `sensitive_data_filtering.md` rules as your primary models. + ### Load Balancing/Failover Issues - Verify all API keys in the `api_keys` array are valid diff --git a/examples/freeride-config.json b/examples/freeride-config.json new file mode 100644 index 000000000..77bd76743 --- /dev/null +++ b/examples/freeride-config.json @@ -0,0 +1,37 @@ +{ + "agents": { + "defaults": { + "model_name": "google/gemma-3-27b-it:free", + "model_fallbacks": [ + "google/gemma-3-27b-it:free", + "nvidia/nemotron-4-340b-instruct:free", + "qwen/qwen-2.5-72b-instruct:free", + "mistralai/mistral-small-24b-it-v1:free" + ], + "max_tokens": 4096, + "max_tool_iterations": 10 + } + }, + "model_list": [ + { + "model_name": "google/gemma-3-27b-it:free", + "model": "google/gemma-3-27b-it:free", + "protocol": "openrouter" + }, + { + "model_name": "nvidia/nemotron-4-340b-instruct:free", + "model": "nvidia/nemotron-4-340b-instruct:free", + "protocol": "openrouter" + }, + { + "model_name": "qwen/qwen-2.5-72b-instruct:free", + "model": "qwen/qwen-2.5-72b-instruct:free", + "protocol": "openrouter" + }, + { + "model_name": "mistralai/mistral-small-24b-it-v1:free", + "model": "mistralai/mistral-small-24b-it-v1:free", + "protocol": "openrouter" + } + ] +} diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 10297c901..c49ef5c54 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -64,6 +64,7 @@ type AgentLoop struct { turnSeq atomic.Uint64 activeRequests sync.WaitGroup configPath string + cooldownPath string reloadFunc func() error @@ -386,7 +387,7 @@ func (al *AgentLoop) ReloadProviderAndConfig( newRL.RegisterCandidates(agent.LightCandidates) } } - al.fallback = providers.NewFallbackChain(providers.NewCooldownTracker(), newRL) + al.fallback = providers.NewFallbackChain(providers.NewCooldownTracker(al.cooldownPath), newRL) al.mu.Unlock() @@ -521,16 +522,17 @@ func (al *AgentLoop) runAgentLoop( } } - if opts.SendResponse && result.finalContent != "" { + finalContent := result.finalContent + if usedFallback, fallbackModel := ts.GetFallbackInfo(); usedFallback { + finalContent += fmt.Sprintf("\n\nšŸ¦ž _(FreeRide: %s)_", fallbackModel) + } + + if opts.SendResponse && finalContent != "" { agentID, sessionKey, scope := outboundTurnMetadata( agent.ID, opts.Dispatch.SessionKey, opts.Dispatch.SessionScope, ) - finalContent := result.finalContent - if usedFallback, fallbackModel := ts.GetFallbackInfo(); usedFallback { - finalContent += fmt.Sprintf("\n\nšŸ¦ž _(FreeRide: %s)_", fallbackModel) - } al.bus.PublishOutbound(ctx, bus.OutboundMessage{ Context: outboundContextFromInbound( opts.Dispatch.InboundContext, @@ -545,7 +547,7 @@ func (al *AgentLoop) runAgentLoop( }) } - return result.finalContent, nil + return finalContent, nil } // selectCandidates returns the model candidates and resolved model name to use diff --git a/pkg/agent/loop_init.go b/pkg/agent/loop_init.go index 234b8890e..5bee267b2 100644 --- a/pkg/agent/loop_init.go +++ b/pkg/agent/loop_init.go @@ -5,6 +5,7 @@ package agent import ( "context" "fmt" + "path/filepath" "time" "github.com/sipeed/picoclaw/pkg/audio/tts" @@ -28,7 +29,8 @@ func NewAgentLoop( registry := NewAgentRegistry(cfg, provider) // Set up shared fallback chain with rate limiting. - cooldown := providers.NewCooldownTracker() + cooldownPath := filepath.Join(filepath.Dir(filepath.Clean(registry.GetDefaultAgent().Workspace)), "cooldowns.json") + cooldown := providers.NewCooldownTracker(cooldownPath) rl := providers.NewRateLimiterRegistry() // Register rate limiters for all agents' candidates so that RPM limits // configured in ModelConfig are enforced before each LLM call. @@ -56,16 +58,17 @@ func NewAgentLoop( } al := &AgentLoop{ - bus: msgBus, - cfg: cfg, - configPath: configPath, - registry: registry, - state: stateManager, - eventBus: eventBus, - fallback: fallbackChain, - cmdRegistry: commands.NewRegistry(commands.BuiltinDefinitions()), - steering: newSteeringQueue(parseSteeringMode(cfg.Agents.Defaults.SteeringMode)), - workerSem: make(chan struct{}, workerPoolSize), + bus: msgBus, + cfg: cfg, + configPath: configPath, + cooldownPath: cooldownPath, + registry: registry, + state: stateManager, + eventBus: eventBus, + fallback: fallbackChain, + cmdRegistry: commands.NewRegistry(commands.BuiltinDefinitions()), + steering: newSteeringQueue(parseSteeringMode(cfg.Agents.Defaults.SteeringMode)), + workerSem: make(chan struct{}, workerPoolSize), } al.providerFactory = providers.CreateProviderFromConfig al.hooks = NewHookManager(eventBus) diff --git a/pkg/agent/loop_test.go b/pkg/agent/loop_test.go index bc16d109e..90025df3e 100644 --- a/pkg/agent/loop_test.go +++ b/pkg/agent/loop_test.go @@ -2646,8 +2646,8 @@ func TestProcessMessage_FallbackUsesPerCandidateProvider(t *testing.T) { Content: "hi", }) - if resp != "fallback reply" { - t.Fatalf("response = %q, want %q (fallback provider)", resp, "fallback reply") + if !strings.HasPrefix(resp, "fallback reply") || !strings.Contains(resp, "šŸ¦ž") { + t.Fatalf("response = %q, want it to contain %q and šŸ¦ž (fallback provider)", resp, "fallback reply") } if primaryCalls == 0 { t.Fatal("primary server was never called; expected at least one attempt") @@ -2723,8 +2723,8 @@ func TestProcessMessage_FallbackUsesActiveProviderWhenCandidateNotRegistered(t * Content: "hi", }) - if resp != "active provider reply" { - t.Fatalf("response = %q, want %q", resp, "active provider reply") + if !strings.HasPrefix(resp, "active provider reply") || !strings.Contains(resp, "šŸ¦ž") { + t.Fatalf("response = %q, want it to contain %q and šŸ¦ž", resp, "active provider reply") } if callCount < 2 { t.Fatalf("primary server calls = %d, want >= 2 (one 429 + one success via activeProvider)", callCount) diff --git a/pkg/agent/loop_turn.go b/pkg/agent/loop_turn.go index b82bad9d4..7bce21b29 100644 --- a/pkg/agent/loop_turn.go +++ b/pkg/agent/loop_turn.go @@ -387,7 +387,11 @@ turnLoop: fbResult.Provider, fbResult.Model, len(fbResult.Attempts)+1), map[string]any{"agent_id": ts.agent.ID, "iteration": iteration}, ) - ts.SetFallbackInfo(true, fbResult.Model) + displayName := fbResult.Model + if strings.HasPrefix(fbResult.IdentityKey, "model_name:") { + displayName = strings.TrimPrefix(fbResult.IdentityKey, "model_name:") + } + ts.SetFallbackInfo(true, displayName) } return fbResult.Response, nil } diff --git a/pkg/config/config.go b/pkg/config/config.go index 5bc96fb12..ca367647c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -530,8 +530,9 @@ type VoiceConfig struct { // Default protocol is "openai" if no prefix is specified. type ModelConfig struct { // Required fields - ModelName string `json:"model_name"` // User-facing alias for the model - Model string `json:"model"` // Protocol/model-identifier (e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4.6") + ModelName string `json:"model_name"` // User-facing alias for the model + Model string `json:"model"` // Protocol/model-identifier (e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4.6") + Protocol string `json:"protocol,omitempty"` // Explicit protocol (e.g., "openai", "openrouter", "anthropic") // HTTP-based providers APIBase string `json:"api_base,omitempty"` // API endpoint URL diff --git a/pkg/providers/common/common.go b/pkg/providers/common/common.go index 90142fb8b..2e137e6b1 100644 --- a/pkg/providers/common/common.go +++ b/pkg/providers/common/common.go @@ -36,7 +36,7 @@ type ( ReasoningDetail = protocoltypes.ReasoningDetail ) -const DefaultRequestTimeout = 120 * time.Second +const DefaultRequestTimeout = 30 * time.Second // NewHTTPClient creates an *http.Client with an optional proxy and the default timeout. func NewHTTPClient(proxy string) *http.Client { diff --git a/pkg/providers/cooldown.go b/pkg/providers/cooldown.go index b0d8608dc..0fa02c8f3 100644 --- a/pkg/providers/cooldown.go +++ b/pkg/providers/cooldown.go @@ -1,7 +1,10 @@ package providers import ( + "encoding/json" "math" + "os" + "path/filepath" "sync" "time" ) @@ -16,6 +19,7 @@ type CooldownTracker struct { mu sync.RWMutex entries map[string]*cooldownEntry failureWindow time.Duration + storagePath string nowFunc func() time.Time // for testing } @@ -29,20 +33,23 @@ type cooldownEntry struct { } // NewCooldownTracker creates a tracker with default 24h failure window. -func NewCooldownTracker() *CooldownTracker { - return &CooldownTracker{ +func NewCooldownTracker(storagePath string) *CooldownTracker { + ct := &CooldownTracker{ entries: make(map[string]*cooldownEntry), failureWindow: defaultFailureWindow, + storagePath: storagePath, nowFunc: time.Now, } + if storagePath != "" { + ct.Load() + } + return ct } // MarkFailure records a failure for a provider and sets appropriate cooldown. // Resets error counts if last failure was more than failureWindow ago. func (ct *CooldownTracker) MarkFailure(provider string, reason FailoverReason) { ct.mu.Lock() - defer ct.mu.Unlock() - now := ct.nowFunc() entry := ct.getOrCreate(provider) @@ -53,6 +60,9 @@ func (ct *CooldownTracker) MarkFailure(provider string, reason FailoverReason) { } entry.ErrorCount++ + if entry.FailureCounts == nil { + entry.FailureCounts = make(map[FailoverReason]int) + } entry.FailureCounts[reason]++ entry.LastFailure = now @@ -63,15 +73,20 @@ func (ct *CooldownTracker) MarkFailure(provider string, reason FailoverReason) { } else { entry.CooldownEnd = now.Add(calculateStandardCooldown(entry.ErrorCount)) } + + // Capture state for saving outside the lock + toSave := ct.copyEntriesLocked() + ct.mu.Unlock() + + ct.persist(toSave) } // MarkSuccess resets all counters and cooldowns for a provider. func (ct *CooldownTracker) MarkSuccess(provider string) { ct.mu.Lock() - defer ct.mu.Unlock() - entry := ct.entries[provider] if entry == nil { + ct.mu.Unlock() return } @@ -80,6 +95,12 @@ func (ct *CooldownTracker) MarkSuccess(provider string) { entry.CooldownEnd = time.Time{} entry.DisabledUntil = time.Time{} entry.DisabledReason = "" + + // Capture state for saving outside the lock + toSave := ct.copyEntriesLocked() + ct.mu.Unlock() + + ct.persist(toSave) } // IsAvailable returns true if the provider is not in cooldown or disabled. @@ -162,6 +183,93 @@ func (ct *CooldownTracker) FailureCount(provider string, reason FailoverReason) return entry.FailureCounts[reason] } +// Load reads cooldown state from disk. +func (ct *CooldownTracker) Load() { + if ct.storagePath == "" { + return + } + + data, err := os.ReadFile(ct.storagePath) + if err != nil { + return // ignore missing file + } + + var loaded map[string]*cooldownEntry + if err := json.Unmarshal(data, &loaded); err != nil { + return + } + + ct.mu.Lock() + defer ct.mu.Unlock() + + now := ct.nowFunc() + for k, v := range loaded { + // Only load entries that aren't fully expired yet + if (!v.CooldownEnd.IsZero() && now.Before(v.CooldownEnd)) || + (!v.DisabledUntil.IsZero() && now.Before(v.DisabledUntil)) || + (!v.LastFailure.IsZero() && now.Sub(v.LastFailure) < ct.failureWindow) { + if v.FailureCounts == nil { + v.FailureCounts = make(map[FailoverReason]int) + } + ct.entries[k] = v + } + } +} + +// Save writes cooldown state to disk. +func (ct *CooldownTracker) Save() { + ct.mu.RLock() + toSave := ct.copyEntriesLocked() + ct.mu.RUnlock() + + ct.persist(toSave) +} + +func (ct *CooldownTracker) copyEntriesLocked() map[string]*cooldownEntry { + toSave := make(map[string]*cooldownEntry) + now := ct.nowFunc() + for k, v := range ct.entries { + if v == nil { + continue + } + // Only save entries that are still relevant + if (!v.CooldownEnd.IsZero() && now.Before(v.CooldownEnd)) || + (!v.DisabledUntil.IsZero() && now.Before(v.DisabledUntil)) || + (!v.LastFailure.IsZero() && now.Sub(v.LastFailure) < ct.failureWindow) { + + // Deep copy the entry to avoid data races when serializing outside the lock + copy := *v + if v.FailureCounts != nil { + copy.FailureCounts = make(map[FailoverReason]int) + for r, c := range v.FailureCounts { + copy.FailureCounts[r] = c + } + } + toSave[k] = © + } + } + return toSave +} + +func (ct *CooldownTracker) persist(toSave map[string]*cooldownEntry) { + if ct.storagePath == "" { + return + } + + if len(toSave) == 0 { + os.Remove(ct.storagePath) // cleanup if empty + return + } + + data, err := json.MarshalIndent(toSave, "", " ") + if err != nil { + return + } + + os.MkdirAll(filepath.Dir(ct.storagePath), 0755) + os.WriteFile(ct.storagePath, data, 0644) +} + func (ct *CooldownTracker) getOrCreate(provider string) *cooldownEntry { entry := ct.entries[provider] if entry == nil { diff --git a/pkg/providers/cooldown_test.go b/pkg/providers/cooldown_test.go index b517e7feb..22e6f6240 100644 --- a/pkg/providers/cooldown_test.go +++ b/pkg/providers/cooldown_test.go @@ -8,13 +8,13 @@ import ( func newTestTracker(now time.Time) (*CooldownTracker, *time.Time) { current := now - ct := NewCooldownTracker() + ct := NewCooldownTracker("") ct.nowFunc = func() time.Time { return current } return ct, ¤t } func TestCooldown_InitiallyAvailable(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") if !ct.IsAvailable("openai") { t.Error("new provider should be available") } @@ -110,7 +110,7 @@ func TestCooldown_BillingCap(t *testing.T) { } func TestCooldown_SuccessReset(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") ct.MarkFailure("openai", FailoverRateLimit) ct.MarkFailure("openai", FailoverBilling) @@ -157,7 +157,7 @@ func TestCooldown_FailureWindowReset(t *testing.T) { } func TestCooldown_PerReasonTracking(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") ct.MarkFailure("openai", FailoverRateLimit) ct.MarkFailure("openai", FailoverRateLimit) @@ -218,7 +218,7 @@ func TestCooldown_CooldownRemaining(t *testing.T) { } func TestCooldown_SuccessOnUnknownProvider(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") // Should not panic ct.MarkSuccess("nonexistent") if !ct.IsAvailable("nonexistent") { @@ -227,7 +227,7 @@ func TestCooldown_SuccessOnUnknownProvider(t *testing.T) { } func TestCooldown_ConcurrentAccess(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") var wg sync.WaitGroup for range 100 { @@ -251,7 +251,7 @@ func TestCooldown_ConcurrentAccess(t *testing.T) { } func TestCooldown_MultipleProviders(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") ct.MarkFailure("openai", FailoverRateLimit) ct.MarkFailure("anthropic", FailoverBilling) diff --git a/pkg/providers/factory_provider.go b/pkg/providers/factory_provider.go index ab68b326a..408fca170 100644 --- a/pkg/providers/factory_provider.go +++ b/pkg/providers/factory_provider.go @@ -109,6 +109,9 @@ func ResolveAPIBase(cfg *config.ModelConfig) string { return strings.TrimRight(apiBase, "/") } protocol, _ := ExtractProtocol(cfg.Model) + if cfg.Protocol != "" { + protocol = cfg.Protocol + } return strings.TrimRight(getDefaultAPIBase(protocol), "/") } @@ -128,6 +131,10 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err } protocol, modelID := ExtractProtocol(cfg.Model) + if cfg.Protocol != "" { + protocol = cfg.Protocol + modelID = cfg.Model + } userAgent := cfg.UserAgent if userAgent == "" { @@ -224,13 +231,13 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err "qwen-us", "dashscope-us", "mistral", "avian", "longcat", "modelscope", "novita", "coding-plan", "alibaba-coding", "qwen-coding", "mimo": // All other OpenAI-compatible HTTP providers - if cfg.APIKey() == "" && cfg.APIBase == "" && !isEmptyAPIKeyAllowed(protocol) { - return nil, "", fmt.Errorf("api_key or api_base is required for HTTP-based protocol %q", protocol) - } apiBase := cfg.APIBase if apiBase == "" { apiBase = getDefaultAPIBase(protocol) } + if cfg.APIKey() == "" && apiBase == "" && !isEmptyAPIKeyAllowed(protocol) { + return nil, "", fmt.Errorf("api_key or api_base is required for HTTP-based protocol %q", protocol) + } return NewHTTPProviderWithMaxTokensFieldAndRequestTimeout( cfg.APIKey(), apiBase, diff --git a/pkg/providers/fallback.go b/pkg/providers/fallback.go index 36092105b..0e46594ba 100644 --- a/pkg/providers/fallback.go +++ b/pkg/providers/fallback.go @@ -32,10 +32,11 @@ func (c FallbackCandidate) StableKey() string { // FallbackResult contains the successful response and metadata about all attempts. type FallbackResult struct { - Response *LLMResponse - Provider string - Model string - Attempts []FallbackAttempt + Response *LLMResponse + Provider string + Model string + IdentityKey string + Attempts []FallbackAttempt } // FallbackAttempt records one attempt in the fallback chain. @@ -187,6 +188,7 @@ func (fc *FallbackChain) Execute( result.Response = resp result.Provider = candidate.Provider result.Model = candidate.Model + result.IdentityKey = candidate.IdentityKey return result, nil } @@ -305,6 +307,7 @@ func (fc *FallbackChain) ExecuteImage( result.Response = resp result.Provider = candidate.Provider result.Model = candidate.Model + result.IdentityKey = candidate.IdentityKey return result, nil } diff --git a/pkg/providers/fallback_multikey_test.go b/pkg/providers/fallback_multikey_test.go index 10481ec61..0efae6d3c 100644 --- a/pkg/providers/fallback_multikey_test.go +++ b/pkg/providers/fallback_multikey_test.go @@ -24,7 +24,7 @@ func TestMultiKeyFailover(t *testing.T) { } // Create fallback chain - cooldown := NewCooldownTracker() + cooldown := NewCooldownTracker("") chain := NewFallbackChain(cooldown, nil) // Mock run function: first call fails with 429, second succeeds @@ -81,7 +81,7 @@ func TestMultiKeyFailoverAllFail(t *testing.T) { candidates := ResolveCandidates(cfg, "zhipu") - cooldown := NewCooldownTracker() + cooldown := NewCooldownTracker("") chain := NewFallbackChain(cooldown, nil) // Mock run function: all calls fail with rate limit @@ -126,7 +126,7 @@ func TestMultiKeyFailoverCooldown(t *testing.T) { candidates := ResolveCandidates(cfg, "zhipu") - cooldown := NewCooldownTracker() + cooldown := NewCooldownTracker("") chain := NewFallbackChain(cooldown, nil) // Put the first model in cooldown (using ModelKey now, not just provider) @@ -182,7 +182,7 @@ func TestMultiKeyFailoverWithFormatError(t *testing.T) { candidates := ResolveCandidates(cfg, "zhipu") - cooldown := NewCooldownTracker() + cooldown := NewCooldownTracker("") chain := NewFallbackChain(cooldown, nil) // Mock run function: first call fails with format error (bad request) @@ -262,7 +262,7 @@ func TestMultiKeyWithModelFallback(t *testing.T) { ) } - cooldown := NewCooldownTracker() + cooldown := NewCooldownTracker("") chain := NewFallbackChain(cooldown, nil) // Mock run function: first two fail, third succeeds (model fallback) @@ -336,7 +336,7 @@ func TestMultiKeyFailoverMixedErrors(t *testing.T) { candidates := ResolveCandidates(cfg, "zhipu") - cooldown := NewCooldownTracker() + cooldown := NewCooldownTracker("") chain := NewFallbackChain(cooldown, nil) // Mock run function: different errors for each key diff --git a/pkg/providers/fallback_test.go b/pkg/providers/fallback_test.go index 07cc01baa..16b35f65d 100644 --- a/pkg/providers/fallback_test.go +++ b/pkg/providers/fallback_test.go @@ -18,7 +18,7 @@ func successRun(content string) func(ctx context.Context, provider, model string } func TestFallback_SingleCandidate_Success(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") fc := NewFallbackChain(ct, nil) candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4")} @@ -35,7 +35,7 @@ func TestFallback_SingleCandidate_Success(t *testing.T) { } func TestFallback_SecondCandidateSuccess(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") fc := NewFallbackChain(ct, nil) candidates := []FallbackCandidate{ @@ -68,7 +68,7 @@ func TestFallback_SecondCandidateSuccess(t *testing.T) { } func TestFallback_AllFail(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") fc := NewFallbackChain(ct, nil) candidates := []FallbackCandidate{ @@ -95,7 +95,7 @@ func TestFallback_AllFail(t *testing.T) { } func TestFallback_ContextCanceled(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") fc := NewFallbackChain(ct, nil) ctx, cancel := context.WithCancel(context.Background()) @@ -122,7 +122,7 @@ func TestFallback_ContextCanceled(t *testing.T) { } func TestFallback_NonRetriableError(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") fc := NewFallbackChain(ct, nil) candidates := []FallbackCandidate{ @@ -192,7 +192,7 @@ func TestFallback_CooldownSkip(t *testing.T) { } func TestFallback_AllInCooldown(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") fc := NewFallbackChain(ct, nil) // Put all models in cooldown (using ModelKey now) @@ -220,7 +220,7 @@ func TestFallback_AllInCooldown(t *testing.T) { } func TestFallback_NoCandidates(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") fc := NewFallbackChain(ct, nil) _, err := fc.Execute(context.Background(), nil, successRun("ok")) @@ -231,7 +231,7 @@ func TestFallback_NoCandidates(t *testing.T) { func TestFallback_EmptyFallbacks(t *testing.T) { // Single primary, no fallbacks: should work like direct call - ct := NewCooldownTracker() + ct := NewCooldownTracker("") fc := NewFallbackChain(ct, nil) candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4")} @@ -245,7 +245,7 @@ func TestFallback_EmptyFallbacks(t *testing.T) { } func TestFallback_UnclassifiedError(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") fc := NewFallbackChain(ct, nil) candidates := []FallbackCandidate{ @@ -278,7 +278,7 @@ func assertFallbackErrorFallsBack( ) { t.Helper() - ct := NewCooldownTracker() + ct := NewCooldownTracker("") fc := NewFallbackChain(ct, nil) candidates := []FallbackCandidate{ @@ -338,7 +338,7 @@ func TestFallback_TimeoutErrorFallsBack(t *testing.T) { } func TestFallback_SuccessResetsCooldown(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") fc := NewFallbackChain(ct, nil) candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4")} @@ -375,7 +375,7 @@ func assertLocalRateLimitSkipsToHealthyFallback( ) { t.Helper() - ct := NewCooldownTracker() + ct := NewCooldownTracker("") rl := NewRateLimiterRegistry() rl.Register(primaryKey, 1) if err := rl.Wait(context.Background(), primaryKey); err != nil { @@ -432,7 +432,7 @@ func TestFallback_LocalRateLimitSkipsToHealthyFallback(t *testing.T) { // --- Image Fallback Tests --- func TestImageFallback_Success(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") fc := NewFallbackChain(ct, nil) candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4o")} @@ -446,7 +446,7 @@ func TestImageFallback_Success(t *testing.T) { } func TestImageFallback_DimensionError(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") fc := NewFallbackChain(ct, nil) candidates := []FallbackCandidate{ @@ -470,7 +470,7 @@ func TestImageFallback_DimensionError(t *testing.T) { } func TestImageFallback_SizeError(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") fc := NewFallbackChain(ct, nil) candidates := []FallbackCandidate{ @@ -494,7 +494,7 @@ func TestImageFallback_SizeError(t *testing.T) { } func TestImageFallback_RetryOnOtherErrors(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") fc := NewFallbackChain(ct, nil) candidates := []FallbackCandidate{ @@ -540,7 +540,7 @@ func TestImageFallback_LocalRateLimitSkipsToHealthyFallback(t *testing.T) { } func TestImageFallback_NoCandidates(t *testing.T) { - ct := NewCooldownTracker() + ct := NewCooldownTracker("") fc := NewFallbackChain(ct, nil) _, err := fc.ExecuteImage(context.Background(), nil, successRun("ok")) diff --git a/pkg/seahorse/.omc/state/last-tool-error.json b/pkg/seahorse/.omc/state/last-tool-error.json new file mode 100644 index 000000000..2e7273e23 --- /dev/null +++ b/pkg/seahorse/.omc/state/last-tool-error.json @@ -0,0 +1,7 @@ +{ + "tool_name": "Bash", + "tool_input_preview": "{\"command\":\"cd /home/yliu/repos/picoclaw && make lint 2>&1\",\"timeout\":120000}", + "error": "Exit code 2\npkg/agent/context_seahorse_test.go:1027:1: File is not properly formatted (gci)\n\t\t\tEarliestAt: &now,\n^\n1 issues:\n* gci: 1\nmake: *** [Makefile:264: lint] Error 1", + "timestamp": "2026-04-04T02:38:32.067Z", + "retry_count": 6 +} \ No newline at end of file diff --git a/pkg/tools/freeride.go b/pkg/tools/freeride.go index 09a6f3ba2..20114f816 100644 --- a/pkg/tools/freeride.go +++ b/pkg/tools/freeride.go @@ -218,7 +218,8 @@ func (t *FreeRideTool) handleAuto(ctx context.Context, limit int) *ToolResult { if !modelExists(cfgObj, modelName) { mc := &config.ModelConfig{ ModelName: modelName, - Model: "openrouter/" + m.ID, + Model: m.ID, + Protocol: "openrouter", Enabled: true, } mc.SetAPIKey("env://OPENROUTER_API_KEY") diff --git a/scratch/check_paths.go b/scratch/check_paths.go new file mode 100644 index 000000000..f6eb96d75 --- /dev/null +++ b/scratch/check_paths.go @@ -0,0 +1,28 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/sipeed/picoclaw/pkg/agent" + "github.com/sipeed/picoclaw/pkg/config" +) + +func main() { + cfg, err := config.LoadConfig(os.ExpandEnv("$HOME/.picoclaw/config.json")) + if err != nil { + fmt.Printf("Error: %v\n", err) + return + } + registry := agent.NewAgentRegistry(cfg, nil) + defaultAgent := registry.GetDefaultAgent() + if defaultAgent == nil { + fmt.Println("No default agent") + return + } + + cooldownPath := filepath.Join(filepath.Dir(filepath.Clean(defaultAgent.Workspace)), "cooldowns.json") + fmt.Printf("Workspace: %s\n", defaultAgent.Workspace) + fmt.Printf("Cooldown Path: %s\n", cooldownPath) +}