Merge FreeRide feature and doc updates into security_shield_v2
This commit is contained in:
commit
383132dbe6
23 changed files with 401 additions and 91 deletions
|
|
@ -96,8 +96,8 @@
|
||||||
👁️ **Vision pipeline**: Send images and files directly to the Agent — automatic base64 encoding for multimodal LLMs.
|
👁️ **Vision pipeline**: Send images and files directly to the Agent — automatic base64 encoding for multimodal LLMs.
|
||||||
|
|
||||||
🧠 **Smart routing**: Rule-based model routing — simple queries go to lightweight models, saving API costs.
|
🧠 **Smart routing**: Rule-based model routing — simple queries go to lightweight models, saving API costs.
|
||||||
|
|
||||||
🧬 **FreeRide**: Intelligent model rotation using OpenRouter's free pool — never pay for basic LLM traffic again. [Learn more](docs/guides/freeride.md).
|
🧬 **FreeRide**: Intelligent model rotation using OpenRouter's free pool — never pay for basic LLM traffic again. [Learn more](docs/guides/freeride.md).
|
||||||
|
❄️ **Persistent Cooldowns**: Remembers rate-limited models across restarts via `cooldowns.json`, ensuring instant failover and zero "cooldown amnesia".
|
||||||
|
|
||||||
_*Recent builds may use 10-20MB due to rapid PR merges. Resource optimization is planned. Boot speed comparison based on 0.8GHz single-core benchmarks (see table below)._
|
_*Recent builds may use 10-20MB due to rapid PR merges. Resource optimization is planned. Boot speed comparison based on 0.8GHz single-core benchmarks (see table below)._
|
||||||
|
|
||||||
|
|
@ -624,7 +624,7 @@ For detailed guides beyond this README:
|
||||||
| [Scheduled Tasks and Cron Jobs](docs/reference/cron.md) | Cron schedule types, deliver modes, command gates, job storage |
|
| [Scheduled Tasks and Cron Jobs](docs/reference/cron.md) | Cron schedule types, deliver modes, command gates, job storage |
|
||||||
| [Providers & Models](docs/guides/providers.md) | 30+ LLM providers, model routing, model_list configuration |
|
| [Providers & Models](docs/guides/providers.md) | 30+ LLM providers, model routing, model_list configuration |
|
||||||
| [Spawn & Async Tasks](docs/guides/spawn-tasks.md) | Quick tasks, long tasks with spawn, async sub-agent orchestration |
|
| [Spawn & Async Tasks](docs/guides/spawn-tasks.md) | Quick tasks, long tasks with spawn, async sub-agent orchestration |
|
||||||
| [FreeRide](docs/guides/freeride.md) | Dynamic free model rotation and K3s secret management |
|
| [FreeRide](docs/freeride.md) | Dynamic free model rotation and K3s secret management |
|
||||||
| [Hooks](docs/architecture/hooks/README.md) | Event-driven hook system: observers, interceptors, approval hooks |
|
| [Hooks](docs/architecture/hooks/README.md) | Event-driven hook system: observers, interceptors, approval hooks |
|
||||||
| [Steering](docs/architecture/steering.md) | Inject messages into a running agent loop between tool calls |
|
| [Steering](docs/architecture/steering.md) | Inject messages into a running agent loop between tool calls |
|
||||||
| [SubTurn](docs/architecture/subturn.md) | Subagent coordination, concurrency control, lifecycle |
|
| [SubTurn](docs/architecture/subturn.md) | Subagent coordination, concurrency control, lifecycle |
|
||||||
|
|
|
||||||
84
cmd/picoclaw/internal/freeride/command.go
Normal file
84
cmd/picoclaw/internal/freeride/command.go
Normal file
|
|
@ -0,0 +1,84 @@
|
||||||
|
package freeride
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"github.com/sipeed/picoclaw/cmd/picoclaw/internal"
|
||||||
|
"github.com/sipeed/picoclaw/pkg/tools"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewFreerideCommand returns a new cobra.Command for managing OpenRouter free models.
|
||||||
|
func NewFreerideCommand() *cobra.Command {
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "freeride",
|
||||||
|
Short: "Manage OpenRouter free models and fallbacks",
|
||||||
|
Long: "FreeRide automatically discovers and configures OpenRouter's best free models as fallbacks for your PicoClaw agent.",
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
return cmd.Help()
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.AddCommand(
|
||||||
|
newListCommand(),
|
||||||
|
newAutoCommand(),
|
||||||
|
newStatusCommand(),
|
||||||
|
)
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func newListCommand() *cobra.Command {
|
||||||
|
var limit int
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "list",
|
||||||
|
Short: "List available free models from OpenRouter",
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
t := tools.NewFreeRideTool(internal.GetConfigPath(), nil)
|
||||||
|
result := t.Execute(context.Background(), map[string]any{
|
||||||
|
"command": "list",
|
||||||
|
"limit": float64(limit),
|
||||||
|
})
|
||||||
|
fmt.Println(result.ForLLM)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
cmd.Flags().IntVarP(&limit, "limit", "l", 10, "Number of models to list")
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func newAutoCommand() *cobra.Command {
|
||||||
|
var limit int
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "auto",
|
||||||
|
Short: "Automatically configure best free models as fallbacks",
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
t := tools.NewFreeRideTool(internal.GetConfigPath(), nil)
|
||||||
|
result := t.Execute(context.Background(), map[string]any{
|
||||||
|
"command": "auto",
|
||||||
|
"limit": float64(limit),
|
||||||
|
})
|
||||||
|
fmt.Println(result.ForLLM)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
cmd.Flags().IntVarP(&limit, "limit", "l", 5, "Number of fallbacks to configure")
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func newStatusCommand() *cobra.Command {
|
||||||
|
return &cobra.Command{
|
||||||
|
Use: "status",
|
||||||
|
Short: "Check current FreeRide configuration",
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
t := tools.NewFreeRideTool(internal.GetConfigPath(), nil)
|
||||||
|
result := t.Execute(context.Background(), map[string]any{
|
||||||
|
"command": "status",
|
||||||
|
})
|
||||||
|
fmt.Println(result.ForLLM)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -18,6 +18,7 @@ import (
|
||||||
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/auth"
|
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/auth"
|
||||||
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/cliui"
|
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/cliui"
|
||||||
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/cron"
|
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/cron"
|
||||||
|
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/freeride"
|
||||||
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/gateway"
|
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/gateway"
|
||||||
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/migrate"
|
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/migrate"
|
||||||
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/model"
|
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/model"
|
||||||
|
|
@ -84,6 +85,7 @@ picoclaw --no-color status`,
|
||||||
onboard.NewOnboardCommand(),
|
onboard.NewOnboardCommand(),
|
||||||
agent.NewAgentCommand(),
|
agent.NewAgentCommand(),
|
||||||
auth.NewAuthCommand(),
|
auth.NewAuthCommand(),
|
||||||
|
freeride.NewFreerideCommand(),
|
||||||
gateway.NewGatewayCommand(),
|
gateway.NewGatewayCommand(),
|
||||||
status.NewStatusCommand(),
|
status.NewStatusCommand(),
|
||||||
cron.NewCronCommand(),
|
cron.NewCronCommand(),
|
||||||
|
|
|
||||||
|
|
@ -40,6 +40,7 @@ func TestNewPicoclawCommand(t *testing.T) {
|
||||||
"agent",
|
"agent",
|
||||||
"auth",
|
"auth",
|
||||||
"cron",
|
"cron",
|
||||||
|
"freeride",
|
||||||
"gateway",
|
"gateway",
|
||||||
"migrate",
|
"migrate",
|
||||||
"model",
|
"model",
|
||||||
|
|
|
||||||
|
|
@ -12,23 +12,16 @@ FreeRide is a dynamic model rotation and failover system for PicoClaw that lever
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
FreeRide is implemented as a native PicoClaw tool.
|
FreeRide is implemented as a native PicoClaw tool. For production environments (especially in the **main branch**), ensure you follow the [Security Configuration](../security/security_configuration.md) to manage your API keys safely.
|
||||||
|
|
||||||
### 1. Enable the Tool
|
### 1. Enable the Tool
|
||||||
Ensure the `freeride` tool is enabled and whitelisted in your `config.json`:
|
Ensure the `skills` tool is enabled in your `config.json` (FreeRide is bundled with the skills system):
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"tools": {
|
"tools": {
|
||||||
"whitelist": ["freeride", ...],
|
"skills": {
|
||||||
"whitelist_enabled": true,
|
"enabled": true
|
||||||
"security_policy": {
|
|
||||||
"enabled": true,
|
|
||||||
"config": {
|
|
||||||
"allowed_tools": {
|
|
||||||
"freeride": true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -106,11 +99,20 @@ env:
|
||||||
key: openrouter-api-key
|
key: openrouter-api-key
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Cooldown Persistence & Timing ❄️
|
||||||
|
|
||||||
|
To prevent the agent from "hanging" or retrying known-failed models, PicoClaw uses a two-pronged approach:
|
||||||
|
|
||||||
|
### 1. Zero-Amnesia Persistence
|
||||||
|
Model failures (e.g., 429 Rate Limits) are saved to `~/.picoclaw/cooldowns.json`. This ensures that if you restart the agent, it **remembers** which models were saturated and skips them instantly. You no longer have to wait through a series of timeouts every time you restart.
|
||||||
|
|
||||||
|
### 2. Aggressive 30s Timeout
|
||||||
|
The default request timeout for LLM calls is **30 seconds**. If a free model is stalled or unresponsive, the agent will move to the next fallback in your pool much faster than the standard HTTP default.
|
||||||
|
|
||||||
## Troubleshooting
|
## Troubleshooting
|
||||||
|
|
||||||
- **404 Errors**: Ensure the model is still available on OpenRouter using `freeride list`. If it's gone, run `freeride auto` to refresh your fallback pool.
|
- **404 Errors**: Ensure the model is still available on OpenRouter using `freeride list`. If it's gone, run `freeride auto` to refresh your fallback pool.
|
||||||
- **429 Rate Limiting**: This is common with free models. PicoClaw will automatically try the next model in your `model_fallbacks` list.
|
- **429 Rate Limiting**: This is common with free models. PicoClaw will automatically try the next model in your `model_fallbacks` list and persist the cooldown to `cooldowns.json`.
|
||||||
- **Security Blocks**: Ensure `freeride` is added to your `security_policy` allowed tools map.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -25,13 +25,20 @@ AgentLoop.callLLM()
|
||||||
The rate limiter runs **after** the cooldown check and **before** the provider call, so:
|
The rate limiter runs **after** the cooldown check and **before** the provider call, so:
|
||||||
- Candidates already in cooldown are skipped entirely (no token consumed)
|
- Candidates already in cooldown are skipped entirely (no token consumed)
|
||||||
- Candidates that are available get throttled to the configured RPM
|
- Candidates that are available get throttled to the configured RPM
|
||||||
|
|
||||||
The same check applies in `ExecuteImage`.
|
## Cooldown Persistence ❄️
|
||||||
|
|
||||||
|
While the Rate Limiter is **proactive**, the Cooldown Tracker is **reactive** (handled *after* a 429 is actually received).
|
||||||
|
|
||||||
|
To ensure stability across restarts, the Cooldown Tracker persists its state to disk:
|
||||||
|
- **Location**: `~/.picoclaw/cooldowns.json` (or sibling to your workspace)
|
||||||
|
- **Behavior**: If the agent is restarted, it loads the failure history and continues to enforce cooldowns. This prevents "initialization hangs" where a new agent process tries a long list of models that are already known to be rate-limited.
|
||||||
|
- **Timing**: Default LLM request timeout is now **30 seconds** to ensure faster failover.
|
||||||
|
|
||||||
### Thread safety
|
### Thread safety
|
||||||
|
|
||||||
`RateLimiterRegistry` is safe for concurrent use. The per-limiter token bucket uses a fine-grained mutex so concurrent goroutines each acquire their own token independently.
|
`RateLimiterRegistry` is safe for concurrent use. The per-limiter token bucket uses a fine-grained mutex so concurrent goroutines each acquire their own token independently.
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
Set `rpm` on any model in `model_list`:
|
Set `rpm` on any model in `model_list`:
|
||||||
|
|
|
||||||
|
|
@ -541,6 +541,19 @@ go test ./pkg/config -run TestSecurityConfig
|
||||||
- Remember: Models, Brave, Tavily, Perplexity MUST use `api_keys` (array format)
|
- Remember: Models, Brave, Tavily, Perplexity MUST use `api_keys` (array format)
|
||||||
- GLMSearch and BaiduSearch MUST use `api_key` (single string format)
|
- GLMSearch and BaiduSearch MUST use `api_key` (single string format)
|
||||||
|
|
||||||
|
## FreeRide & Dynamic Failover Security
|
||||||
|
|
||||||
|
When using the **FreeRide** dynamic model failover system, PicoClaw dynamically adds models to your `model_list`. To ensure these models stay secure:
|
||||||
|
|
||||||
|
1. **Protocol-Level Security**: You can define a single entry in `.security.yml` that matches a specific provider/protocol. For example, to provide an identity for ALL OpenRouter models added by FreeRide:
|
||||||
|
```yaml
|
||||||
|
model_list:
|
||||||
|
openrouter:
|
||||||
|
api_keys: ["sk-or-v1-your-global-key"]
|
||||||
|
```
|
||||||
|
2. **Environment Variable Fallback**: FreeRide is configured to look for `OPENROUTER_API_KEY` in the environment. In production (`main` branch), it is recommended to use K3s Secrets or `SecureString` to inject this.
|
||||||
|
3. **Sensitive Filtering**: All models added by FreeRide are subject to the same `sensitive_data_filtering.md` rules as your primary models.
|
||||||
|
|
||||||
### Load Balancing/Failover Issues
|
### Load Balancing/Failover Issues
|
||||||
|
|
||||||
- Verify all API keys in the `api_keys` array are valid
|
- Verify all API keys in the `api_keys` array are valid
|
||||||
|
|
|
||||||
37
examples/freeride-config.json
Normal file
37
examples/freeride-config.json
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
{
|
||||||
|
"agents": {
|
||||||
|
"defaults": {
|
||||||
|
"model_name": "google/gemma-3-27b-it:free",
|
||||||
|
"model_fallbacks": [
|
||||||
|
"google/gemma-3-27b-it:free",
|
||||||
|
"nvidia/nemotron-4-340b-instruct:free",
|
||||||
|
"qwen/qwen-2.5-72b-instruct:free",
|
||||||
|
"mistralai/mistral-small-24b-it-v1:free"
|
||||||
|
],
|
||||||
|
"max_tokens": 4096,
|
||||||
|
"max_tool_iterations": 10
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"model_list": [
|
||||||
|
{
|
||||||
|
"model_name": "google/gemma-3-27b-it:free",
|
||||||
|
"model": "google/gemma-3-27b-it:free",
|
||||||
|
"protocol": "openrouter"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"model_name": "nvidia/nemotron-4-340b-instruct:free",
|
||||||
|
"model": "nvidia/nemotron-4-340b-instruct:free",
|
||||||
|
"protocol": "openrouter"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"model_name": "qwen/qwen-2.5-72b-instruct:free",
|
||||||
|
"model": "qwen/qwen-2.5-72b-instruct:free",
|
||||||
|
"protocol": "openrouter"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"model_name": "mistralai/mistral-small-24b-it-v1:free",
|
||||||
|
"model": "mistralai/mistral-small-24b-it-v1:free",
|
||||||
|
"protocol": "openrouter"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -64,6 +64,7 @@ type AgentLoop struct {
|
||||||
turnSeq atomic.Uint64
|
turnSeq atomic.Uint64
|
||||||
activeRequests sync.WaitGroup
|
activeRequests sync.WaitGroup
|
||||||
configPath string
|
configPath string
|
||||||
|
cooldownPath string
|
||||||
|
|
||||||
reloadFunc func() error
|
reloadFunc func() error
|
||||||
|
|
||||||
|
|
@ -386,7 +387,7 @@ func (al *AgentLoop) ReloadProviderAndConfig(
|
||||||
newRL.RegisterCandidates(agent.LightCandidates)
|
newRL.RegisterCandidates(agent.LightCandidates)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
al.fallback = providers.NewFallbackChain(providers.NewCooldownTracker(), newRL)
|
al.fallback = providers.NewFallbackChain(providers.NewCooldownTracker(al.cooldownPath), newRL)
|
||||||
|
|
||||||
al.mu.Unlock()
|
al.mu.Unlock()
|
||||||
|
|
||||||
|
|
@ -521,16 +522,17 @@ func (al *AgentLoop) runAgentLoop(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if opts.SendResponse && result.finalContent != "" {
|
finalContent := result.finalContent
|
||||||
|
if usedFallback, fallbackModel := ts.GetFallbackInfo(); usedFallback {
|
||||||
|
finalContent += fmt.Sprintf("\n\n🦞 _(FreeRide: %s)_", fallbackModel)
|
||||||
|
}
|
||||||
|
|
||||||
|
if opts.SendResponse && finalContent != "" {
|
||||||
agentID, sessionKey, scope := outboundTurnMetadata(
|
agentID, sessionKey, scope := outboundTurnMetadata(
|
||||||
agent.ID,
|
agent.ID,
|
||||||
opts.Dispatch.SessionKey,
|
opts.Dispatch.SessionKey,
|
||||||
opts.Dispatch.SessionScope,
|
opts.Dispatch.SessionScope,
|
||||||
)
|
)
|
||||||
finalContent := result.finalContent
|
|
||||||
if usedFallback, fallbackModel := ts.GetFallbackInfo(); usedFallback {
|
|
||||||
finalContent += fmt.Sprintf("\n\n🦞 _(FreeRide: %s)_", fallbackModel)
|
|
||||||
}
|
|
||||||
al.bus.PublishOutbound(ctx, bus.OutboundMessage{
|
al.bus.PublishOutbound(ctx, bus.OutboundMessage{
|
||||||
Context: outboundContextFromInbound(
|
Context: outboundContextFromInbound(
|
||||||
opts.Dispatch.InboundContext,
|
opts.Dispatch.InboundContext,
|
||||||
|
|
@ -545,7 +547,7 @@ func (al *AgentLoop) runAgentLoop(
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
return result.finalContent, nil
|
return finalContent, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// selectCandidates returns the model candidates and resolved model name to use
|
// selectCandidates returns the model candidates and resolved model name to use
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,7 @@ package agent
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/sipeed/picoclaw/pkg/audio/tts"
|
"github.com/sipeed/picoclaw/pkg/audio/tts"
|
||||||
|
|
@ -28,7 +29,8 @@ func NewAgentLoop(
|
||||||
registry := NewAgentRegistry(cfg, provider)
|
registry := NewAgentRegistry(cfg, provider)
|
||||||
|
|
||||||
// Set up shared fallback chain with rate limiting.
|
// Set up shared fallback chain with rate limiting.
|
||||||
cooldown := providers.NewCooldownTracker()
|
cooldownPath := filepath.Join(filepath.Dir(filepath.Clean(registry.GetDefaultAgent().Workspace)), "cooldowns.json")
|
||||||
|
cooldown := providers.NewCooldownTracker(cooldownPath)
|
||||||
rl := providers.NewRateLimiterRegistry()
|
rl := providers.NewRateLimiterRegistry()
|
||||||
// Register rate limiters for all agents' candidates so that RPM limits
|
// Register rate limiters for all agents' candidates so that RPM limits
|
||||||
// configured in ModelConfig are enforced before each LLM call.
|
// configured in ModelConfig are enforced before each LLM call.
|
||||||
|
|
@ -56,16 +58,17 @@ func NewAgentLoop(
|
||||||
}
|
}
|
||||||
|
|
||||||
al := &AgentLoop{
|
al := &AgentLoop{
|
||||||
bus: msgBus,
|
bus: msgBus,
|
||||||
cfg: cfg,
|
cfg: cfg,
|
||||||
configPath: configPath,
|
configPath: configPath,
|
||||||
registry: registry,
|
cooldownPath: cooldownPath,
|
||||||
state: stateManager,
|
registry: registry,
|
||||||
eventBus: eventBus,
|
state: stateManager,
|
||||||
fallback: fallbackChain,
|
eventBus: eventBus,
|
||||||
cmdRegistry: commands.NewRegistry(commands.BuiltinDefinitions()),
|
fallback: fallbackChain,
|
||||||
steering: newSteeringQueue(parseSteeringMode(cfg.Agents.Defaults.SteeringMode)),
|
cmdRegistry: commands.NewRegistry(commands.BuiltinDefinitions()),
|
||||||
workerSem: make(chan struct{}, workerPoolSize),
|
steering: newSteeringQueue(parseSteeringMode(cfg.Agents.Defaults.SteeringMode)),
|
||||||
|
workerSem: make(chan struct{}, workerPoolSize),
|
||||||
}
|
}
|
||||||
al.providerFactory = providers.CreateProviderFromConfig
|
al.providerFactory = providers.CreateProviderFromConfig
|
||||||
al.hooks = NewHookManager(eventBus)
|
al.hooks = NewHookManager(eventBus)
|
||||||
|
|
|
||||||
|
|
@ -2646,8 +2646,8 @@ func TestProcessMessage_FallbackUsesPerCandidateProvider(t *testing.T) {
|
||||||
Content: "hi",
|
Content: "hi",
|
||||||
})
|
})
|
||||||
|
|
||||||
if resp != "fallback reply" {
|
if !strings.HasPrefix(resp, "fallback reply") || !strings.Contains(resp, "🦞") {
|
||||||
t.Fatalf("response = %q, want %q (fallback provider)", resp, "fallback reply")
|
t.Fatalf("response = %q, want it to contain %q and 🦞 (fallback provider)", resp, "fallback reply")
|
||||||
}
|
}
|
||||||
if primaryCalls == 0 {
|
if primaryCalls == 0 {
|
||||||
t.Fatal("primary server was never called; expected at least one attempt")
|
t.Fatal("primary server was never called; expected at least one attempt")
|
||||||
|
|
@ -2723,8 +2723,8 @@ func TestProcessMessage_FallbackUsesActiveProviderWhenCandidateNotRegistered(t *
|
||||||
Content: "hi",
|
Content: "hi",
|
||||||
})
|
})
|
||||||
|
|
||||||
if resp != "active provider reply" {
|
if !strings.HasPrefix(resp, "active provider reply") || !strings.Contains(resp, "🦞") {
|
||||||
t.Fatalf("response = %q, want %q", resp, "active provider reply")
|
t.Fatalf("response = %q, want it to contain %q and 🦞", resp, "active provider reply")
|
||||||
}
|
}
|
||||||
if callCount < 2 {
|
if callCount < 2 {
|
||||||
t.Fatalf("primary server calls = %d, want >= 2 (one 429 + one success via activeProvider)", callCount)
|
t.Fatalf("primary server calls = %d, want >= 2 (one 429 + one success via activeProvider)", callCount)
|
||||||
|
|
|
||||||
|
|
@ -387,7 +387,11 @@ turnLoop:
|
||||||
fbResult.Provider, fbResult.Model, len(fbResult.Attempts)+1),
|
fbResult.Provider, fbResult.Model, len(fbResult.Attempts)+1),
|
||||||
map[string]any{"agent_id": ts.agent.ID, "iteration": iteration},
|
map[string]any{"agent_id": ts.agent.ID, "iteration": iteration},
|
||||||
)
|
)
|
||||||
ts.SetFallbackInfo(true, fbResult.Model)
|
displayName := fbResult.Model
|
||||||
|
if strings.HasPrefix(fbResult.IdentityKey, "model_name:") {
|
||||||
|
displayName = strings.TrimPrefix(fbResult.IdentityKey, "model_name:")
|
||||||
|
}
|
||||||
|
ts.SetFallbackInfo(true, displayName)
|
||||||
}
|
}
|
||||||
return fbResult.Response, nil
|
return fbResult.Response, nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -530,8 +530,9 @@ type VoiceConfig struct {
|
||||||
// Default protocol is "openai" if no prefix is specified.
|
// Default protocol is "openai" if no prefix is specified.
|
||||||
type ModelConfig struct {
|
type ModelConfig struct {
|
||||||
// Required fields
|
// Required fields
|
||||||
ModelName string `json:"model_name"` // User-facing alias for the model
|
ModelName string `json:"model_name"` // User-facing alias for the model
|
||||||
Model string `json:"model"` // Protocol/model-identifier (e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4.6")
|
Model string `json:"model"` // Protocol/model-identifier (e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4.6")
|
||||||
|
Protocol string `json:"protocol,omitempty"` // Explicit protocol (e.g., "openai", "openrouter", "anthropic")
|
||||||
|
|
||||||
// HTTP-based providers
|
// HTTP-based providers
|
||||||
APIBase string `json:"api_base,omitempty"` // API endpoint URL
|
APIBase string `json:"api_base,omitempty"` // API endpoint URL
|
||||||
|
|
|
||||||
|
|
@ -36,7 +36,7 @@ type (
|
||||||
ReasoningDetail = protocoltypes.ReasoningDetail
|
ReasoningDetail = protocoltypes.ReasoningDetail
|
||||||
)
|
)
|
||||||
|
|
||||||
const DefaultRequestTimeout = 120 * time.Second
|
const DefaultRequestTimeout = 30 * time.Second
|
||||||
|
|
||||||
// NewHTTPClient creates an *http.Client with an optional proxy and the default timeout.
|
// NewHTTPClient creates an *http.Client with an optional proxy and the default timeout.
|
||||||
func NewHTTPClient(proxy string) *http.Client {
|
func NewHTTPClient(proxy string) *http.Client {
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,10 @@
|
||||||
package providers
|
package providers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"math"
|
"math"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
@ -16,6 +19,7 @@ type CooldownTracker struct {
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
entries map[string]*cooldownEntry
|
entries map[string]*cooldownEntry
|
||||||
failureWindow time.Duration
|
failureWindow time.Duration
|
||||||
|
storagePath string
|
||||||
nowFunc func() time.Time // for testing
|
nowFunc func() time.Time // for testing
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -29,20 +33,23 @@ type cooldownEntry struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewCooldownTracker creates a tracker with default 24h failure window.
|
// NewCooldownTracker creates a tracker with default 24h failure window.
|
||||||
func NewCooldownTracker() *CooldownTracker {
|
func NewCooldownTracker(storagePath string) *CooldownTracker {
|
||||||
return &CooldownTracker{
|
ct := &CooldownTracker{
|
||||||
entries: make(map[string]*cooldownEntry),
|
entries: make(map[string]*cooldownEntry),
|
||||||
failureWindow: defaultFailureWindow,
|
failureWindow: defaultFailureWindow,
|
||||||
|
storagePath: storagePath,
|
||||||
nowFunc: time.Now,
|
nowFunc: time.Now,
|
||||||
}
|
}
|
||||||
|
if storagePath != "" {
|
||||||
|
ct.Load()
|
||||||
|
}
|
||||||
|
return ct
|
||||||
}
|
}
|
||||||
|
|
||||||
// MarkFailure records a failure for a provider and sets appropriate cooldown.
|
// MarkFailure records a failure for a provider and sets appropriate cooldown.
|
||||||
// Resets error counts if last failure was more than failureWindow ago.
|
// Resets error counts if last failure was more than failureWindow ago.
|
||||||
func (ct *CooldownTracker) MarkFailure(provider string, reason FailoverReason) {
|
func (ct *CooldownTracker) MarkFailure(provider string, reason FailoverReason) {
|
||||||
ct.mu.Lock()
|
ct.mu.Lock()
|
||||||
defer ct.mu.Unlock()
|
|
||||||
|
|
||||||
now := ct.nowFunc()
|
now := ct.nowFunc()
|
||||||
entry := ct.getOrCreate(provider)
|
entry := ct.getOrCreate(provider)
|
||||||
|
|
||||||
|
|
@ -53,6 +60,9 @@ func (ct *CooldownTracker) MarkFailure(provider string, reason FailoverReason) {
|
||||||
}
|
}
|
||||||
|
|
||||||
entry.ErrorCount++
|
entry.ErrorCount++
|
||||||
|
if entry.FailureCounts == nil {
|
||||||
|
entry.FailureCounts = make(map[FailoverReason]int)
|
||||||
|
}
|
||||||
entry.FailureCounts[reason]++
|
entry.FailureCounts[reason]++
|
||||||
entry.LastFailure = now
|
entry.LastFailure = now
|
||||||
|
|
||||||
|
|
@ -63,15 +73,20 @@ func (ct *CooldownTracker) MarkFailure(provider string, reason FailoverReason) {
|
||||||
} else {
|
} else {
|
||||||
entry.CooldownEnd = now.Add(calculateStandardCooldown(entry.ErrorCount))
|
entry.CooldownEnd = now.Add(calculateStandardCooldown(entry.ErrorCount))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Capture state for saving outside the lock
|
||||||
|
toSave := ct.copyEntriesLocked()
|
||||||
|
ct.mu.Unlock()
|
||||||
|
|
||||||
|
ct.persist(toSave)
|
||||||
}
|
}
|
||||||
|
|
||||||
// MarkSuccess resets all counters and cooldowns for a provider.
|
// MarkSuccess resets all counters and cooldowns for a provider.
|
||||||
func (ct *CooldownTracker) MarkSuccess(provider string) {
|
func (ct *CooldownTracker) MarkSuccess(provider string) {
|
||||||
ct.mu.Lock()
|
ct.mu.Lock()
|
||||||
defer ct.mu.Unlock()
|
|
||||||
|
|
||||||
entry := ct.entries[provider]
|
entry := ct.entries[provider]
|
||||||
if entry == nil {
|
if entry == nil {
|
||||||
|
ct.mu.Unlock()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -80,6 +95,12 @@ func (ct *CooldownTracker) MarkSuccess(provider string) {
|
||||||
entry.CooldownEnd = time.Time{}
|
entry.CooldownEnd = time.Time{}
|
||||||
entry.DisabledUntil = time.Time{}
|
entry.DisabledUntil = time.Time{}
|
||||||
entry.DisabledReason = ""
|
entry.DisabledReason = ""
|
||||||
|
|
||||||
|
// Capture state for saving outside the lock
|
||||||
|
toSave := ct.copyEntriesLocked()
|
||||||
|
ct.mu.Unlock()
|
||||||
|
|
||||||
|
ct.persist(toSave)
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsAvailable returns true if the provider is not in cooldown or disabled.
|
// IsAvailable returns true if the provider is not in cooldown or disabled.
|
||||||
|
|
@ -162,6 +183,93 @@ func (ct *CooldownTracker) FailureCount(provider string, reason FailoverReason)
|
||||||
return entry.FailureCounts[reason]
|
return entry.FailureCounts[reason]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Load reads cooldown state from disk.
|
||||||
|
func (ct *CooldownTracker) Load() {
|
||||||
|
if ct.storagePath == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(ct.storagePath)
|
||||||
|
if err != nil {
|
||||||
|
return // ignore missing file
|
||||||
|
}
|
||||||
|
|
||||||
|
var loaded map[string]*cooldownEntry
|
||||||
|
if err := json.Unmarshal(data, &loaded); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
ct.mu.Lock()
|
||||||
|
defer ct.mu.Unlock()
|
||||||
|
|
||||||
|
now := ct.nowFunc()
|
||||||
|
for k, v := range loaded {
|
||||||
|
// Only load entries that aren't fully expired yet
|
||||||
|
if (!v.CooldownEnd.IsZero() && now.Before(v.CooldownEnd)) ||
|
||||||
|
(!v.DisabledUntil.IsZero() && now.Before(v.DisabledUntil)) ||
|
||||||
|
(!v.LastFailure.IsZero() && now.Sub(v.LastFailure) < ct.failureWindow) {
|
||||||
|
if v.FailureCounts == nil {
|
||||||
|
v.FailureCounts = make(map[FailoverReason]int)
|
||||||
|
}
|
||||||
|
ct.entries[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save writes cooldown state to disk.
|
||||||
|
func (ct *CooldownTracker) Save() {
|
||||||
|
ct.mu.RLock()
|
||||||
|
toSave := ct.copyEntriesLocked()
|
||||||
|
ct.mu.RUnlock()
|
||||||
|
|
||||||
|
ct.persist(toSave)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ct *CooldownTracker) copyEntriesLocked() map[string]*cooldownEntry {
|
||||||
|
toSave := make(map[string]*cooldownEntry)
|
||||||
|
now := ct.nowFunc()
|
||||||
|
for k, v := range ct.entries {
|
||||||
|
if v == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Only save entries that are still relevant
|
||||||
|
if (!v.CooldownEnd.IsZero() && now.Before(v.CooldownEnd)) ||
|
||||||
|
(!v.DisabledUntil.IsZero() && now.Before(v.DisabledUntil)) ||
|
||||||
|
(!v.LastFailure.IsZero() && now.Sub(v.LastFailure) < ct.failureWindow) {
|
||||||
|
|
||||||
|
// Deep copy the entry to avoid data races when serializing outside the lock
|
||||||
|
copy := *v
|
||||||
|
if v.FailureCounts != nil {
|
||||||
|
copy.FailureCounts = make(map[FailoverReason]int)
|
||||||
|
for r, c := range v.FailureCounts {
|
||||||
|
copy.FailureCounts[r] = c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
toSave[k] = ©
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return toSave
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ct *CooldownTracker) persist(toSave map[string]*cooldownEntry) {
|
||||||
|
if ct.storagePath == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(toSave) == 0 {
|
||||||
|
os.Remove(ct.storagePath) // cleanup if empty
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := json.MarshalIndent(toSave, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
os.MkdirAll(filepath.Dir(ct.storagePath), 0755)
|
||||||
|
os.WriteFile(ct.storagePath, data, 0644)
|
||||||
|
}
|
||||||
|
|
||||||
func (ct *CooldownTracker) getOrCreate(provider string) *cooldownEntry {
|
func (ct *CooldownTracker) getOrCreate(provider string) *cooldownEntry {
|
||||||
entry := ct.entries[provider]
|
entry := ct.entries[provider]
|
||||||
if entry == nil {
|
if entry == nil {
|
||||||
|
|
|
||||||
|
|
@ -8,13 +8,13 @@ import (
|
||||||
|
|
||||||
func newTestTracker(now time.Time) (*CooldownTracker, *time.Time) {
|
func newTestTracker(now time.Time) (*CooldownTracker, *time.Time) {
|
||||||
current := now
|
current := now
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
ct.nowFunc = func() time.Time { return current }
|
ct.nowFunc = func() time.Time { return current }
|
||||||
return ct, ¤t
|
return ct, ¤t
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCooldown_InitiallyAvailable(t *testing.T) {
|
func TestCooldown_InitiallyAvailable(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
if !ct.IsAvailable("openai") {
|
if !ct.IsAvailable("openai") {
|
||||||
t.Error("new provider should be available")
|
t.Error("new provider should be available")
|
||||||
}
|
}
|
||||||
|
|
@ -110,7 +110,7 @@ func TestCooldown_BillingCap(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCooldown_SuccessReset(t *testing.T) {
|
func TestCooldown_SuccessReset(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
|
|
||||||
ct.MarkFailure("openai", FailoverRateLimit)
|
ct.MarkFailure("openai", FailoverRateLimit)
|
||||||
ct.MarkFailure("openai", FailoverBilling)
|
ct.MarkFailure("openai", FailoverBilling)
|
||||||
|
|
@ -157,7 +157,7 @@ func TestCooldown_FailureWindowReset(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCooldown_PerReasonTracking(t *testing.T) {
|
func TestCooldown_PerReasonTracking(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
|
|
||||||
ct.MarkFailure("openai", FailoverRateLimit)
|
ct.MarkFailure("openai", FailoverRateLimit)
|
||||||
ct.MarkFailure("openai", FailoverRateLimit)
|
ct.MarkFailure("openai", FailoverRateLimit)
|
||||||
|
|
@ -218,7 +218,7 @@ func TestCooldown_CooldownRemaining(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCooldown_SuccessOnUnknownProvider(t *testing.T) {
|
func TestCooldown_SuccessOnUnknownProvider(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
// Should not panic
|
// Should not panic
|
||||||
ct.MarkSuccess("nonexistent")
|
ct.MarkSuccess("nonexistent")
|
||||||
if !ct.IsAvailable("nonexistent") {
|
if !ct.IsAvailable("nonexistent") {
|
||||||
|
|
@ -227,7 +227,7 @@ func TestCooldown_SuccessOnUnknownProvider(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCooldown_ConcurrentAccess(t *testing.T) {
|
func TestCooldown_ConcurrentAccess(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
for range 100 {
|
for range 100 {
|
||||||
|
|
@ -251,7 +251,7 @@ func TestCooldown_ConcurrentAccess(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCooldown_MultipleProviders(t *testing.T) {
|
func TestCooldown_MultipleProviders(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
|
|
||||||
ct.MarkFailure("openai", FailoverRateLimit)
|
ct.MarkFailure("openai", FailoverRateLimit)
|
||||||
ct.MarkFailure("anthropic", FailoverBilling)
|
ct.MarkFailure("anthropic", FailoverBilling)
|
||||||
|
|
|
||||||
|
|
@ -109,6 +109,9 @@ func ResolveAPIBase(cfg *config.ModelConfig) string {
|
||||||
return strings.TrimRight(apiBase, "/")
|
return strings.TrimRight(apiBase, "/")
|
||||||
}
|
}
|
||||||
protocol, _ := ExtractProtocol(cfg.Model)
|
protocol, _ := ExtractProtocol(cfg.Model)
|
||||||
|
if cfg.Protocol != "" {
|
||||||
|
protocol = cfg.Protocol
|
||||||
|
}
|
||||||
return strings.TrimRight(getDefaultAPIBase(protocol), "/")
|
return strings.TrimRight(getDefaultAPIBase(protocol), "/")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -128,6 +131,10 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err
|
||||||
}
|
}
|
||||||
|
|
||||||
protocol, modelID := ExtractProtocol(cfg.Model)
|
protocol, modelID := ExtractProtocol(cfg.Model)
|
||||||
|
if cfg.Protocol != "" {
|
||||||
|
protocol = cfg.Protocol
|
||||||
|
modelID = cfg.Model
|
||||||
|
}
|
||||||
|
|
||||||
userAgent := cfg.UserAgent
|
userAgent := cfg.UserAgent
|
||||||
if userAgent == "" {
|
if userAgent == "" {
|
||||||
|
|
@ -224,13 +231,13 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err
|
||||||
"qwen-us", "dashscope-us", "mistral", "avian", "longcat", "modelscope", "novita",
|
"qwen-us", "dashscope-us", "mistral", "avian", "longcat", "modelscope", "novita",
|
||||||
"coding-plan", "alibaba-coding", "qwen-coding", "mimo":
|
"coding-plan", "alibaba-coding", "qwen-coding", "mimo":
|
||||||
// All other OpenAI-compatible HTTP providers
|
// All other OpenAI-compatible HTTP providers
|
||||||
if cfg.APIKey() == "" && cfg.APIBase == "" && !isEmptyAPIKeyAllowed(protocol) {
|
|
||||||
return nil, "", fmt.Errorf("api_key or api_base is required for HTTP-based protocol %q", protocol)
|
|
||||||
}
|
|
||||||
apiBase := cfg.APIBase
|
apiBase := cfg.APIBase
|
||||||
if apiBase == "" {
|
if apiBase == "" {
|
||||||
apiBase = getDefaultAPIBase(protocol)
|
apiBase = getDefaultAPIBase(protocol)
|
||||||
}
|
}
|
||||||
|
if cfg.APIKey() == "" && apiBase == "" && !isEmptyAPIKeyAllowed(protocol) {
|
||||||
|
return nil, "", fmt.Errorf("api_key or api_base is required for HTTP-based protocol %q", protocol)
|
||||||
|
}
|
||||||
return NewHTTPProviderWithMaxTokensFieldAndRequestTimeout(
|
return NewHTTPProviderWithMaxTokensFieldAndRequestTimeout(
|
||||||
cfg.APIKey(),
|
cfg.APIKey(),
|
||||||
apiBase,
|
apiBase,
|
||||||
|
|
|
||||||
|
|
@ -32,10 +32,11 @@ func (c FallbackCandidate) StableKey() string {
|
||||||
|
|
||||||
// FallbackResult contains the successful response and metadata about all attempts.
|
// FallbackResult contains the successful response and metadata about all attempts.
|
||||||
type FallbackResult struct {
|
type FallbackResult struct {
|
||||||
Response *LLMResponse
|
Response *LLMResponse
|
||||||
Provider string
|
Provider string
|
||||||
Model string
|
Model string
|
||||||
Attempts []FallbackAttempt
|
IdentityKey string
|
||||||
|
Attempts []FallbackAttempt
|
||||||
}
|
}
|
||||||
|
|
||||||
// FallbackAttempt records one attempt in the fallback chain.
|
// FallbackAttempt records one attempt in the fallback chain.
|
||||||
|
|
@ -187,6 +188,7 @@ func (fc *FallbackChain) Execute(
|
||||||
result.Response = resp
|
result.Response = resp
|
||||||
result.Provider = candidate.Provider
|
result.Provider = candidate.Provider
|
||||||
result.Model = candidate.Model
|
result.Model = candidate.Model
|
||||||
|
result.IdentityKey = candidate.IdentityKey
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -305,6 +307,7 @@ func (fc *FallbackChain) ExecuteImage(
|
||||||
result.Response = resp
|
result.Response = resp
|
||||||
result.Provider = candidate.Provider
|
result.Provider = candidate.Provider
|
||||||
result.Model = candidate.Model
|
result.Model = candidate.Model
|
||||||
|
result.IdentityKey = candidate.IdentityKey
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -24,7 +24,7 @@ func TestMultiKeyFailover(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create fallback chain
|
// Create fallback chain
|
||||||
cooldown := NewCooldownTracker()
|
cooldown := NewCooldownTracker("")
|
||||||
chain := NewFallbackChain(cooldown, nil)
|
chain := NewFallbackChain(cooldown, nil)
|
||||||
|
|
||||||
// Mock run function: first call fails with 429, second succeeds
|
// Mock run function: first call fails with 429, second succeeds
|
||||||
|
|
@ -81,7 +81,7 @@ func TestMultiKeyFailoverAllFail(t *testing.T) {
|
||||||
|
|
||||||
candidates := ResolveCandidates(cfg, "zhipu")
|
candidates := ResolveCandidates(cfg, "zhipu")
|
||||||
|
|
||||||
cooldown := NewCooldownTracker()
|
cooldown := NewCooldownTracker("")
|
||||||
chain := NewFallbackChain(cooldown, nil)
|
chain := NewFallbackChain(cooldown, nil)
|
||||||
|
|
||||||
// Mock run function: all calls fail with rate limit
|
// Mock run function: all calls fail with rate limit
|
||||||
|
|
@ -126,7 +126,7 @@ func TestMultiKeyFailoverCooldown(t *testing.T) {
|
||||||
|
|
||||||
candidates := ResolveCandidates(cfg, "zhipu")
|
candidates := ResolveCandidates(cfg, "zhipu")
|
||||||
|
|
||||||
cooldown := NewCooldownTracker()
|
cooldown := NewCooldownTracker("")
|
||||||
chain := NewFallbackChain(cooldown, nil)
|
chain := NewFallbackChain(cooldown, nil)
|
||||||
|
|
||||||
// Put the first model in cooldown (using ModelKey now, not just provider)
|
// Put the first model in cooldown (using ModelKey now, not just provider)
|
||||||
|
|
@ -182,7 +182,7 @@ func TestMultiKeyFailoverWithFormatError(t *testing.T) {
|
||||||
|
|
||||||
candidates := ResolveCandidates(cfg, "zhipu")
|
candidates := ResolveCandidates(cfg, "zhipu")
|
||||||
|
|
||||||
cooldown := NewCooldownTracker()
|
cooldown := NewCooldownTracker("")
|
||||||
chain := NewFallbackChain(cooldown, nil)
|
chain := NewFallbackChain(cooldown, nil)
|
||||||
|
|
||||||
// Mock run function: first call fails with format error (bad request)
|
// Mock run function: first call fails with format error (bad request)
|
||||||
|
|
@ -262,7 +262,7 @@ func TestMultiKeyWithModelFallback(t *testing.T) {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
cooldown := NewCooldownTracker()
|
cooldown := NewCooldownTracker("")
|
||||||
chain := NewFallbackChain(cooldown, nil)
|
chain := NewFallbackChain(cooldown, nil)
|
||||||
|
|
||||||
// Mock run function: first two fail, third succeeds (model fallback)
|
// Mock run function: first two fail, third succeeds (model fallback)
|
||||||
|
|
@ -336,7 +336,7 @@ func TestMultiKeyFailoverMixedErrors(t *testing.T) {
|
||||||
|
|
||||||
candidates := ResolveCandidates(cfg, "zhipu")
|
candidates := ResolveCandidates(cfg, "zhipu")
|
||||||
|
|
||||||
cooldown := NewCooldownTracker()
|
cooldown := NewCooldownTracker("")
|
||||||
chain := NewFallbackChain(cooldown, nil)
|
chain := NewFallbackChain(cooldown, nil)
|
||||||
|
|
||||||
// Mock run function: different errors for each key
|
// Mock run function: different errors for each key
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ func successRun(content string) func(ctx context.Context, provider, model string
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFallback_SingleCandidate_Success(t *testing.T) {
|
func TestFallback_SingleCandidate_Success(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
fc := NewFallbackChain(ct, nil)
|
fc := NewFallbackChain(ct, nil)
|
||||||
|
|
||||||
candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4")}
|
candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4")}
|
||||||
|
|
@ -35,7 +35,7 @@ func TestFallback_SingleCandidate_Success(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFallback_SecondCandidateSuccess(t *testing.T) {
|
func TestFallback_SecondCandidateSuccess(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
fc := NewFallbackChain(ct, nil)
|
fc := NewFallbackChain(ct, nil)
|
||||||
|
|
||||||
candidates := []FallbackCandidate{
|
candidates := []FallbackCandidate{
|
||||||
|
|
@ -68,7 +68,7 @@ func TestFallback_SecondCandidateSuccess(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFallback_AllFail(t *testing.T) {
|
func TestFallback_AllFail(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
fc := NewFallbackChain(ct, nil)
|
fc := NewFallbackChain(ct, nil)
|
||||||
|
|
||||||
candidates := []FallbackCandidate{
|
candidates := []FallbackCandidate{
|
||||||
|
|
@ -95,7 +95,7 @@ func TestFallback_AllFail(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFallback_ContextCanceled(t *testing.T) {
|
func TestFallback_ContextCanceled(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
fc := NewFallbackChain(ct, nil)
|
fc := NewFallbackChain(ct, nil)
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
|
@ -122,7 +122,7 @@ func TestFallback_ContextCanceled(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFallback_NonRetriableError(t *testing.T) {
|
func TestFallback_NonRetriableError(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
fc := NewFallbackChain(ct, nil)
|
fc := NewFallbackChain(ct, nil)
|
||||||
|
|
||||||
candidates := []FallbackCandidate{
|
candidates := []FallbackCandidate{
|
||||||
|
|
@ -192,7 +192,7 @@ func TestFallback_CooldownSkip(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFallback_AllInCooldown(t *testing.T) {
|
func TestFallback_AllInCooldown(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
fc := NewFallbackChain(ct, nil)
|
fc := NewFallbackChain(ct, nil)
|
||||||
|
|
||||||
// Put all models in cooldown (using ModelKey now)
|
// Put all models in cooldown (using ModelKey now)
|
||||||
|
|
@ -220,7 +220,7 @@ func TestFallback_AllInCooldown(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFallback_NoCandidates(t *testing.T) {
|
func TestFallback_NoCandidates(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
fc := NewFallbackChain(ct, nil)
|
fc := NewFallbackChain(ct, nil)
|
||||||
|
|
||||||
_, err := fc.Execute(context.Background(), nil, successRun("ok"))
|
_, err := fc.Execute(context.Background(), nil, successRun("ok"))
|
||||||
|
|
@ -231,7 +231,7 @@ func TestFallback_NoCandidates(t *testing.T) {
|
||||||
|
|
||||||
func TestFallback_EmptyFallbacks(t *testing.T) {
|
func TestFallback_EmptyFallbacks(t *testing.T) {
|
||||||
// Single primary, no fallbacks: should work like direct call
|
// Single primary, no fallbacks: should work like direct call
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
fc := NewFallbackChain(ct, nil)
|
fc := NewFallbackChain(ct, nil)
|
||||||
|
|
||||||
candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4")}
|
candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4")}
|
||||||
|
|
@ -245,7 +245,7 @@ func TestFallback_EmptyFallbacks(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFallback_UnclassifiedError(t *testing.T) {
|
func TestFallback_UnclassifiedError(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
fc := NewFallbackChain(ct, nil)
|
fc := NewFallbackChain(ct, nil)
|
||||||
|
|
||||||
candidates := []FallbackCandidate{
|
candidates := []FallbackCandidate{
|
||||||
|
|
@ -278,7 +278,7 @@ func assertFallbackErrorFallsBack(
|
||||||
) {
|
) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
fc := NewFallbackChain(ct, nil)
|
fc := NewFallbackChain(ct, nil)
|
||||||
|
|
||||||
candidates := []FallbackCandidate{
|
candidates := []FallbackCandidate{
|
||||||
|
|
@ -338,7 +338,7 @@ func TestFallback_TimeoutErrorFallsBack(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFallback_SuccessResetsCooldown(t *testing.T) {
|
func TestFallback_SuccessResetsCooldown(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
fc := NewFallbackChain(ct, nil)
|
fc := NewFallbackChain(ct, nil)
|
||||||
|
|
||||||
candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4")}
|
candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4")}
|
||||||
|
|
@ -375,7 +375,7 @@ func assertLocalRateLimitSkipsToHealthyFallback(
|
||||||
) {
|
) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
rl := NewRateLimiterRegistry()
|
rl := NewRateLimiterRegistry()
|
||||||
rl.Register(primaryKey, 1)
|
rl.Register(primaryKey, 1)
|
||||||
if err := rl.Wait(context.Background(), primaryKey); err != nil {
|
if err := rl.Wait(context.Background(), primaryKey); err != nil {
|
||||||
|
|
@ -432,7 +432,7 @@ func TestFallback_LocalRateLimitSkipsToHealthyFallback(t *testing.T) {
|
||||||
// --- Image Fallback Tests ---
|
// --- Image Fallback Tests ---
|
||||||
|
|
||||||
func TestImageFallback_Success(t *testing.T) {
|
func TestImageFallback_Success(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
fc := NewFallbackChain(ct, nil)
|
fc := NewFallbackChain(ct, nil)
|
||||||
|
|
||||||
candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4o")}
|
candidates := []FallbackCandidate{makeCandidate("openai", "gpt-4o")}
|
||||||
|
|
@ -446,7 +446,7 @@ func TestImageFallback_Success(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestImageFallback_DimensionError(t *testing.T) {
|
func TestImageFallback_DimensionError(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
fc := NewFallbackChain(ct, nil)
|
fc := NewFallbackChain(ct, nil)
|
||||||
|
|
||||||
candidates := []FallbackCandidate{
|
candidates := []FallbackCandidate{
|
||||||
|
|
@ -470,7 +470,7 @@ func TestImageFallback_DimensionError(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestImageFallback_SizeError(t *testing.T) {
|
func TestImageFallback_SizeError(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
fc := NewFallbackChain(ct, nil)
|
fc := NewFallbackChain(ct, nil)
|
||||||
|
|
||||||
candidates := []FallbackCandidate{
|
candidates := []FallbackCandidate{
|
||||||
|
|
@ -494,7 +494,7 @@ func TestImageFallback_SizeError(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestImageFallback_RetryOnOtherErrors(t *testing.T) {
|
func TestImageFallback_RetryOnOtherErrors(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
fc := NewFallbackChain(ct, nil)
|
fc := NewFallbackChain(ct, nil)
|
||||||
|
|
||||||
candidates := []FallbackCandidate{
|
candidates := []FallbackCandidate{
|
||||||
|
|
@ -540,7 +540,7 @@ func TestImageFallback_LocalRateLimitSkipsToHealthyFallback(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestImageFallback_NoCandidates(t *testing.T) {
|
func TestImageFallback_NoCandidates(t *testing.T) {
|
||||||
ct := NewCooldownTracker()
|
ct := NewCooldownTracker("")
|
||||||
fc := NewFallbackChain(ct, nil)
|
fc := NewFallbackChain(ct, nil)
|
||||||
|
|
||||||
_, err := fc.ExecuteImage(context.Background(), nil, successRun("ok"))
|
_, err := fc.ExecuteImage(context.Background(), nil, successRun("ok"))
|
||||||
|
|
|
||||||
7
pkg/seahorse/.omc/state/last-tool-error.json
Normal file
7
pkg/seahorse/.omc/state/last-tool-error.json
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
{
|
||||||
|
"tool_name": "Bash",
|
||||||
|
"tool_input_preview": "{\"command\":\"cd /home/yliu/repos/picoclaw && make lint 2>&1\",\"timeout\":120000}",
|
||||||
|
"error": "Exit code 2\npkg/agent/context_seahorse_test.go:1027:1: File is not properly formatted (gci)\n\t\t\tEarliestAt: &now,\n^\n1 issues:\n* gci: 1\nmake: *** [Makefile:264: lint] Error 1",
|
||||||
|
"timestamp": "2026-04-04T02:38:32.067Z",
|
||||||
|
"retry_count": 6
|
||||||
|
}
|
||||||
|
|
@ -218,7 +218,8 @@ func (t *FreeRideTool) handleAuto(ctx context.Context, limit int) *ToolResult {
|
||||||
if !modelExists(cfgObj, modelName) {
|
if !modelExists(cfgObj, modelName) {
|
||||||
mc := &config.ModelConfig{
|
mc := &config.ModelConfig{
|
||||||
ModelName: modelName,
|
ModelName: modelName,
|
||||||
Model: "openrouter/" + m.ID,
|
Model: m.ID,
|
||||||
|
Protocol: "openrouter",
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
}
|
}
|
||||||
mc.SetAPIKey("env://OPENROUTER_API_KEY")
|
mc.SetAPIKey("env://OPENROUTER_API_KEY")
|
||||||
|
|
|
||||||
28
scratch/check_paths.go
Normal file
28
scratch/check_paths.go
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"github.com/sipeed/picoclaw/pkg/agent"
|
||||||
|
"github.com/sipeed/picoclaw/pkg/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
cfg, err := config.LoadConfig(os.ExpandEnv("$HOME/.picoclaw/config.json"))
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
registry := agent.NewAgentRegistry(cfg, nil)
|
||||||
|
defaultAgent := registry.GetDefaultAgent()
|
||||||
|
if defaultAgent == nil {
|
||||||
|
fmt.Println("No default agent")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cooldownPath := filepath.Join(filepath.Dir(filepath.Clean(defaultAgent.Workspace)), "cooldowns.json")
|
||||||
|
fmt.Printf("Workspace: %s\n", defaultAgent.Workspace)
|
||||||
|
fmt.Printf("Cooldown Path: %s\n", cooldownPath)
|
||||||
|
}
|
||||||
Loading…
Add table
Reference in a new issue