fix(agent): retry transient provider failures
This commit is contained in:
parent
8a188cf7fc
commit
382975a7af
2 changed files with 56 additions and 4 deletions
|
|
@ -1128,8 +1128,9 @@ func (al *AgentLoop) runLLMIteration(
|
|||
return agent.Provider.Chat(ctx, messages, providerToolDefs, activeModel, llmOpts)
|
||||
}
|
||||
|
||||
// Retry loop for context/token errors
|
||||
// Retry loop for context/token errors and transient single-provider failures.
|
||||
maxRetries := 2
|
||||
singleCandidate := len(activeCandidates) == 1
|
||||
for retry := 0; retry <= maxRetries; retry++ {
|
||||
response, err = callLLM()
|
||||
if err == nil {
|
||||
|
|
@ -1137,6 +1138,10 @@ func (al *AgentLoop) runLLMIteration(
|
|||
}
|
||||
|
||||
errMsg := strings.ToLower(err.Error())
|
||||
var classifiedErr *providers.FailoverError
|
||||
if singleCandidate {
|
||||
classifiedErr = providers.ClassifyError(err, activeCandidates[0].Provider, activeModel)
|
||||
}
|
||||
|
||||
// Check if this is a network/HTTP timeout — not a context window error.
|
||||
isTimeoutError := errors.Is(err, context.DeadlineExceeded) ||
|
||||
|
|
@ -1144,9 +1149,10 @@ func (al *AgentLoop) runLLMIteration(
|
|||
strings.Contains(errMsg, "client.timeout") ||
|
||||
strings.Contains(errMsg, "timed out") ||
|
||||
strings.Contains(errMsg, "timeout exceeded")
|
||||
isTransientProviderError := classifiedErr != nil && classifiedErr.Reason == providers.FailoverTimeout
|
||||
|
||||
// Detect real context window / token limit errors, excluding network timeouts.
|
||||
isContextError := !isTimeoutError && (strings.Contains(errMsg, "context_length_exceeded") ||
|
||||
isContextError := !isTimeoutError && !isTransientProviderError && (strings.Contains(errMsg, "context_length_exceeded") ||
|
||||
strings.Contains(errMsg, "context window") ||
|
||||
strings.Contains(errMsg, "maximum context length") ||
|
||||
strings.Contains(errMsg, "token limit") ||
|
||||
|
|
@ -1156,9 +1162,9 @@ func (al *AgentLoop) runLLMIteration(
|
|||
strings.Contains(errMsg, "prompt is too long") ||
|
||||
strings.Contains(errMsg, "request too large"))
|
||||
|
||||
if isTimeoutError && retry < maxRetries {
|
||||
if (isTimeoutError || isTransientProviderError) && retry < maxRetries {
|
||||
backoff := time.Duration(retry+1) * 5 * time.Second
|
||||
logger.WarnCF("agent", "Timeout error, retrying after backoff", map[string]any{
|
||||
logger.WarnCF("agent", "Transient provider error, retrying after backoff", map[string]any{
|
||||
"error": err.Error(),
|
||||
"retry": retry,
|
||||
"backoff": backoff.String(),
|
||||
|
|
|
|||
|
|
@ -845,6 +845,52 @@ func TestAgentLoop_ContextExhaustionRetry(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestAgentLoop_TransientProviderRetry(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "agent-test-*")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
cfg := &config.Config{
|
||||
Agents: config.AgentsConfig{
|
||||
Defaults: config.AgentDefaults{
|
||||
Workspace: tmpDir,
|
||||
Model: "test-model",
|
||||
MaxTokens: 4096,
|
||||
MaxToolIterations: 10,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
msgBus := bus.NewMessageBus()
|
||||
provider := &failFirstMockProvider{
|
||||
failures: 1,
|
||||
failError: fmt.Errorf("API error: status: 500 temporary upstream failure"),
|
||||
successResp: "Recovered after transient provider failure",
|
||||
}
|
||||
|
||||
al := NewAgentLoop(cfg, msgBus, provider)
|
||||
|
||||
response, err := al.ProcessDirectWithChannel(
|
||||
context.Background(),
|
||||
"retry this request",
|
||||
"test-session-transient",
|
||||
"cli",
|
||||
"direct",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Expected success after transient retry, got error: %v", err)
|
||||
}
|
||||
|
||||
if response != "Recovered after transient provider failure" {
|
||||
t.Fatalf("response = %q, want %q", response, "Recovered after transient provider failure")
|
||||
}
|
||||
if provider.currentCall != 2 {
|
||||
t.Fatalf("expected 2 calls (1 fail + 1 success), got %d", provider.currentCall)
|
||||
}
|
||||
}
|
||||
|
||||
// TestProcessDirectWithChannel_TriggersMCPInitialization verifies that
|
||||
// ProcessDirectWithChannel triggers MCP initialization when MCP is enabled.
|
||||
// Note: Manager is only initialized when at least one MCP server is configured
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue