fix(agent): retry transient provider failures
This commit is contained in:
parent
8a188cf7fc
commit
382975a7af
2 changed files with 56 additions and 4 deletions
|
|
@ -1128,8 +1128,9 @@ func (al *AgentLoop) runLLMIteration(
|
||||||
return agent.Provider.Chat(ctx, messages, providerToolDefs, activeModel, llmOpts)
|
return agent.Provider.Chat(ctx, messages, providerToolDefs, activeModel, llmOpts)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Retry loop for context/token errors
|
// Retry loop for context/token errors and transient single-provider failures.
|
||||||
maxRetries := 2
|
maxRetries := 2
|
||||||
|
singleCandidate := len(activeCandidates) == 1
|
||||||
for retry := 0; retry <= maxRetries; retry++ {
|
for retry := 0; retry <= maxRetries; retry++ {
|
||||||
response, err = callLLM()
|
response, err = callLLM()
|
||||||
if err == nil {
|
if err == nil {
|
||||||
|
|
@ -1137,6 +1138,10 @@ func (al *AgentLoop) runLLMIteration(
|
||||||
}
|
}
|
||||||
|
|
||||||
errMsg := strings.ToLower(err.Error())
|
errMsg := strings.ToLower(err.Error())
|
||||||
|
var classifiedErr *providers.FailoverError
|
||||||
|
if singleCandidate {
|
||||||
|
classifiedErr = providers.ClassifyError(err, activeCandidates[0].Provider, activeModel)
|
||||||
|
}
|
||||||
|
|
||||||
// Check if this is a network/HTTP timeout — not a context window error.
|
// Check if this is a network/HTTP timeout — not a context window error.
|
||||||
isTimeoutError := errors.Is(err, context.DeadlineExceeded) ||
|
isTimeoutError := errors.Is(err, context.DeadlineExceeded) ||
|
||||||
|
|
@ -1144,9 +1149,10 @@ func (al *AgentLoop) runLLMIteration(
|
||||||
strings.Contains(errMsg, "client.timeout") ||
|
strings.Contains(errMsg, "client.timeout") ||
|
||||||
strings.Contains(errMsg, "timed out") ||
|
strings.Contains(errMsg, "timed out") ||
|
||||||
strings.Contains(errMsg, "timeout exceeded")
|
strings.Contains(errMsg, "timeout exceeded")
|
||||||
|
isTransientProviderError := classifiedErr != nil && classifiedErr.Reason == providers.FailoverTimeout
|
||||||
|
|
||||||
// Detect real context window / token limit errors, excluding network timeouts.
|
// Detect real context window / token limit errors, excluding network timeouts.
|
||||||
isContextError := !isTimeoutError && (strings.Contains(errMsg, "context_length_exceeded") ||
|
isContextError := !isTimeoutError && !isTransientProviderError && (strings.Contains(errMsg, "context_length_exceeded") ||
|
||||||
strings.Contains(errMsg, "context window") ||
|
strings.Contains(errMsg, "context window") ||
|
||||||
strings.Contains(errMsg, "maximum context length") ||
|
strings.Contains(errMsg, "maximum context length") ||
|
||||||
strings.Contains(errMsg, "token limit") ||
|
strings.Contains(errMsg, "token limit") ||
|
||||||
|
|
@ -1156,9 +1162,9 @@ func (al *AgentLoop) runLLMIteration(
|
||||||
strings.Contains(errMsg, "prompt is too long") ||
|
strings.Contains(errMsg, "prompt is too long") ||
|
||||||
strings.Contains(errMsg, "request too large"))
|
strings.Contains(errMsg, "request too large"))
|
||||||
|
|
||||||
if isTimeoutError && retry < maxRetries {
|
if (isTimeoutError || isTransientProviderError) && retry < maxRetries {
|
||||||
backoff := time.Duration(retry+1) * 5 * time.Second
|
backoff := time.Duration(retry+1) * 5 * time.Second
|
||||||
logger.WarnCF("agent", "Timeout error, retrying after backoff", map[string]any{
|
logger.WarnCF("agent", "Transient provider error, retrying after backoff", map[string]any{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
"retry": retry,
|
"retry": retry,
|
||||||
"backoff": backoff.String(),
|
"backoff": backoff.String(),
|
||||||
|
|
|
||||||
|
|
@ -845,6 +845,52 @@ func TestAgentLoop_ContextExhaustionRetry(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAgentLoop_TransientProviderRetry(t *testing.T) {
|
||||||
|
tmpDir, err := os.MkdirTemp("", "agent-test-*")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create temp dir: %v", err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tmpDir)
|
||||||
|
|
||||||
|
cfg := &config.Config{
|
||||||
|
Agents: config.AgentsConfig{
|
||||||
|
Defaults: config.AgentDefaults{
|
||||||
|
Workspace: tmpDir,
|
||||||
|
Model: "test-model",
|
||||||
|
MaxTokens: 4096,
|
||||||
|
MaxToolIterations: 10,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
msgBus := bus.NewMessageBus()
|
||||||
|
provider := &failFirstMockProvider{
|
||||||
|
failures: 1,
|
||||||
|
failError: fmt.Errorf("API error: status: 500 temporary upstream failure"),
|
||||||
|
successResp: "Recovered after transient provider failure",
|
||||||
|
}
|
||||||
|
|
||||||
|
al := NewAgentLoop(cfg, msgBus, provider)
|
||||||
|
|
||||||
|
response, err := al.ProcessDirectWithChannel(
|
||||||
|
context.Background(),
|
||||||
|
"retry this request",
|
||||||
|
"test-session-transient",
|
||||||
|
"cli",
|
||||||
|
"direct",
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Expected success after transient retry, got error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if response != "Recovered after transient provider failure" {
|
||||||
|
t.Fatalf("response = %q, want %q", response, "Recovered after transient provider failure")
|
||||||
|
}
|
||||||
|
if provider.currentCall != 2 {
|
||||||
|
t.Fatalf("expected 2 calls (1 fail + 1 success), got %d", provider.currentCall)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestProcessDirectWithChannel_TriggersMCPInitialization verifies that
|
// TestProcessDirectWithChannel_TriggersMCPInitialization verifies that
|
||||||
// ProcessDirectWithChannel triggers MCP initialization when MCP is enabled.
|
// ProcessDirectWithChannel triggers MCP initialization when MCP is enabled.
|
||||||
// Note: Manager is only initialized when at least one MCP server is configured
|
// Note: Manager is only initialized when at least one MCP server is configured
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue