From 5f53e4026038fd213b8328535f2e79593f71db9f Mon Sep 17 00:00:00 2001 From: muava12 Date: Tue, 24 Feb 2026 23:56:19 +0800 Subject: [PATCH] fix(providers): handle model-invalid 400 errors with fallback and warning Add FailoverModelInvalid reason for 400 errors where the model ID is invalid or unavailable (e.g. 'not a valid model ID'). Previously, all 400 errors were classified as FailoverFormat (non-retriable), which aborted the fallback chain immediately instead of trying the next model. Model-invalid patterns are now checked BEFORE HTTP status code classification, allowing fallback to proceed. When fallback succeeds after a model-invalid error, a warning message is sent to the user's channel so they know to fix the config. --- pkg/agent/loop.go | 16 ++++++++ pkg/providers/error_classifier.go | 29 +++++++++++++- pkg/providers/error_classifier_test.go | 54 ++++++++++++++++++++++++++ pkg/providers/types.go | 20 ++++++---- 4 files changed, 111 insertions(+), 8 deletions(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index e80728fbb..095e1012d 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -557,6 +557,22 @@ func (al *AgentLoop) runLLMIteration( logger.InfoCF("agent", fmt.Sprintf("Fallback: succeeded with %s/%s after %d attempts", fbResult.Provider, fbResult.Model, len(fbResult.Attempts)+1), map[string]any{"agent_id": agent.ID, "iteration": iteration}) + + // Send warning for model-invalid errors so user knows to fix config + if !constants.IsInternalChannel(opts.Channel) { + for _, attempt := range fbResult.Attempts { + if failErr, ok := attempt.Error.(*providers.FailoverError); ok && failErr.IsModelInvalid() { + al.bus.PublishOutbound(bus.OutboundMessage{ + Channel: opts.Channel, + ChatID: opts.ChatID, + Content: fmt.Sprintf("⚠️ Model %s/%s is invalid or unavailable: %v\nUsing fallback: %s/%s", + attempt.Provider, attempt.Model, failErr.Wrapped, + fbResult.Provider, fbResult.Model), + }) + break // Only send one warning even if multiple model-invalid errors + } + } + } } return fbResult.Response, nil } diff --git a/pkg/providers/error_classifier.go b/pkg/providers/error_classifier.go index a0f003006..935bd7e54 100644 --- a/pkg/providers/error_classifier.go +++ b/pkg/providers/error_classifier.go @@ -78,6 +78,21 @@ var ( substr("invalid request format"), } + // Model invalid/not found patterns: these are 400 errors that should be + // retriable (fallback to next model), NOT treated as format errors. + modelInvalidPatterns = []errorPattern{ + substr("not a valid model"), + substr("model not found"), + substr("model_not_found"), + substr("model not available"), + substr("does not exist"), + substr("no such model"), + substr("invalid model"), + rxp(`model.*not.*supported`), + rxp(`model.*is.*unavailable`), + rxp(`model.*is.*deprecated`), + } + imageDimensionPatterns = []errorPattern{ rxp(`image dimensions exceed max`), } @@ -128,7 +143,19 @@ func ClassifyError(err error, provider, model string) *FailoverError { } } - // Try HTTP status code extraction first. + // Model invalid/not found: retriable, should fallback to another model. + // This MUST run before HTTP status classification, because 400 + "not a valid model" + // would otherwise be classified as non-retriable FailoverFormat. + if matchesAny(msg, modelInvalidPatterns) { + return &FailoverError{ + Reason: FailoverModelInvalid, + Provider: provider, + Model: model, + Wrapped: err, + } + } + + // Try HTTP status code extraction. if status := extractHTTPStatus(msg); status > 0 { if reason := classifyByStatus(status); reason != "" { return &FailoverError{ diff --git a/pkg/providers/error_classifier_test.go b/pkg/providers/error_classifier_test.go index 865aea57a..4f472abb2 100644 --- a/pkg/providers/error_classifier_test.go +++ b/pkg/providers/error_classifier_test.go @@ -207,6 +207,59 @@ func TestClassifyError_FormatPatterns(t *testing.T) { } } +func TestClassifyError_ModelInvalidPatterns(t *testing.T) { + patterns := []string{ + "nemotron-3-nano-30b-a3b:free is not a valid model ID", + "model not found", + "model_not_found: the requested model does not exist", + "model not available in this region", + "the model does not exist or you do not have access", + "no such model: gpt-5-turbo", + "invalid model specified", + "model llama-3-8b is not supported", + "model gpt-4o-mini is unavailable", + "model codellama is deprecated", + } + + for _, msg := range patterns { + err := errors.New(msg) + result := ClassifyError(err, "nvidia", "test-model") + if result == nil { + t.Errorf("pattern %q: expected non-nil", msg) + continue + } + if result.Reason != FailoverModelInvalid { + t.Errorf("pattern %q: reason = %q, want model_invalid", msg, result.Reason) + } + if !result.IsRetriable() { + t.Errorf("pattern %q: should be retriable to allow fallback", msg) + } + if !result.IsModelInvalid() { + t.Errorf("pattern %q: should be classified as model invalid", msg) + } + } +} + +func TestClassifyError_ModelInvalid_OverridesStatus400(t *testing.T) { + // This is the exact production error: status 400 + "not a valid model". + // Before the fix, status 400 was classified as FailoverFormat (non-retriable), + // which prevented fallback to other models. + err := fmt.Errorf("API request failed:\n Status: 400\n Body: {\"error\":{\"message\":\"nemotron-3-nano-30b-a3b:free is not a valid model ID\",\"code\":400}}") + result := ClassifyError(err, "nvidia", "nemotron-3-nano-30b-a3b:free") + if result == nil { + t.Fatal("expected non-nil for model-invalid 400 error") + } + if result.Reason != FailoverModelInvalid { + t.Errorf("reason = %q, want model_invalid (should override status 400)", result.Reason) + } + if !result.IsRetriable() { + t.Error("model-invalid error should be retriable to allow fallback to next model") + } + if !result.IsModelInvalid() { + t.Error("should be classified as model invalid") + } +} + func TestClassifyError_ImageDimensionError(t *testing.T) { err := errors.New("image dimensions exceed max allowed 2048x2048") result := ClassifyError(err, "openai", "gpt-4o") @@ -264,6 +317,7 @@ func TestFailoverError_IsRetriable(t *testing.T) { {FailoverBilling, true}, {FailoverTimeout, true}, {FailoverOverloaded, true}, + {FailoverModelInvalid, true}, {FailoverFormat, false}, {FailoverUnknown, true}, } diff --git a/pkg/providers/types.go b/pkg/providers/types.go index b2dda04a5..d16c2aafa 100644 --- a/pkg/providers/types.go +++ b/pkg/providers/types.go @@ -39,13 +39,14 @@ type StatefulProvider interface { type FailoverReason string const ( - FailoverAuth FailoverReason = "auth" - FailoverRateLimit FailoverReason = "rate_limit" - FailoverBilling FailoverReason = "billing" - FailoverTimeout FailoverReason = "timeout" - FailoverFormat FailoverReason = "format" - FailoverOverloaded FailoverReason = "overloaded" - FailoverUnknown FailoverReason = "unknown" + FailoverAuth FailoverReason = "auth" + FailoverRateLimit FailoverReason = "rate_limit" + FailoverBilling FailoverReason = "billing" + FailoverTimeout FailoverReason = "timeout" + FailoverFormat FailoverReason = "format" + FailoverOverloaded FailoverReason = "overloaded" + FailoverModelInvalid FailoverReason = "model_invalid" + FailoverUnknown FailoverReason = "unknown" ) // FailoverError wraps an LLM provider error with classification metadata. @@ -72,6 +73,11 @@ func (e *FailoverError) IsRetriable() bool { return e.Reason != FailoverFormat } +// IsModelInvalid returns true if this error is due to an invalid/unavailable model. +func (e *FailoverError) IsModelInvalid() bool { + return e.Reason == FailoverModelInvalid +} + // ModelConfig holds primary model and fallback list. type ModelConfig struct { Primary string