fix(providers): handle model-invalid 400 errors with fallback and warning

Add FailoverModelInvalid reason for 400 errors where the model ID is
invalid or unavailable (e.g. 'not a valid model ID'). Previously, all
400 errors were classified as FailoverFormat (non-retriable), which
aborted the fallback chain immediately instead of trying the next model.

Model-invalid patterns are now checked BEFORE HTTP status code
classification, allowing fallback to proceed. When fallback succeeds
after a model-invalid error, a warning message is sent to the user's
channel so they know to fix the config.
This commit is contained in:
muava12 2026-02-24 23:56:19 +08:00
parent 28f65201d3
commit 5f53e40260
4 changed files with 111 additions and 8 deletions

View file

@ -557,6 +557,22 @@ func (al *AgentLoop) runLLMIteration(
logger.InfoCF("agent", fmt.Sprintf("Fallback: succeeded with %s/%s after %d attempts",
fbResult.Provider, fbResult.Model, len(fbResult.Attempts)+1),
map[string]any{"agent_id": agent.ID, "iteration": iteration})
// Send warning for model-invalid errors so user knows to fix config
if !constants.IsInternalChannel(opts.Channel) {
for _, attempt := range fbResult.Attempts {
if failErr, ok := attempt.Error.(*providers.FailoverError); ok && failErr.IsModelInvalid() {
al.bus.PublishOutbound(bus.OutboundMessage{
Channel: opts.Channel,
ChatID: opts.ChatID,
Content: fmt.Sprintf("⚠️ Model %s/%s is invalid or unavailable: %v\nUsing fallback: %s/%s",
attempt.Provider, attempt.Model, failErr.Wrapped,
fbResult.Provider, fbResult.Model),
})
break // Only send one warning even if multiple model-invalid errors
}
}
}
}
return fbResult.Response, nil
}

View file

@ -78,6 +78,21 @@ var (
substr("invalid request format"),
}
// Model invalid/not found patterns: these are 400 errors that should be
// retriable (fallback to next model), NOT treated as format errors.
modelInvalidPatterns = []errorPattern{
substr("not a valid model"),
substr("model not found"),
substr("model_not_found"),
substr("model not available"),
substr("does not exist"),
substr("no such model"),
substr("invalid model"),
rxp(`model.*not.*supported`),
rxp(`model.*is.*unavailable`),
rxp(`model.*is.*deprecated`),
}
imageDimensionPatterns = []errorPattern{
rxp(`image dimensions exceed max`),
}
@ -128,7 +143,19 @@ func ClassifyError(err error, provider, model string) *FailoverError {
}
}
// Try HTTP status code extraction first.
// Model invalid/not found: retriable, should fallback to another model.
// This MUST run before HTTP status classification, because 400 + "not a valid model"
// would otherwise be classified as non-retriable FailoverFormat.
if matchesAny(msg, modelInvalidPatterns) {
return &FailoverError{
Reason: FailoverModelInvalid,
Provider: provider,
Model: model,
Wrapped: err,
}
}
// Try HTTP status code extraction.
if status := extractHTTPStatus(msg); status > 0 {
if reason := classifyByStatus(status); reason != "" {
return &FailoverError{

View file

@ -207,6 +207,59 @@ func TestClassifyError_FormatPatterns(t *testing.T) {
}
}
func TestClassifyError_ModelInvalidPatterns(t *testing.T) {
patterns := []string{
"nemotron-3-nano-30b-a3b:free is not a valid model ID",
"model not found",
"model_not_found: the requested model does not exist",
"model not available in this region",
"the model does not exist or you do not have access",
"no such model: gpt-5-turbo",
"invalid model specified",
"model llama-3-8b is not supported",
"model gpt-4o-mini is unavailable",
"model codellama is deprecated",
}
for _, msg := range patterns {
err := errors.New(msg)
result := ClassifyError(err, "nvidia", "test-model")
if result == nil {
t.Errorf("pattern %q: expected non-nil", msg)
continue
}
if result.Reason != FailoverModelInvalid {
t.Errorf("pattern %q: reason = %q, want model_invalid", msg, result.Reason)
}
if !result.IsRetriable() {
t.Errorf("pattern %q: should be retriable to allow fallback", msg)
}
if !result.IsModelInvalid() {
t.Errorf("pattern %q: should be classified as model invalid", msg)
}
}
}
func TestClassifyError_ModelInvalid_OverridesStatus400(t *testing.T) {
// This is the exact production error: status 400 + "not a valid model".
// Before the fix, status 400 was classified as FailoverFormat (non-retriable),
// which prevented fallback to other models.
err := fmt.Errorf("API request failed:\n Status: 400\n Body: {\"error\":{\"message\":\"nemotron-3-nano-30b-a3b:free is not a valid model ID\",\"code\":400}}")
result := ClassifyError(err, "nvidia", "nemotron-3-nano-30b-a3b:free")
if result == nil {
t.Fatal("expected non-nil for model-invalid 400 error")
}
if result.Reason != FailoverModelInvalid {
t.Errorf("reason = %q, want model_invalid (should override status 400)", result.Reason)
}
if !result.IsRetriable() {
t.Error("model-invalid error should be retriable to allow fallback to next model")
}
if !result.IsModelInvalid() {
t.Error("should be classified as model invalid")
}
}
func TestClassifyError_ImageDimensionError(t *testing.T) {
err := errors.New("image dimensions exceed max allowed 2048x2048")
result := ClassifyError(err, "openai", "gpt-4o")
@ -264,6 +317,7 @@ func TestFailoverError_IsRetriable(t *testing.T) {
{FailoverBilling, true},
{FailoverTimeout, true},
{FailoverOverloaded, true},
{FailoverModelInvalid, true},
{FailoverFormat, false},
{FailoverUnknown, true},
}

View file

@ -39,13 +39,14 @@ type StatefulProvider interface {
type FailoverReason string
const (
FailoverAuth FailoverReason = "auth"
FailoverRateLimit FailoverReason = "rate_limit"
FailoverBilling FailoverReason = "billing"
FailoverTimeout FailoverReason = "timeout"
FailoverFormat FailoverReason = "format"
FailoverOverloaded FailoverReason = "overloaded"
FailoverUnknown FailoverReason = "unknown"
FailoverAuth FailoverReason = "auth"
FailoverRateLimit FailoverReason = "rate_limit"
FailoverBilling FailoverReason = "billing"
FailoverTimeout FailoverReason = "timeout"
FailoverFormat FailoverReason = "format"
FailoverOverloaded FailoverReason = "overloaded"
FailoverModelInvalid FailoverReason = "model_invalid"
FailoverUnknown FailoverReason = "unknown"
)
// FailoverError wraps an LLM provider error with classification metadata.
@ -72,6 +73,11 @@ func (e *FailoverError) IsRetriable() bool {
return e.Reason != FailoverFormat
}
// IsModelInvalid returns true if this error is due to an invalid/unavailable model.
func (e *FailoverError) IsModelInvalid() bool {
return e.Reason == FailoverModelInvalid
}
// ModelConfig holds primary model and fallback list.
type ModelConfig struct {
Primary string