fix(providers): handle model-invalid 400 errors with fallback and warning
Add FailoverModelInvalid reason for 400 errors where the model ID is invalid or unavailable (e.g. 'not a valid model ID'). Previously, all 400 errors were classified as FailoverFormat (non-retriable), which aborted the fallback chain immediately instead of trying the next model. Model-invalid patterns are now checked BEFORE HTTP status code classification, allowing fallback to proceed. When fallback succeeds after a model-invalid error, a warning message is sent to the user's channel so they know to fix the config.
This commit is contained in:
parent
28f65201d3
commit
5f53e40260
4 changed files with 111 additions and 8 deletions
|
|
@ -557,6 +557,22 @@ func (al *AgentLoop) runLLMIteration(
|
|||
logger.InfoCF("agent", fmt.Sprintf("Fallback: succeeded with %s/%s after %d attempts",
|
||||
fbResult.Provider, fbResult.Model, len(fbResult.Attempts)+1),
|
||||
map[string]any{"agent_id": agent.ID, "iteration": iteration})
|
||||
|
||||
// Send warning for model-invalid errors so user knows to fix config
|
||||
if !constants.IsInternalChannel(opts.Channel) {
|
||||
for _, attempt := range fbResult.Attempts {
|
||||
if failErr, ok := attempt.Error.(*providers.FailoverError); ok && failErr.IsModelInvalid() {
|
||||
al.bus.PublishOutbound(bus.OutboundMessage{
|
||||
Channel: opts.Channel,
|
||||
ChatID: opts.ChatID,
|
||||
Content: fmt.Sprintf("⚠️ Model %s/%s is invalid or unavailable: %v\nUsing fallback: %s/%s",
|
||||
attempt.Provider, attempt.Model, failErr.Wrapped,
|
||||
fbResult.Provider, fbResult.Model),
|
||||
})
|
||||
break // Only send one warning even if multiple model-invalid errors
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return fbResult.Response, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,6 +78,21 @@ var (
|
|||
substr("invalid request format"),
|
||||
}
|
||||
|
||||
// Model invalid/not found patterns: these are 400 errors that should be
|
||||
// retriable (fallback to next model), NOT treated as format errors.
|
||||
modelInvalidPatterns = []errorPattern{
|
||||
substr("not a valid model"),
|
||||
substr("model not found"),
|
||||
substr("model_not_found"),
|
||||
substr("model not available"),
|
||||
substr("does not exist"),
|
||||
substr("no such model"),
|
||||
substr("invalid model"),
|
||||
rxp(`model.*not.*supported`),
|
||||
rxp(`model.*is.*unavailable`),
|
||||
rxp(`model.*is.*deprecated`),
|
||||
}
|
||||
|
||||
imageDimensionPatterns = []errorPattern{
|
||||
rxp(`image dimensions exceed max`),
|
||||
}
|
||||
|
|
@ -128,7 +143,19 @@ func ClassifyError(err error, provider, model string) *FailoverError {
|
|||
}
|
||||
}
|
||||
|
||||
// Try HTTP status code extraction first.
|
||||
// Model invalid/not found: retriable, should fallback to another model.
|
||||
// This MUST run before HTTP status classification, because 400 + "not a valid model"
|
||||
// would otherwise be classified as non-retriable FailoverFormat.
|
||||
if matchesAny(msg, modelInvalidPatterns) {
|
||||
return &FailoverError{
|
||||
Reason: FailoverModelInvalid,
|
||||
Provider: provider,
|
||||
Model: model,
|
||||
Wrapped: err,
|
||||
}
|
||||
}
|
||||
|
||||
// Try HTTP status code extraction.
|
||||
if status := extractHTTPStatus(msg); status > 0 {
|
||||
if reason := classifyByStatus(status); reason != "" {
|
||||
return &FailoverError{
|
||||
|
|
|
|||
|
|
@ -207,6 +207,59 @@ func TestClassifyError_FormatPatterns(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestClassifyError_ModelInvalidPatterns(t *testing.T) {
|
||||
patterns := []string{
|
||||
"nemotron-3-nano-30b-a3b:free is not a valid model ID",
|
||||
"model not found",
|
||||
"model_not_found: the requested model does not exist",
|
||||
"model not available in this region",
|
||||
"the model does not exist or you do not have access",
|
||||
"no such model: gpt-5-turbo",
|
||||
"invalid model specified",
|
||||
"model llama-3-8b is not supported",
|
||||
"model gpt-4o-mini is unavailable",
|
||||
"model codellama is deprecated",
|
||||
}
|
||||
|
||||
for _, msg := range patterns {
|
||||
err := errors.New(msg)
|
||||
result := ClassifyError(err, "nvidia", "test-model")
|
||||
if result == nil {
|
||||
t.Errorf("pattern %q: expected non-nil", msg)
|
||||
continue
|
||||
}
|
||||
if result.Reason != FailoverModelInvalid {
|
||||
t.Errorf("pattern %q: reason = %q, want model_invalid", msg, result.Reason)
|
||||
}
|
||||
if !result.IsRetriable() {
|
||||
t.Errorf("pattern %q: should be retriable to allow fallback", msg)
|
||||
}
|
||||
if !result.IsModelInvalid() {
|
||||
t.Errorf("pattern %q: should be classified as model invalid", msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyError_ModelInvalid_OverridesStatus400(t *testing.T) {
|
||||
// This is the exact production error: status 400 + "not a valid model".
|
||||
// Before the fix, status 400 was classified as FailoverFormat (non-retriable),
|
||||
// which prevented fallback to other models.
|
||||
err := fmt.Errorf("API request failed:\n Status: 400\n Body: {\"error\":{\"message\":\"nemotron-3-nano-30b-a3b:free is not a valid model ID\",\"code\":400}}")
|
||||
result := ClassifyError(err, "nvidia", "nemotron-3-nano-30b-a3b:free")
|
||||
if result == nil {
|
||||
t.Fatal("expected non-nil for model-invalid 400 error")
|
||||
}
|
||||
if result.Reason != FailoverModelInvalid {
|
||||
t.Errorf("reason = %q, want model_invalid (should override status 400)", result.Reason)
|
||||
}
|
||||
if !result.IsRetriable() {
|
||||
t.Error("model-invalid error should be retriable to allow fallback to next model")
|
||||
}
|
||||
if !result.IsModelInvalid() {
|
||||
t.Error("should be classified as model invalid")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyError_ImageDimensionError(t *testing.T) {
|
||||
err := errors.New("image dimensions exceed max allowed 2048x2048")
|
||||
result := ClassifyError(err, "openai", "gpt-4o")
|
||||
|
|
@ -264,6 +317,7 @@ func TestFailoverError_IsRetriable(t *testing.T) {
|
|||
{FailoverBilling, true},
|
||||
{FailoverTimeout, true},
|
||||
{FailoverOverloaded, true},
|
||||
{FailoverModelInvalid, true},
|
||||
{FailoverFormat, false},
|
||||
{FailoverUnknown, true},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ const (
|
|||
FailoverTimeout FailoverReason = "timeout"
|
||||
FailoverFormat FailoverReason = "format"
|
||||
FailoverOverloaded FailoverReason = "overloaded"
|
||||
FailoverModelInvalid FailoverReason = "model_invalid"
|
||||
FailoverUnknown FailoverReason = "unknown"
|
||||
)
|
||||
|
||||
|
|
@ -72,6 +73,11 @@ func (e *FailoverError) IsRetriable() bool {
|
|||
return e.Reason != FailoverFormat
|
||||
}
|
||||
|
||||
// IsModelInvalid returns true if this error is due to an invalid/unavailable model.
|
||||
func (e *FailoverError) IsModelInvalid() bool {
|
||||
return e.Reason == FailoverModelInvalid
|
||||
}
|
||||
|
||||
// ModelConfig holds primary model and fallback list.
|
||||
type ModelConfig struct {
|
||||
Primary string
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue