feat: implement error categorization for agent loop observability
This commit introduces error categorization for the agent loop to distinguish between infrastructure, model, and logic failures, fulfilling Phase 5 of the AGENT_LOOP_IMPROVEMENTS.md roadmap. Changes include: - `pkg/agent/loop_execute_llm.go`: Classifies LLM errors using `providers.ClassifyError`. Errors related to formatting or context length are categorized as `model_failure`, while others (e.g., timeout, rate limits, auth) default to `infrastructure_failure`. - `pkg/agent/loop_llm.go`: Checks `r.result.IsError` during tool execution processing. If true, logs the failure with the `error_category` set to `logic_failure`. - `AGENT_LOOP_IMPROVEMENTS.md`: Marks the Error Categorization task as complete. Co-authored-by: hobbyistlabs-coder <267281733+hobbyistlabs-coder@users.noreply.github.com>
This commit is contained in:
parent
98015267d6
commit
060fb7c675
3 changed files with 21 additions and 4 deletions
|
|
@ -33,7 +33,7 @@ This document outlines a series of tasks to improve the main loop of the agentic
|
||||||
## Phase 5: Observability & Logging Dashboard
|
## Phase 5: Observability & Logging Dashboard
|
||||||
|
|
||||||
* [ ] **Session Replay:** Capture the full "Chain of Thought" (CoT), including tool-call inputs/outputs, and state transitions to facilitate deep-dive session reviews.
|
* [ ] **Session Replay:** Capture the full "Chain of Thought" (CoT), including tool-call inputs/outputs, and state transitions to facilitate deep-dive session reviews.
|
||||||
* [ ] **Error Categorization:** Distinguish between Model Failures (hallucinations/refusals), Infrastructure Failures (API timeouts), and Logic Failures (code execution errors).
|
* [x] **Error Categorization:** Distinguish between Model Failures (hallucinations/refusals), Infrastructure Failures (API timeouts), and Logic Failures (code execution errors).
|
||||||
* [ ] **Real-time Visuals:** Implement a lightweight UI/Dashboard using a Go-compatible framework like a custom Bubble Tea TUI to monitor the agent's health.
|
* [ ] **Real-time Visuals:** Implement a lightweight UI/Dashboard using a Go-compatible framework like a custom Bubble Tea TUI to monitor the agent's health.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
|
||||||
|
|
@ -142,11 +142,19 @@ func (al *AgentLoop) executeLLMWithRetry(
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
errorCategory := "infrastructure_failure"
|
||||||
|
if failErr := providers.ClassifyError(err, agent.ID, activeModel); failErr != nil {
|
||||||
|
if failErr.Reason == providers.FailoverFormat || failErr.Reason == providers.FailoverContextLength {
|
||||||
|
errorCategory = "model_failure"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
logger.ErrorCF("agent", "LLM call failed",
|
logger.ErrorCF("agent", "LLM call failed",
|
||||||
map[string]any{
|
map[string]any{
|
||||||
"agent_id": agent.ID,
|
"agent_id": agent.ID,
|
||||||
"iteration": iteration,
|
"iteration": iteration,
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
|
"error_category": errorCategory,
|
||||||
})
|
})
|
||||||
return nil, fmt.Errorf("LLM call failed after retries: %w", err)
|
return nil, fmt.Errorf("LLM call failed after retries: %w", err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -361,6 +361,15 @@ func (al *AgentLoop) runLLMIteration(
|
||||||
contentForLLM = r.result.Err.Error()
|
contentForLLM = r.result.Err.Error()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if r.result.IsError {
|
||||||
|
logger.ErrorCF("agent", "Tool execution failed",
|
||||||
|
map[string]any{
|
||||||
|
"tool": r.tc.Name,
|
||||||
|
"error": contentForLLM,
|
||||||
|
"error_category": "logic_failure",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
toolResultMsg := providers.Message{
|
toolResultMsg := providers.Message{
|
||||||
Role: "tool",
|
Role: "tool",
|
||||||
Content: contentForLLM,
|
Content: contentForLLM,
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue