feat: add AgentLoop telemetry using expvar

- Added cumulative tracking for iteration durations.
- Added cumulative tracking for tool execution durations.
- Added counter for loop execution failure counts.
- Updated `ETL_TODO.md` to mark this task as complete.

Co-authored-by: hobbyistlabs-coder <267281733+hobbyistlabs-coder@users.noreply.github.com>

docs/design/ETL_TODO.md

@@ -8,7 +8,7 @@ This document tracks the tasks required to implement the "Ultimate Visibility" E
 - [x] **Basic Metrics Implementation:** Introduce a metrics package (e.g., using `expvar` or a Prometheus client) to expose basic application metrics.
 - [x] **Goroutine Tracking:** Implement a metric to track the number of active Goroutines.
 - [x] **Memory Tracking:** Implement a metric to track heap allocation and GC pauses.
-- [ ] **AgentLoop Telemetry:** Add specific instrumentation to the `AgentLoop` (iteration duration, tool execution duration, failure counts).
+- [x] **AgentLoop Telemetry:** Add specific instrumentation to the `AgentLoop` (iteration duration, tool execution duration, failure counts).
 - [ ] **LLM Provider Telemetry:** Track API call latency, token usage, and failover reasons for LLM providers.
 - [ ] **API Gateway Telemetry:** Track request rates (RPS), latency percentiles, and error rates for HTTP and WebSocket endpoints.
 - [ ] **Tracing Instrumentation:** Introduce trace IDs at entry points (HTTP, WebSocket) and propagate them via context to track end-to-end execution flow.

pkg/agent/loop_execute_tools.go

@@ -3,6 +3,7 @@ package agent
 import (
 	"context"
 	"encoding/json"
+	"expvar"
 	"fmt"
 	"sync"
 	"time"
@@ -14,6 +15,10 @@ import (
 	"jane/pkg/utils"
 )
 
+var (
+	metricsToolExecutionDuration = expvar.NewFloat("agentloop_tool_execution_duration_seconds")
+)
+
 type indexedAgentResult struct {
 	result *tools.ToolResult
 	tc     providers.ToolCall
@@ -106,6 +111,7 @@ func (al *AgentLoop) executeToolBatch(
 				})
 			}
 
+			startToolTime := time.Now()
 			toolResult := agent.Tools.ExecuteWithContext(
 				ctx,
 				tc.Name,
@@ -114,6 +120,7 @@ func (al *AgentLoop) executeToolBatch(
 				opts.ChatID,
 				asyncCallback,
 			)
+			metricsToolExecutionDuration.Add(time.Since(startToolTime).Seconds())
 			agentResults[idx].result = toolResult
 		}(i, tc)
 	}

pkg/agent/loop_llm.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"expvar"
 	"fmt"
 	"time"
 
@@ -20,6 +21,11 @@ import (
 	"jane/pkg/utils"
 )
 
+var (
+	metricsIterationDuration = expvar.NewFloat("agentloop_iteration_duration_seconds")
+	metricsFailureCounts     = expvar.NewInt("agentloop_failure_counts")
+)
+
 // runAgentLoop is the core message processing logic.
 func (al *AgentLoop) runAgentLoop(
 	ctx context.Context,
@@ -64,8 +70,11 @@ func (al *AgentLoop) runAgentLoop(
 	agent.Sessions.AddMessage(opts.SessionKey, "user", opts.UserMessage)
 
 	// 3. Run LLM iteration loop
+	startTime := time.Now()
 	finalContent, iteration, err := al.runLLMIteration(ctx, agent, messages, opts)
+	metricsIterationDuration.Add(time.Since(startTime).Seconds())
 	if err != nil {
+		metricsFailureCounts.Add(1)
 		return "", err
 	}