opencode/picoclaw
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: implement LLM provider telemetry using expvar

Browse source 
- Added expvar metrics for tracking API call latency, token usage, and failover reasons in `FallbackChain.Execute` and `FallbackChain.ExecuteImage` methods.
- Checked off the "LLM Provider Telemetry" task in docs/design/ETL_TODO.md.

Co-authored-by: hobbyistlabs-coder <267281733+hobbyistlabs-coder@users.noreply.github.com>
This commit is contained in:
google-labs-jules[bot] 2026-03-18 18:37:38 +00:00
parent 43b8f055f3
commit 564121059d
2 changed files with 27 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docs/design/ETL_TODO.md
View file

@ -9,7 +9,7 @@ This document tracks the tasks required to implement the "Ultimate Visibility" E
- [x] **Goroutine Tracking:** Implement a metric to track the number of active Goroutines.
- [x] **Memory Tracking:** Implement a metric to track heap allocation and GC pauses.
- [x] **AgentLoop Telemetry:** Add specific instrumentation to the `AgentLoop` (iteration duration, tool execution duration, failure counts).
- [ ] **LLM Provider Telemetry:** Track API call latency, token usage, and failover reasons for LLM providers.
- [x] **LLM Provider Telemetry:** Track API call latency, token usage, and failover reasons for LLM providers.
- [ ] **API Gateway Telemetry:** Track request rates (RPS), latency percentiles, and error rates for HTTP and WebSocket endpoints.
- [ ] **Tracing Instrumentation:** Introduce trace IDs at entry points (HTTP, WebSocket) and propagate them via context to track end-to-end execution flow.

26
pkg/providers/fallback.go
View file

@ -2,11 +2,18 @@ package providers
import (
"context"
"expvar"
"fmt"
"strings"
"time"
)
var (
metricsLLMProviderDuration = expvar.NewMap("llm_provider_duration_seconds")
metricsLLMProviderFailover = expvar.NewMap("llm_provider_failover_counts")
metricsLLMProviderTokenUsage = expvar.NewMap("llm_provider_token_usage")
)
// FallbackChain orchestrates model fallback across multiple candidates.
type FallbackChain struct {
cooldown *CooldownTracker
@ -139,8 +146,15 @@ func (fc *FallbackChain) Execute(
resp, err := run(ctx, candidate.Provider, candidate.Model)
elapsed := time.Since(start)
metricsLLMProviderDuration.AddFloat(candidate.Provider, elapsed.Seconds())
if err == nil {
// Success.
if resp != nil && resp.Usage != nil {
metricsLLMProviderTokenUsage.Add(candidate.Provider+"_prompt", int64(resp.Usage.PromptTokens))
metricsLLMProviderTokenUsage.Add(candidate.Provider+"_completion", int64(resp.Usage.CompletionTokens))
metricsLLMProviderTokenUsage.Add(candidate.Provider+"_total", int64(resp.Usage.TotalTokens))
}
fc.cooldown.MarkSuccess(candidate.Provider)
result.Response = resp
result.Provider = candidate.Provider
@ -170,10 +184,13 @@ func (fc *FallbackChain) Execute(
Error: err,
Duration: elapsed,
})
metricsLLMProviderFailover.Add(candidate.Provider+"_"+string(FailoverUnknown), 1)
return nil, fmt.Errorf("fallback: unclassified error from %s/%s: %w",
candidate.Provider, candidate.Model, err)
}
metricsLLMProviderFailover.Add(candidate.Provider+"_"+string(failErr.Reason), 1)
// Non-retriable error: abort immediately.
if !failErr.IsRetriable() {
result.Attempts = append(result.Attempts, FallbackAttempt{
@ -231,7 +248,14 @@ func (fc *FallbackChain) ExecuteImage(
resp, err := run(ctx, candidate.Provider, candidate.Model)
elapsed := time.Since(start)
metricsLLMProviderDuration.AddFloat(candidate.Provider, elapsed.Seconds())
if err == nil {
if resp != nil && resp.Usage != nil {
metricsLLMProviderTokenUsage.Add(candidate.Provider+"_prompt", int64(resp.Usage.PromptTokens))
metricsLLMProviderTokenUsage.Add(candidate.Provider+"_completion", int64(resp.Usage.CompletionTokens))
metricsLLMProviderTokenUsage.Add(candidate.Provider+"_total", int64(resp.Usage.TotalTokens))
}
result.Response = resp
result.Provider = candidate.Provider
result.Model = candidate.Model
@ -251,6 +275,7 @@ func (fc *FallbackChain) ExecuteImage(
// Image dimension/size errors are non-retriable.
errMsg := strings.ToLower(err.Error())
if IsImageDimensionError(errMsg) || IsImageSizeError(errMsg) {
metricsLLMProviderFailover.Add(candidate.Provider+"_"+string(FailoverFormat), 1)
result.Attempts = append(result.Attempts, FallbackAttempt{
Provider: candidate.Provider,
Model: candidate.Model,
@ -267,6 +292,7 @@ func (fc *FallbackChain) ExecuteImage(
}
// Any other error: record and try next.
metricsLLMProviderFailover.Add(candidate.Provider+"_"+string(FailoverUnknown), 1)
result.Attempts = append(result.Attempts, FallbackAttempt{
Provider: candidate.Provider,
Model: candidate.Model,