feat:add usage

This commit is contained in:
xiongfei 2026-04-01 19:34:52 +08:00
parent 9713de691a
commit 7b09313b69
7 changed files with 170 additions and 70 deletions

View file

@ -2158,49 +2158,7 @@ turnLoop:
} }
logger.DebugCF("agent", "LLM response", llmResponseFields) logger.DebugCF("agent", "LLM response", llmResponseFields)
if len(response.ToolCalls) == 0 || gracefulTerminal { // Build assistant message with usage info (before any break/continue)
responseContent := response.Content
if responseContent == "" && response.ReasoningContent != "" {
responseContent = response.ReasoningContent
}
if steerMsgs := al.dequeueSteeringMessagesForScope(ts.sessionKey); len(steerMsgs) > 0 {
logger.InfoCF("agent", "Steering arrived after direct LLM response; continuing turn",
map[string]any{
"agent_id": ts.agent.ID,
"iteration": iteration,
"steering_count": len(steerMsgs),
})
pendingMessages = append(pendingMessages, steerMsgs...)
continue
}
finalContent = responseContent
logger.InfoCF("agent", "LLM response without tool calls (direct answer)",
map[string]any{
"agent_id": ts.agent.ID,
"iteration": iteration,
"content_chars": len(finalContent),
})
break
}
normalizedToolCalls := make([]providers.ToolCall, 0, len(response.ToolCalls))
for _, tc := range response.ToolCalls {
normalizedToolCalls = append(normalizedToolCalls, providers.NormalizeToolCall(tc))
}
toolNames := make([]string, 0, len(normalizedToolCalls))
for _, tc := range normalizedToolCalls {
toolNames = append(toolNames, tc.Name)
}
logger.InfoCF("agent", "LLM requested tool calls",
map[string]any{
"agent_id": ts.agent.ID,
"tools": toolNames,
"count": len(normalizedToolCalls),
"iteration": iteration,
})
allResponsesHandled := len(normalizedToolCalls) > 0
assistantMsg := providers.Message{ assistantMsg := providers.Message{
Role: "assistant", Role: "assistant",
Content: response.Content, Content: response.Content,
@ -2231,13 +2189,28 @@ turnLoop:
}(), }(),
}) })
if response.Usage != nil { if response.Usage != nil {
// Store usage as extra content for persistence and retrieval // Determine the provider and model actually used
// Check if fallback was used and got the result
var usedProvider string
var usedModel string
if fbResult, hasFB := al.fallback.GetLastResult(); hasFB && fbResult.Provider != "" {
usedProvider = fbResult.Provider
usedModel = fbResult.Model
} else if len(activeCandidates) > 0 {
// No fallback or fallback not used, get from active candidates
usedProvider = activeCandidates[0].Provider
usedModel = llmModel
}
// Store usage and model info as extra content for persistence and retrieval
assistantMsg.ExtraContent = &providers.MessageExtra{ assistantMsg.ExtraContent = &providers.MessageExtra{
Usage: map[string]any{ Usage: map[string]any{
"prompt_tokens": response.Usage.PromptTokens, "prompt_tokens": response.Usage.PromptTokens,
"completion_tokens": response.Usage.CompletionTokens, "completion_tokens": response.Usage.CompletionTokens,
"total_tokens": response.Usage.TotalTokens, "total_tokens": response.Usage.TotalTokens,
}, },
// Store model and provider info for usage statistics
Model: usedModel,
Provider: usedProvider,
} }
logger.InfoCF("agent", "Saved token usage to assistant message", map[string]any{ logger.InfoCF("agent", "Saved token usage to assistant message", map[string]any{
"agent_id": ts.agent.ID, "agent_id": ts.agent.ID,
@ -2251,6 +2224,62 @@ turnLoop:
"iteration": iteration, "iteration": iteration,
}) })
} }
if len(response.ToolCalls) == 0 || gracefulTerminal {
responseContent := response.Content
if responseContent == "" && response.ReasoningContent != "" {
responseContent = response.ReasoningContent
}
if steerMsgs := al.dequeueSteeringMessagesForScope(ts.sessionKey); len(steerMsgs) > 0 {
logger.InfoCF("agent", "Steering arrived after direct LLM response; continuing turn",
map[string]any{
"agent_id": ts.agent.ID,
"iteration": iteration,
"steering_count": len(steerMsgs),
})
pendingMessages = append(pendingMessages, steerMsgs...)
// Still need to save the assistant message before continuing
messages = append(messages, assistantMsg)
if !ts.opts.NoHistory {
ts.agent.Sessions.AddFullMessage(ts.sessionKey, assistantMsg)
ts.recordPersistedMessage(assistantMsg)
}
continue
}
finalContent = responseContent
logger.InfoCF("agent", "LLM response without tool calls (direct answer)",
map[string]any{
"agent_id": ts.agent.ID,
"iteration": iteration,
"content_chars": len(finalContent),
})
// Save the assistant message before breaking
messages = append(messages, assistantMsg)
if !ts.opts.NoHistory {
ts.agent.Sessions.AddFullMessage(ts.sessionKey, assistantMsg)
ts.recordPersistedMessage(assistantMsg)
}
break
}
normalizedToolCalls := make([]providers.ToolCall, 0, len(response.ToolCalls))
for _, tc := range response.ToolCalls {
normalizedToolCalls = append(normalizedToolCalls, providers.NormalizeToolCall(tc))
}
toolNames := make([]string, 0, len(normalizedToolCalls))
for _, tc := range normalizedToolCalls {
toolNames = append(toolNames, tc.Name)
}
logger.InfoCF("agent", "LLM requested tool calls",
map[string]any{
"agent_id": ts.agent.ID,
"tools": toolNames,
"count": len(normalizedToolCalls),
"iteration": iteration,
})
allResponsesHandled := len(normalizedToolCalls) > 0
for _, tc := range normalizedToolCalls { for _, tc := range normalizedToolCalls {
argumentsJSON, _ := json.Marshal(tc.Arguments) argumentsJSON, _ := json.Marshal(tc.Arguments)
extraContent := tc.ExtraContent extraContent := tc.ExtraContent

View file

@ -9,7 +9,8 @@ import (
// FallbackChain orchestrates model fallback across multiple candidates. // FallbackChain orchestrates model fallback across multiple candidates.
type FallbackChain struct { type FallbackChain struct {
cooldown *CooldownTracker cooldown *CooldownTracker
lastResult *FallbackResult // stores the last successful result for usage tracking
} }
// FallbackCandidate represents one model/provider to try. // FallbackCandidate represents one model/provider to try.
@ -41,6 +42,15 @@ func NewFallbackChain(cooldown *CooldownTracker) *FallbackChain {
return &FallbackChain{cooldown: cooldown} return &FallbackChain{cooldown: cooldown}
} }
// GetLastResult returns the last successful fallback result for usage tracking.
// The first return value is the result, the second indicates if a result exists.
func (fc *FallbackChain) GetLastResult() (*FallbackResult, bool) {
if fc == nil || fc.lastResult == nil {
return nil, false
}
return fc.lastResult, true
}
// ResolveCandidates parses model config into a deduplicated candidate list. // ResolveCandidates parses model config into a deduplicated candidate list.
func ResolveCandidates(cfg ModelConfig, defaultProvider string) []FallbackCandidate { func ResolveCandidates(cfg ModelConfig, defaultProvider string) []FallbackCandidate {
return ResolveCandidatesWithLookup(cfg, defaultProvider, nil) return ResolveCandidatesWithLookup(cfg, defaultProvider, nil)
@ -147,6 +157,8 @@ func (fc *FallbackChain) Execute(
result.Response = resp result.Response = resp
result.Provider = candidate.Provider result.Provider = candidate.Provider
result.Model = candidate.Model result.Model = candidate.Model
// Store last result for usage tracking
fc.lastResult = result
return result, nil return result, nil
} }

View file

@ -75,8 +75,9 @@ type Message struct {
// MessageExtra holds optional metadata attached to a message. // MessageExtra holds optional metadata attached to a message.
type MessageExtra struct { type MessageExtra struct {
Usage map[string]any `json:"usage,omitempty"` // token usage info Usage map[string]any `json:"usage,omitempty"` // token usage info
Model string `json:"model,omitempty"` // model name used Model string `json:"model,omitempty"` // model name used
Provider string `json:"provider,omitempty"` // provider name used
} }
type ToolDefinition struct { type ToolDefinition struct {

View file

@ -4,6 +4,7 @@ import (
"bufio" "bufio"
"encoding/json" "encoding/json"
"fmt" "fmt"
"log"
"net/http" "net/http"
"os" "os"
"path/filepath" "path/filepath"
@ -24,6 +25,7 @@ func (h *Handler) registerUsageRoutes(mux *http.ServeMux) {
type UsageStats struct { type UsageStats struct {
ModelName string `json:"model_name"` ModelName string `json:"model_name"`
Model string `json:"model"` Model string `json:"model"`
Provider string `json:"provider,omitempty"`
MessageCount int `json:"message_count"` MessageCount int `json:"message_count"`
InputTokens int `json:"input_tokens"` InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"` OutputTokens int `json:"output_tokens"`
@ -239,14 +241,19 @@ type messageWithUsage struct {
ReasoningContent string `json:"reasoning_content,omitempty"` ReasoningContent string `json:"reasoning_content,omitempty"`
ToolCalls []providers.ToolCall `json:"tool_calls,omitempty"` ToolCalls []providers.ToolCall `json:"tool_calls,omitempty"`
ToolCallID string `json:"tool_call_id,omitempty"` ToolCallID string `json:"tool_call_id,omitempty"`
// Usage fields stored by agent loop // Usage fields stored by agent loop in extra_content
PromptTokens int `json:"prompt_tokens,omitempty"` ExtraContent *messageExtraContent `json:"extra_content,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
// Model info // Model info
Model string `json:"model,omitempty"` Model string `json:"model,omitempty"`
} }
// messageExtraContent holds optional metadata attached to a message.
type messageExtraContent struct {
Usage map[string]any `json:"usage,omitempty"` // token usage info
Model string `json:"model,omitempty"` // model name used
Provider string `json:"provider,omitempty"` // provider name used
}
// handleGetUsage returns usage statistics aggregated by model. // handleGetUsage returns usage statistics aggregated by model.
// //
// GET /api/usage?start_date=2024-01-01&end_date=2024-01-31 // GET /api/usage?start_date=2024-01-01&end_date=2024-01-31
@ -255,14 +262,16 @@ func (h *Handler) handleGetUsage(w http.ResponseWriter, r *http.Request) {
dir, err := h.sessionsDir() dir, err := h.sessionsDir()
if err != nil { if err != nil {
http.Error(w, "failed to resolve sessions directory", http.StatusInternalServerError) log.Printf("handleGetUsage: sessionsDir error: %v", err)
http.Error(w, fmt.Sprintf("failed to resolve sessions directory: %v", err), http.StatusInternalServerError)
return return
} }
// Load config to map model names to model identifiers // Load config to map model names to model identifiers
cfg, err := config.LoadConfig(h.configPath) cfg, err := config.LoadConfig(h.configPath)
if err != nil { if err != nil {
http.Error(w, "failed to load config", http.StatusInternalServerError) log.Printf("handleGetUsage: LoadConfig error: %v", err)
http.Error(w, fmt.Sprintf("failed to load config: %v", err), http.StatusInternalServerError)
return return
} }
@ -350,8 +359,14 @@ func (h *Handler) handleGetUsage(w http.ResponseWriter, r *http.Request) {
continue continue
} }
// Determine model name // Determine model name - first try extra_content.model, then message.model (deprecated)
modelName := msgWithUsage.Model modelName := ""
if msgWithUsage.ExtraContent != nil && msgWithUsage.ExtraContent.Model != "" {
modelName = msgWithUsage.ExtraContent.Model
}
if modelName == "" {
modelName = msgWithUsage.Model
}
if modelName == "" { if modelName == "" {
modelName = defaultModelName modelName = defaultModelName
} }
@ -360,17 +375,39 @@ func (h *Handler) handleGetUsage(w http.ResponseWriter, r *http.Request) {
modelName = "unknown" modelName = "unknown"
} }
if _, exists := statsByModel[modelName]; !exists { // Get provider name from extra_content
statsByModel[modelName] = &modelStats{ providerName := ""
if msgWithUsage.ExtraContent != nil && msgWithUsage.ExtraContent.Provider != "" {
providerName = msgWithUsage.ExtraContent.Provider
}
// Create composite key for model+provider combination
statsKey := modelName
if providerName != "" {
statsKey = providerName + "/" + modelName
}
if _, exists := statsByModel[statsKey]; !exists {
statsByModel[statsKey] = &modelStats{
SessionKeys: make(map[string]struct{}), SessionKeys: make(map[string]struct{}),
} }
} }
ms := statsByModel[modelName] ms := statsByModel[statsKey]
ms.MessageCount++ ms.MessageCount++
ms.InputTokens += msgWithUsage.PromptTokens // Extract token usage from extra_content.usage
ms.OutputTokens += msgWithUsage.CompletionTokens if msgWithUsage.ExtraContent != nil && msgWithUsage.ExtraContent.Usage != nil {
ms.TotalTokens += msgWithUsage.TotalTokens usage := msgWithUsage.ExtraContent.Usage
if promptTokens, ok := usage["prompt_tokens"].(float64); ok {
ms.InputTokens += int(promptTokens)
}
if completionTokens, ok := usage["completion_tokens"].(float64); ok {
ms.OutputTokens += int(completionTokens)
}
if totalTokens, ok := usage["total_tokens"].(float64); ok {
ms.TotalTokens += int(totalTokens)
}
}
ms.SessionKeys[sess.Key] = struct{}{} ms.SessionKeys[sess.Key] = struct{}{}
} }
} }
@ -380,23 +417,34 @@ func (h *Handler) handleGetUsage(w http.ResponseWriter, r *http.Request) {
var totalInputTokens, totalOutputTokens, totalTokens, totalMessageCount int var totalInputTokens, totalOutputTokens, totalTokens, totalMessageCount int
var totalEstimatedCost float64 var totalEstimatedCost float64
for modelName, ms := range statsByModel { for statsKey, ms := range statsByModel {
modelIdentifier := modelNameToModel[modelName] // statsKey is either "model" or "provider/model"
if modelIdentifier == "" { // Extract model name and provider for display
modelIdentifier = modelName var modelIdentifier, providerIdentifier string
if strings.Contains(statsKey, "/") {
parts := strings.SplitN(statsKey, "/", 2)
providerIdentifier = parts[0]
modelIdentifier = parts[1]
} else {
modelIdentifier = statsKey
}
// Try to look up the full model name from config
if mappedModel := modelNameToModel[modelIdentifier]; mappedModel != "" {
modelIdentifier = mappedModel
} }
pricing := getModelPricing(modelIdentifier) pricing := getModelPricing(modelIdentifier)
if pricing.InputPricePerMTok == 0 && pricing.OutputPricePerMTok == 0 { if pricing.InputPricePerMTok == 0 && pricing.OutputPricePerMTok == 0 {
// Try with model name as well // Try with model name as well - modelIdentifier already contains the model name
pricing = getModelPricing(modelName)
} }
estimatedCost := calculateCost(ms.InputTokens, ms.OutputTokens, pricing) estimatedCost := calculateCost(ms.InputTokens, ms.OutputTokens, pricing)
stat := UsageStats{ stat := UsageStats{
ModelName: modelName, ModelName: modelIdentifier,
Model: modelIdentifier, Model: modelIdentifier,
Provider: providerIdentifier,
MessageCount: ms.MessageCount, MessageCount: ms.MessageCount,
InputTokens: ms.InputTokens, InputTokens: ms.InputTokens,
OutputTokens: ms.OutputTokens, OutputTokens: ms.OutputTokens,

View file

@ -173,6 +173,8 @@ func isPublicLauncherDashboardPath(method, p string) bool {
return method == http.MethodPost return method == http.MethodPost
case "/api/auth/status": case "/api/auth/status":
return method == http.MethodGet return method == http.MethodGet
case "/api/usage":
return method == http.MethodGet
} }
return false return false
} }

View file

@ -5,6 +5,7 @@ import { launcherFetch } from "@/api/http"
export interface UsageStats { export interface UsageStats {
model_name: string model_name: string
model: string model: string
provider?: string
message_count: number message_count: number
input_tokens: number input_tokens: number
output_tokens: number output_tokens: number

View file

@ -248,9 +248,16 @@ export function UsagePage() {
> >
<td className="py-3 px-4 text-sm font-medium"> <td className="py-3 px-4 text-sm font-medium">
<div> <div>
<div>{stat.model_name}</div> <div className="flex items-center gap-2">
<span>{stat.model_name}</span>
{stat.provider && (
<span className="inline-flex items-center rounded-full bg-muted px-2 py-0.5 text-[10px] font-normal text-muted-foreground uppercase tracking-wider">
{stat.provider}
</span>
)}
</div>
{stat.model !== stat.model_name && ( {stat.model !== stat.model_name && (
<div className="text-muted-foreground text-xs"> <div className="text-muted-foreground text-xs font-normal">
{stat.model} {stat.model}
</div> </div>
)} )}