feat:add usage
This commit is contained in:
parent
9713de691a
commit
7b09313b69
7 changed files with 170 additions and 70 deletions
|
|
@ -2158,49 +2158,7 @@ turnLoop:
|
||||||
}
|
}
|
||||||
logger.DebugCF("agent", "LLM response", llmResponseFields)
|
logger.DebugCF("agent", "LLM response", llmResponseFields)
|
||||||
|
|
||||||
if len(response.ToolCalls) == 0 || gracefulTerminal {
|
// Build assistant message with usage info (before any break/continue)
|
||||||
responseContent := response.Content
|
|
||||||
if responseContent == "" && response.ReasoningContent != "" {
|
|
||||||
responseContent = response.ReasoningContent
|
|
||||||
}
|
|
||||||
if steerMsgs := al.dequeueSteeringMessagesForScope(ts.sessionKey); len(steerMsgs) > 0 {
|
|
||||||
logger.InfoCF("agent", "Steering arrived after direct LLM response; continuing turn",
|
|
||||||
map[string]any{
|
|
||||||
"agent_id": ts.agent.ID,
|
|
||||||
"iteration": iteration,
|
|
||||||
"steering_count": len(steerMsgs),
|
|
||||||
})
|
|
||||||
pendingMessages = append(pendingMessages, steerMsgs...)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
finalContent = responseContent
|
|
||||||
logger.InfoCF("agent", "LLM response without tool calls (direct answer)",
|
|
||||||
map[string]any{
|
|
||||||
"agent_id": ts.agent.ID,
|
|
||||||
"iteration": iteration,
|
|
||||||
"content_chars": len(finalContent),
|
|
||||||
})
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
normalizedToolCalls := make([]providers.ToolCall, 0, len(response.ToolCalls))
|
|
||||||
for _, tc := range response.ToolCalls {
|
|
||||||
normalizedToolCalls = append(normalizedToolCalls, providers.NormalizeToolCall(tc))
|
|
||||||
}
|
|
||||||
|
|
||||||
toolNames := make([]string, 0, len(normalizedToolCalls))
|
|
||||||
for _, tc := range normalizedToolCalls {
|
|
||||||
toolNames = append(toolNames, tc.Name)
|
|
||||||
}
|
|
||||||
logger.InfoCF("agent", "LLM requested tool calls",
|
|
||||||
map[string]any{
|
|
||||||
"agent_id": ts.agent.ID,
|
|
||||||
"tools": toolNames,
|
|
||||||
"count": len(normalizedToolCalls),
|
|
||||||
"iteration": iteration,
|
|
||||||
})
|
|
||||||
|
|
||||||
allResponsesHandled := len(normalizedToolCalls) > 0
|
|
||||||
assistantMsg := providers.Message{
|
assistantMsg := providers.Message{
|
||||||
Role: "assistant",
|
Role: "assistant",
|
||||||
Content: response.Content,
|
Content: response.Content,
|
||||||
|
|
@ -2231,13 +2189,28 @@ turnLoop:
|
||||||
}(),
|
}(),
|
||||||
})
|
})
|
||||||
if response.Usage != nil {
|
if response.Usage != nil {
|
||||||
// Store usage as extra content for persistence and retrieval
|
// Determine the provider and model actually used
|
||||||
|
// Check if fallback was used and got the result
|
||||||
|
var usedProvider string
|
||||||
|
var usedModel string
|
||||||
|
if fbResult, hasFB := al.fallback.GetLastResult(); hasFB && fbResult.Provider != "" {
|
||||||
|
usedProvider = fbResult.Provider
|
||||||
|
usedModel = fbResult.Model
|
||||||
|
} else if len(activeCandidates) > 0 {
|
||||||
|
// No fallback or fallback not used, get from active candidates
|
||||||
|
usedProvider = activeCandidates[0].Provider
|
||||||
|
usedModel = llmModel
|
||||||
|
}
|
||||||
|
// Store usage and model info as extra content for persistence and retrieval
|
||||||
assistantMsg.ExtraContent = &providers.MessageExtra{
|
assistantMsg.ExtraContent = &providers.MessageExtra{
|
||||||
Usage: map[string]any{
|
Usage: map[string]any{
|
||||||
"prompt_tokens": response.Usage.PromptTokens,
|
"prompt_tokens": response.Usage.PromptTokens,
|
||||||
"completion_tokens": response.Usage.CompletionTokens,
|
"completion_tokens": response.Usage.CompletionTokens,
|
||||||
"total_tokens": response.Usage.TotalTokens,
|
"total_tokens": response.Usage.TotalTokens,
|
||||||
},
|
},
|
||||||
|
// Store model and provider info for usage statistics
|
||||||
|
Model: usedModel,
|
||||||
|
Provider: usedProvider,
|
||||||
}
|
}
|
||||||
logger.InfoCF("agent", "Saved token usage to assistant message", map[string]any{
|
logger.InfoCF("agent", "Saved token usage to assistant message", map[string]any{
|
||||||
"agent_id": ts.agent.ID,
|
"agent_id": ts.agent.ID,
|
||||||
|
|
@ -2251,6 +2224,62 @@ turnLoop:
|
||||||
"iteration": iteration,
|
"iteration": iteration,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(response.ToolCalls) == 0 || gracefulTerminal {
|
||||||
|
responseContent := response.Content
|
||||||
|
if responseContent == "" && response.ReasoningContent != "" {
|
||||||
|
responseContent = response.ReasoningContent
|
||||||
|
}
|
||||||
|
if steerMsgs := al.dequeueSteeringMessagesForScope(ts.sessionKey); len(steerMsgs) > 0 {
|
||||||
|
logger.InfoCF("agent", "Steering arrived after direct LLM response; continuing turn",
|
||||||
|
map[string]any{
|
||||||
|
"agent_id": ts.agent.ID,
|
||||||
|
"iteration": iteration,
|
||||||
|
"steering_count": len(steerMsgs),
|
||||||
|
})
|
||||||
|
pendingMessages = append(pendingMessages, steerMsgs...)
|
||||||
|
// Still need to save the assistant message before continuing
|
||||||
|
messages = append(messages, assistantMsg)
|
||||||
|
if !ts.opts.NoHistory {
|
||||||
|
ts.agent.Sessions.AddFullMessage(ts.sessionKey, assistantMsg)
|
||||||
|
ts.recordPersistedMessage(assistantMsg)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
finalContent = responseContent
|
||||||
|
logger.InfoCF("agent", "LLM response without tool calls (direct answer)",
|
||||||
|
map[string]any{
|
||||||
|
"agent_id": ts.agent.ID,
|
||||||
|
"iteration": iteration,
|
||||||
|
"content_chars": len(finalContent),
|
||||||
|
})
|
||||||
|
// Save the assistant message before breaking
|
||||||
|
messages = append(messages, assistantMsg)
|
||||||
|
if !ts.opts.NoHistory {
|
||||||
|
ts.agent.Sessions.AddFullMessage(ts.sessionKey, assistantMsg)
|
||||||
|
ts.recordPersistedMessage(assistantMsg)
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
normalizedToolCalls := make([]providers.ToolCall, 0, len(response.ToolCalls))
|
||||||
|
for _, tc := range response.ToolCalls {
|
||||||
|
normalizedToolCalls = append(normalizedToolCalls, providers.NormalizeToolCall(tc))
|
||||||
|
}
|
||||||
|
|
||||||
|
toolNames := make([]string, 0, len(normalizedToolCalls))
|
||||||
|
for _, tc := range normalizedToolCalls {
|
||||||
|
toolNames = append(toolNames, tc.Name)
|
||||||
|
}
|
||||||
|
logger.InfoCF("agent", "LLM requested tool calls",
|
||||||
|
map[string]any{
|
||||||
|
"agent_id": ts.agent.ID,
|
||||||
|
"tools": toolNames,
|
||||||
|
"count": len(normalizedToolCalls),
|
||||||
|
"iteration": iteration,
|
||||||
|
})
|
||||||
|
|
||||||
|
allResponsesHandled := len(normalizedToolCalls) > 0
|
||||||
for _, tc := range normalizedToolCalls {
|
for _, tc := range normalizedToolCalls {
|
||||||
argumentsJSON, _ := json.Marshal(tc.Arguments)
|
argumentsJSON, _ := json.Marshal(tc.Arguments)
|
||||||
extraContent := tc.ExtraContent
|
extraContent := tc.ExtraContent
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ import (
|
||||||
// FallbackChain orchestrates model fallback across multiple candidates.
|
// FallbackChain orchestrates model fallback across multiple candidates.
|
||||||
type FallbackChain struct {
|
type FallbackChain struct {
|
||||||
cooldown *CooldownTracker
|
cooldown *CooldownTracker
|
||||||
|
lastResult *FallbackResult // stores the last successful result for usage tracking
|
||||||
}
|
}
|
||||||
|
|
||||||
// FallbackCandidate represents one model/provider to try.
|
// FallbackCandidate represents one model/provider to try.
|
||||||
|
|
@ -41,6 +42,15 @@ func NewFallbackChain(cooldown *CooldownTracker) *FallbackChain {
|
||||||
return &FallbackChain{cooldown: cooldown}
|
return &FallbackChain{cooldown: cooldown}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetLastResult returns the last successful fallback result for usage tracking.
|
||||||
|
// The first return value is the result, the second indicates if a result exists.
|
||||||
|
func (fc *FallbackChain) GetLastResult() (*FallbackResult, bool) {
|
||||||
|
if fc == nil || fc.lastResult == nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
return fc.lastResult, true
|
||||||
|
}
|
||||||
|
|
||||||
// ResolveCandidates parses model config into a deduplicated candidate list.
|
// ResolveCandidates parses model config into a deduplicated candidate list.
|
||||||
func ResolveCandidates(cfg ModelConfig, defaultProvider string) []FallbackCandidate {
|
func ResolveCandidates(cfg ModelConfig, defaultProvider string) []FallbackCandidate {
|
||||||
return ResolveCandidatesWithLookup(cfg, defaultProvider, nil)
|
return ResolveCandidatesWithLookup(cfg, defaultProvider, nil)
|
||||||
|
|
@ -147,6 +157,8 @@ func (fc *FallbackChain) Execute(
|
||||||
result.Response = resp
|
result.Response = resp
|
||||||
result.Provider = candidate.Provider
|
result.Provider = candidate.Provider
|
||||||
result.Model = candidate.Model
|
result.Model = candidate.Model
|
||||||
|
// Store last result for usage tracking
|
||||||
|
fc.lastResult = result
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -77,6 +77,7 @@ type Message struct {
|
||||||
type MessageExtra struct {
|
type MessageExtra struct {
|
||||||
Usage map[string]any `json:"usage,omitempty"` // token usage info
|
Usage map[string]any `json:"usage,omitempty"` // token usage info
|
||||||
Model string `json:"model,omitempty"` // model name used
|
Model string `json:"model,omitempty"` // model name used
|
||||||
|
Provider string `json:"provider,omitempty"` // provider name used
|
||||||
}
|
}
|
||||||
|
|
||||||
type ToolDefinition struct {
|
type ToolDefinition struct {
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ import (
|
||||||
"bufio"
|
"bufio"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
@ -24,6 +25,7 @@ func (h *Handler) registerUsageRoutes(mux *http.ServeMux) {
|
||||||
type UsageStats struct {
|
type UsageStats struct {
|
||||||
ModelName string `json:"model_name"`
|
ModelName string `json:"model_name"`
|
||||||
Model string `json:"model"`
|
Model string `json:"model"`
|
||||||
|
Provider string `json:"provider,omitempty"`
|
||||||
MessageCount int `json:"message_count"`
|
MessageCount int `json:"message_count"`
|
||||||
InputTokens int `json:"input_tokens"`
|
InputTokens int `json:"input_tokens"`
|
||||||
OutputTokens int `json:"output_tokens"`
|
OutputTokens int `json:"output_tokens"`
|
||||||
|
|
@ -239,14 +241,19 @@ type messageWithUsage struct {
|
||||||
ReasoningContent string `json:"reasoning_content,omitempty"`
|
ReasoningContent string `json:"reasoning_content,omitempty"`
|
||||||
ToolCalls []providers.ToolCall `json:"tool_calls,omitempty"`
|
ToolCalls []providers.ToolCall `json:"tool_calls,omitempty"`
|
||||||
ToolCallID string `json:"tool_call_id,omitempty"`
|
ToolCallID string `json:"tool_call_id,omitempty"`
|
||||||
// Usage fields stored by agent loop
|
// Usage fields stored by agent loop in extra_content
|
||||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
ExtraContent *messageExtraContent `json:"extra_content,omitempty"`
|
||||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
|
||||||
TotalTokens int `json:"total_tokens,omitempty"`
|
|
||||||
// Model info
|
// Model info
|
||||||
Model string `json:"model,omitempty"`
|
Model string `json:"model,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// messageExtraContent holds optional metadata attached to a message.
|
||||||
|
type messageExtraContent struct {
|
||||||
|
Usage map[string]any `json:"usage,omitempty"` // token usage info
|
||||||
|
Model string `json:"model,omitempty"` // model name used
|
||||||
|
Provider string `json:"provider,omitempty"` // provider name used
|
||||||
|
}
|
||||||
|
|
||||||
// handleGetUsage returns usage statistics aggregated by model.
|
// handleGetUsage returns usage statistics aggregated by model.
|
||||||
//
|
//
|
||||||
// GET /api/usage?start_date=2024-01-01&end_date=2024-01-31
|
// GET /api/usage?start_date=2024-01-01&end_date=2024-01-31
|
||||||
|
|
@ -255,14 +262,16 @@ func (h *Handler) handleGetUsage(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
dir, err := h.sessionsDir()
|
dir, err := h.sessionsDir()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, "failed to resolve sessions directory", http.StatusInternalServerError)
|
log.Printf("handleGetUsage: sessionsDir error: %v", err)
|
||||||
|
http.Error(w, fmt.Sprintf("failed to resolve sessions directory: %v", err), http.StatusInternalServerError)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load config to map model names to model identifiers
|
// Load config to map model names to model identifiers
|
||||||
cfg, err := config.LoadConfig(h.configPath)
|
cfg, err := config.LoadConfig(h.configPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, "failed to load config", http.StatusInternalServerError)
|
log.Printf("handleGetUsage: LoadConfig error: %v", err)
|
||||||
|
http.Error(w, fmt.Sprintf("failed to load config: %v", err), http.StatusInternalServerError)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -350,8 +359,14 @@ func (h *Handler) handleGetUsage(w http.ResponseWriter, r *http.Request) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Determine model name
|
// Determine model name - first try extra_content.model, then message.model (deprecated)
|
||||||
modelName := msgWithUsage.Model
|
modelName := ""
|
||||||
|
if msgWithUsage.ExtraContent != nil && msgWithUsage.ExtraContent.Model != "" {
|
||||||
|
modelName = msgWithUsage.ExtraContent.Model
|
||||||
|
}
|
||||||
|
if modelName == "" {
|
||||||
|
modelName = msgWithUsage.Model
|
||||||
|
}
|
||||||
if modelName == "" {
|
if modelName == "" {
|
||||||
modelName = defaultModelName
|
modelName = defaultModelName
|
||||||
}
|
}
|
||||||
|
|
@ -360,17 +375,39 @@ func (h *Handler) handleGetUsage(w http.ResponseWriter, r *http.Request) {
|
||||||
modelName = "unknown"
|
modelName = "unknown"
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, exists := statsByModel[modelName]; !exists {
|
// Get provider name from extra_content
|
||||||
statsByModel[modelName] = &modelStats{
|
providerName := ""
|
||||||
|
if msgWithUsage.ExtraContent != nil && msgWithUsage.ExtraContent.Provider != "" {
|
||||||
|
providerName = msgWithUsage.ExtraContent.Provider
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create composite key for model+provider combination
|
||||||
|
statsKey := modelName
|
||||||
|
if providerName != "" {
|
||||||
|
statsKey = providerName + "/" + modelName
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, exists := statsByModel[statsKey]; !exists {
|
||||||
|
statsByModel[statsKey] = &modelStats{
|
||||||
SessionKeys: make(map[string]struct{}),
|
SessionKeys: make(map[string]struct{}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ms := statsByModel[modelName]
|
ms := statsByModel[statsKey]
|
||||||
ms.MessageCount++
|
ms.MessageCount++
|
||||||
ms.InputTokens += msgWithUsage.PromptTokens
|
// Extract token usage from extra_content.usage
|
||||||
ms.OutputTokens += msgWithUsage.CompletionTokens
|
if msgWithUsage.ExtraContent != nil && msgWithUsage.ExtraContent.Usage != nil {
|
||||||
ms.TotalTokens += msgWithUsage.TotalTokens
|
usage := msgWithUsage.ExtraContent.Usage
|
||||||
|
if promptTokens, ok := usage["prompt_tokens"].(float64); ok {
|
||||||
|
ms.InputTokens += int(promptTokens)
|
||||||
|
}
|
||||||
|
if completionTokens, ok := usage["completion_tokens"].(float64); ok {
|
||||||
|
ms.OutputTokens += int(completionTokens)
|
||||||
|
}
|
||||||
|
if totalTokens, ok := usage["total_tokens"].(float64); ok {
|
||||||
|
ms.TotalTokens += int(totalTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
ms.SessionKeys[sess.Key] = struct{}{}
|
ms.SessionKeys[sess.Key] = struct{}{}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -380,23 +417,34 @@ func (h *Handler) handleGetUsage(w http.ResponseWriter, r *http.Request) {
|
||||||
var totalInputTokens, totalOutputTokens, totalTokens, totalMessageCount int
|
var totalInputTokens, totalOutputTokens, totalTokens, totalMessageCount int
|
||||||
var totalEstimatedCost float64
|
var totalEstimatedCost float64
|
||||||
|
|
||||||
for modelName, ms := range statsByModel {
|
for statsKey, ms := range statsByModel {
|
||||||
modelIdentifier := modelNameToModel[modelName]
|
// statsKey is either "model" or "provider/model"
|
||||||
if modelIdentifier == "" {
|
// Extract model name and provider for display
|
||||||
modelIdentifier = modelName
|
var modelIdentifier, providerIdentifier string
|
||||||
|
if strings.Contains(statsKey, "/") {
|
||||||
|
parts := strings.SplitN(statsKey, "/", 2)
|
||||||
|
providerIdentifier = parts[0]
|
||||||
|
modelIdentifier = parts[1]
|
||||||
|
} else {
|
||||||
|
modelIdentifier = statsKey
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to look up the full model name from config
|
||||||
|
if mappedModel := modelNameToModel[modelIdentifier]; mappedModel != "" {
|
||||||
|
modelIdentifier = mappedModel
|
||||||
}
|
}
|
||||||
|
|
||||||
pricing := getModelPricing(modelIdentifier)
|
pricing := getModelPricing(modelIdentifier)
|
||||||
if pricing.InputPricePerMTok == 0 && pricing.OutputPricePerMTok == 0 {
|
if pricing.InputPricePerMTok == 0 && pricing.OutputPricePerMTok == 0 {
|
||||||
// Try with model name as well
|
// Try with model name as well - modelIdentifier already contains the model name
|
||||||
pricing = getModelPricing(modelName)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
estimatedCost := calculateCost(ms.InputTokens, ms.OutputTokens, pricing)
|
estimatedCost := calculateCost(ms.InputTokens, ms.OutputTokens, pricing)
|
||||||
|
|
||||||
stat := UsageStats{
|
stat := UsageStats{
|
||||||
ModelName: modelName,
|
ModelName: modelIdentifier,
|
||||||
Model: modelIdentifier,
|
Model: modelIdentifier,
|
||||||
|
Provider: providerIdentifier,
|
||||||
MessageCount: ms.MessageCount,
|
MessageCount: ms.MessageCount,
|
||||||
InputTokens: ms.InputTokens,
|
InputTokens: ms.InputTokens,
|
||||||
OutputTokens: ms.OutputTokens,
|
OutputTokens: ms.OutputTokens,
|
||||||
|
|
|
||||||
|
|
@ -173,6 +173,8 @@ func isPublicLauncherDashboardPath(method, p string) bool {
|
||||||
return method == http.MethodPost
|
return method == http.MethodPost
|
||||||
case "/api/auth/status":
|
case "/api/auth/status":
|
||||||
return method == http.MethodGet
|
return method == http.MethodGet
|
||||||
|
case "/api/usage":
|
||||||
|
return method == http.MethodGet
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,7 @@ import { launcherFetch } from "@/api/http"
|
||||||
export interface UsageStats {
|
export interface UsageStats {
|
||||||
model_name: string
|
model_name: string
|
||||||
model: string
|
model: string
|
||||||
|
provider?: string
|
||||||
message_count: number
|
message_count: number
|
||||||
input_tokens: number
|
input_tokens: number
|
||||||
output_tokens: number
|
output_tokens: number
|
||||||
|
|
|
||||||
|
|
@ -248,9 +248,16 @@ export function UsagePage() {
|
||||||
>
|
>
|
||||||
<td className="py-3 px-4 text-sm font-medium">
|
<td className="py-3 px-4 text-sm font-medium">
|
||||||
<div>
|
<div>
|
||||||
<div>{stat.model_name}</div>
|
<div className="flex items-center gap-2">
|
||||||
|
<span>{stat.model_name}</span>
|
||||||
|
{stat.provider && (
|
||||||
|
<span className="inline-flex items-center rounded-full bg-muted px-2 py-0.5 text-[10px] font-normal text-muted-foreground uppercase tracking-wider">
|
||||||
|
{stat.provider}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
{stat.model !== stat.model_name && (
|
{stat.model !== stat.model_name && (
|
||||||
<div className="text-muted-foreground text-xs">
|
<div className="text-muted-foreground text-xs font-normal">
|
||||||
{stat.model}
|
{stat.model}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue