perf: align with readme-magno optimizations (timestamp, cache logging)
- Change system prompt timestamp to daily-only format (was HH:MM, invalidating dynamic block every minute unnecessarily) - Add CacheCreatedTokens and CacheReadTokens to UsageInfo - Extract Anthropic cache_creation/cache_read tokens in parseResponse - Log cache hit/miss stats per session for cost monitoring Made-with: Cursor
This commit is contained in:
parent
7359fb51a4
commit
7fe4deff0d
4 changed files with 21 additions and 13 deletions
|
|
@ -367,7 +367,7 @@ func (cb *ContextBuilder) LoadBootstrapFiles() string {
|
||||||
// See: https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching
|
// See: https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching
|
||||||
// See: https://platform.openai.com/docs/guides/prompt-caching
|
// See: https://platform.openai.com/docs/guides/prompt-caching
|
||||||
func (cb *ContextBuilder) buildDynamicContext(channel, chatID string) string {
|
func (cb *ContextBuilder) buildDynamicContext(channel, chatID string) string {
|
||||||
now := time.Now().Format("2006-01-02 15:04 (Monday)")
|
now := time.Now().Format("2006-01-02 (Monday)")
|
||||||
rt := fmt.Sprintf("%s %s, Go %s", runtime.GOOS, runtime.GOARCH, runtime.Version())
|
rt := fmt.Sprintf("%s %s, Go %s", runtime.GOOS, runtime.GOARCH, runtime.Version())
|
||||||
|
|
||||||
var sb strings.Builder
|
var sb strings.Builder
|
||||||
|
|
|
||||||
|
|
@ -661,7 +661,7 @@ func (al *AgentLoop) runLLMIteration(
|
||||||
) (string, int, error) {
|
) (string, int, error) {
|
||||||
iteration := 0
|
iteration := 0
|
||||||
var finalContent string
|
var finalContent string
|
||||||
var totalInputTokens, totalOutputTokens int
|
var totalInputTokens, totalOutputTokens, totalCacheCreated, totalCacheRead int
|
||||||
|
|
||||||
for iteration < agent.MaxIterations {
|
for iteration < agent.MaxIterations {
|
||||||
iteration++
|
iteration++
|
||||||
|
|
@ -809,6 +809,8 @@ func (al *AgentLoop) runLLMIteration(
|
||||||
if response.Usage != nil {
|
if response.Usage != nil {
|
||||||
totalInputTokens += response.Usage.PromptTokens
|
totalInputTokens += response.Usage.PromptTokens
|
||||||
totalOutputTokens += response.Usage.CompletionTokens
|
totalOutputTokens += response.Usage.CompletionTokens
|
||||||
|
totalCacheCreated += response.Usage.CacheCreatedTokens
|
||||||
|
totalCacheRead += response.Usage.CacheReadTokens
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.DebugCF("agent", "LLM response",
|
logger.DebugCF("agent", "LLM response",
|
||||||
|
|
@ -993,6 +995,8 @@ func (al *AgentLoop) runLLMIteration(
|
||||||
"input_tokens": totalInputTokens,
|
"input_tokens": totalInputTokens,
|
||||||
"output_tokens": totalOutputTokens,
|
"output_tokens": totalOutputTokens,
|
||||||
"total_tokens": totalInputTokens + totalOutputTokens,
|
"total_tokens": totalInputTokens + totalOutputTokens,
|
||||||
|
"cache_created": totalCacheCreated,
|
||||||
|
"cache_read": totalCacheRead,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -306,6 +306,8 @@ func parseResponse(resp *anthropic.Message) *LLMResponse {
|
||||||
PromptTokens: int(resp.Usage.InputTokens),
|
PromptTokens: int(resp.Usage.InputTokens),
|
||||||
CompletionTokens: int(resp.Usage.OutputTokens),
|
CompletionTokens: int(resp.Usage.OutputTokens),
|
||||||
TotalTokens: int(resp.Usage.InputTokens + resp.Usage.OutputTokens),
|
TotalTokens: int(resp.Usage.InputTokens + resp.Usage.OutputTokens),
|
||||||
|
CacheCreatedTokens: int(resp.Usage.CacheCreationInputTokens),
|
||||||
|
CacheReadTokens: int(resp.Usage.CacheReadInputTokens),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -45,6 +45,8 @@ type UsageInfo struct {
|
||||||
PromptTokens int `json:"prompt_tokens"`
|
PromptTokens int `json:"prompt_tokens"`
|
||||||
CompletionTokens int `json:"completion_tokens"`
|
CompletionTokens int `json:"completion_tokens"`
|
||||||
TotalTokens int `json:"total_tokens"`
|
TotalTokens int `json:"total_tokens"`
|
||||||
|
CacheCreatedTokens int `json:"cache_created_tokens,omitempty"`
|
||||||
|
CacheReadTokens int `json:"cache_read_tokens,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// CacheControl marks a content block for LLM-side prefix caching.
|
// CacheControl marks a content block for LLM-side prefix caching.
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue