This commit is contained in:
afjcjsbx 2026-03-27 21:44:33 +01:00
parent 85711cce94
commit 8733fa2935

View file

@ -42,7 +42,7 @@ type Features struct {
// the returned struct. // the returned struct.
func ExtractFeatures(msg string, history []providers.Message) Features { func ExtractFeatures(msg string, history []providers.Message) Features {
return Features{ return Features{
TokenEstimate: EstimateTokens(msg), TokenEstimate: estimateTokens(msg),
CodeBlockCount: countCodeBlocks(msg), CodeBlockCount: countCodeBlocks(msg),
RecentToolCalls: countRecentToolCalls(history), RecentToolCalls: countRecentToolCalls(history),
ConversationDepth: len(history), ConversationDepth: len(history),
@ -50,12 +50,12 @@ func ExtractFeatures(msg string, history []providers.Message) Features {
} }
} }
// EstimateTokens returns a token count proxy that handles both CJK and Latin text. // estimateTokens returns a token count proxy that handles both CJK and Latin text.
// CJK runes (U+2E80U+9FFF, U+F900U+FAFF, U+AC00U+D7AF) map to roughly one // CJK runes (U+2E80U+9FFF, U+F900U+FAFF, U+AC00U+D7AF) map to roughly one
// token each, while non-CJK runes average ~0.25 tokens/rune (≈4 chars per token // token each, while non-CJK runes average ~0.25 tokens/rune (≈4 chars per token
// for English). Splitting the count this way avoids the 3x underestimation that a // for English). Splitting the count this way avoids the 3x underestimation that a
// flat rune_count/3 would produce for Chinese, Japanese, and Korean text. // flat rune_count/3 would produce for Chinese, Japanese, and Korean text.
func EstimateTokens(msg string) int { func estimateTokens(msg string) int {
total := utf8.RuneCountInString(msg) total := utf8.RuneCountInString(msg)
if total == 0 { if total == 0 {
return 0 return 0