fix: address PR review feedback for thinking support

- Add ThinkingCapable interface for provider capability detection
- Warn when thinking_level is set but provider doesn't support it
- Warn when temperature is cleared due to thinking enabled
- Adjust budget values per Anthropic best practices (medium=16K, xhigh=64K)
- Add budget clamp warning and 80% threshold warning
- Add parseResponse thinking block tests
- Add thinking_level field to config.example.json
This commit is contained in:
larrykoo 2026-03-04 20:34:49 +08:00
parent 1dc7ff753f
commit 6ed8f8cb64
5 changed files with 94 additions and 11 deletions

View file

@ -6,7 +6,8 @@
"model_name": "gpt4",
"max_tokens": 8192,
"temperature": 0.7,
"max_tool_iterations": 20
"max_tool_iterations": 20,
"thinking_level": ""
}
},
"model_list": [

View file

@ -779,7 +779,12 @@ func (al *AgentLoop) runLLMIteration(
// parseThinkingLevel guarantees ThinkingOff for empty/unknown values,
// so checking != ThinkingOff is sufficient.
if agent.ThinkingLevel != ThinkingOff {
if tc, ok := agent.Provider.(providers.ThinkingCapable); ok && tc.SupportsThinking() {
llmOpts["thinking_level"] = string(agent.ThinkingLevel)
} else {
logger.WarnCF("agent", "thinking_level is set but current provider does not support it, ignoring",
map[string]any{"agent_id": agent.ID, "thinking_level": string(agent.ThinkingLevel)})
}
}
callLLM := func() (*providers.LLMResponse, error) {

View file

@ -31,6 +31,9 @@ type Provider struct {
baseURL string
}
// SupportsThinking implements providers.ThinkingCapable.
func (p *Provider) SupportsThinking() bool { return true }
func NewProvider(token string) *Provider {
return NewProviderWithBaseURL(token, "")
}
@ -202,6 +205,9 @@ func buildParams(
func applyThinkingConfig(params *anthropic.MessageNewParams, level string) {
// Anthropic API rejects requests with temperature set alongside thinking.
// Reset to zero value (omitted from JSON serialization).
if params.Temperature.Valid() {
log.Printf("anthropic: temperature cleared because thinking is enabled (level=%s)", level)
}
params.Temperature = anthropic.MessageNewParams{}.Temperature
if level == "adaptive" {
@ -220,22 +226,34 @@ func applyThinkingConfig(params *anthropic.MessageNewParams, level string) {
// budget_tokens must be < max_tokens; clamp to respect user's max_tokens setting.
if budget >= params.MaxTokens {
log.Printf("anthropic: budget_tokens (%d) clamped to %d (max_tokens-1)", budget, params.MaxTokens-1)
budget = params.MaxTokens - 1
} else if budget > params.MaxTokens*80/100 {
log.Printf("anthropic: thinking budget (%d) exceeds 80%% of max_tokens (%d), output may be truncated",
budget, params.MaxTokens)
}
params.Thinking = anthropic.ThinkingConfigParamOfEnabled(budget)
}
// levelToBudget maps a thinking level string to budget_tokens for legacy models.
// levelToBudget maps a thinking level to budget_tokens.
// Values are based on Anthropic's recommendations and community best practices:
//
// low = 4,096 — simple reasoning, quick debugging (Claude Code "think")
// medium = 16,384 — Anthropic recommended sweet spot for most tasks
// high = 32,000 — complex architecture, deep analysis (diminishing returns above this)
// xhigh = 64,000 — extreme reasoning, research problems, benchmarks
//
// Note: For Claude 4.6+, prefer adaptive thinking over manual budget_tokens.
func levelToBudget(level string) int {
switch level {
case "low":
return 4096
case "medium":
return 10000
return 16384
case "high":
return 32000
case "xhigh":
return 128000
return 64000
default:
return 0
}

View file

@ -1,6 +1,7 @@
package anthropicprovider
import (
"encoding/json"
"testing"
"github.com/anthropics/anthropic-sdk-go"
@ -33,9 +34,9 @@ func TestApplyThinkingConfig_BudgetLevels(t *testing.T) {
wantBudget int64
}{
{"low", 4096},
{"medium", 10000},
{"medium", 16384},
{"high", 32000},
{"xhigh", 128000},
{"xhigh", 64000},
}
for _, tt := range tests {
@ -100,9 +101,9 @@ func TestLevelToBudget(t *testing.T) {
want int
}{
{"low", "low", 4096},
{"medium", "medium", 10000},
{"medium", "medium", 16384},
{"high", "high", 32000},
{"xhigh", "xhigh", 128000},
{"xhigh", "xhigh", 64000},
{"off", "off", 0},
{"empty", "", 0},
}
@ -135,8 +136,59 @@ func TestBuildParams_ThinkingClearsTemperature(t *testing.T) {
if params.Thinking.OfEnabled == nil {
t.Fatal("expected enabled thinking")
}
if params.Thinking.OfEnabled.BudgetTokens != 10000 {
t.Errorf("budget_tokens = %d, want 10000", params.Thinking.OfEnabled.BudgetTokens)
if params.Thinking.OfEnabled.BudgetTokens != 16384 {
t.Errorf("budget_tokens = %d, want 16384", params.Thinking.OfEnabled.BudgetTokens)
}
}
// unmarshalBlocks constructs []ContentBlockUnion via JSON round-trip so that
// the internal JSON.raw field is populated (required by AsText/AsThinking).
func unmarshalBlocks(t *testing.T, jsonStr string) []anthropic.ContentBlockUnion {
t.Helper()
var blocks []anthropic.ContentBlockUnion
if err := json.Unmarshal([]byte(jsonStr), &blocks); err != nil {
t.Fatalf("unmarshalBlocks: %v", err)
}
return blocks
}
func TestParseResponse_ThinkingBlock(t *testing.T) {
resp := &anthropic.Message{
Content: unmarshalBlocks(t, `[
{"type":"thinking","thinking":"Let me reason step by step...","signature":"sig"},
{"type":"text","text":"The answer is 42."}
]`),
StopReason: anthropic.StopReasonEndTurn,
}
result := parseResponse(resp)
if result.Reasoning != "Let me reason step by step..." {
t.Errorf("Reasoning = %q, want thinking content", result.Reasoning)
}
if result.Content != "The answer is 42." {
t.Errorf("Content = %q, want text content", result.Content)
}
if result.FinishReason != "stop" {
t.Errorf("FinishReason = %q, want stop", result.FinishReason)
}
}
func TestParseResponse_NoThinkingBlock(t *testing.T) {
resp := &anthropic.Message{
Content: unmarshalBlocks(t, `[
{"type":"text","text":"Just a normal response."}
]`),
StopReason: anthropic.StopReasonEndTurn,
}
result := parseResponse(resp)
if result.Reasoning != "" {
t.Errorf("Reasoning = %q, want empty", result.Reasoning)
}
if result.Content != "Just a normal response." {
t.Errorf("Content = %q, want text content", result.Content)
}
}

View file

@ -37,6 +37,13 @@ type StatefulProvider interface {
Close()
}
// ThinkingCapable is an optional interface for providers that support
// extended thinking (e.g. Anthropic). Used by the agent loop to warn
// when thinking_level is configured but the active provider cannot use it.
type ThinkingCapable interface {
SupportsThinking() bool
}
// FailoverReason classifies why an LLM request failed for fallback decisions.
type FailoverReason string