opencode/yao
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add DeepSeek API configuration and enhance reasoning support in LLM adapters

Browse source 
- Added DeepSeek API keys and model configurations to the GitHub workflows for both unit and PR tests.
- Introduced a new `ReasoningEffort` parameter in the `CompletionOptions` struct to manage reasoning levels for models like o1 and GPT-5.
- Updated the `ReasoningAdapter` to handle the new `ReasoningEffort` parameter, ensuring it is stripped if not supported by the model.
- Enhanced the OpenAI provider to preprocess options through adapters, improving the handling of reasoning content and ensuring compatibility with DeepSeek R1 reasoning format.
This commit is contained in:
Max 2025-11-17 08:07:20 +08:00
parent 3d149057fd
commit 494c8dc98e
11 changed files with 1385 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.github/workflows/pr-test.yml vendored
View file

@ -42,6 +42,13 @@ env:
TEST_MOAPI_SECRET: ${{ secrets.OPENAI_TEST_KEY }}
TEST_MOAPI_MIRROR: https://api.openai.com
# DeepSeek API Configuration
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
DEEPSEEK_API_PROXY: ${{ secrets.DEEPSEEK_API_PROXY }}
DEEPSEEK_MODELS_R1: ${{ secrets.DEEPSEEK_MODELS_R1 }}
DEEPSEEK_MODELS_V3: ${{ secrets.DEEPSEEK_MODELS_V3 }}
DEEPSEEK_MODELS_V3_1: ${{ secrets.DEEPSEEK_MODELS_V3_1 }}
TAB_NAME: "::PET ADMIN"
PAGE_SIZE: "20"
PAGE_LINK: "https://yaoapps.com"

7
.github/workflows/unit-test.yml vendored
View file

@ -46,6 +46,13 @@ env:
TEST_MOAPI_SECRET: ${{ secrets.OPENAI_TEST_KEY }}
TEST_MOAPI_MIRROR: https://api.openai.com
# DeepSeek API Configuration
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
DEEPSEEK_API_PROXY: ${{ secrets.DEEPSEEK_API_PROXY }}
DEEPSEEK_MODELS_R1: ${{ secrets.DEEPSEEK_MODELS_R1 }}
DEEPSEEK_MODELS_V3: ${{ secrets.DEEPSEEK_MODELS_V3 }}
DEEPSEEK_MODELS_V3_1: ${{ secrets.DEEPSEEK_MODELS_V3_1 }}
TAB_NAME: "::PET ADMIN"
PAGE_SIZE: "20"
PAGE_LINK: "https://yaoapps.com"

3
agent/context/types_llm.go
View file

@ -61,6 +61,9 @@ type CompletionOptions struct {
Stream *bool `json:"stream,omitempty"` // If true, stream partial message deltas
StreamOptions *StreamOptions `json:"stream_options,omitempty"` // Options for streaming response
// Reasoning configuration (for reasoning models like o1, GPT-5)
ReasoningEffort *string `json:"reasoning_effort,omitempty"` // Reasoning effort level: "low", "medium", "high" (o1 and GPT-5 only)
// CUI Context information (from Context)
Route string `json:"route,omitempty"` // Route of the request for CUI context
Metadata map[string]interface{} `json:"metadata,omitempty"` // Metadata to pass to the page for CUI context

63
agent/llm/adapters/reasoning.go
View file

@ -9,24 +9,53 @@ type ReasoningFormat string
const (
ReasoningFormatNone ReasoningFormat = "none" // No reasoning support
ReasoningFormatOpenAI ReasoningFormat = "openai-o1" // OpenAI o1 format
ReasoningFormatDeepSeek ReasoningFormat = "deepseek-r1" // DeepSeek R1 format
ReasoningFormatGPTThink ReasoningFormat = "gpt-think" // Future GPT with thinking
ReasoningFormatOpenAI ReasoningFormat = "openai-o1" // OpenAI o1 format (hidden reasoning)
ReasoningFormatGPT5 ReasoningFormat = "gpt-5" // GPT-5 format (hidden reasoning)
ReasoningFormatDeepSeek ReasoningFormat = "deepseek-r1" // DeepSeek R1 format (visible reasoning)
)
// ReasoningAdapter handles reasoning content capability
// Parses reasoning_content from different model formats
// - Manages reasoning_effort parameter (o1, GPT-5)
// - Extracts reasoning_tokens from usage
// - Parses visible reasoning content (DeepSeek R1)
type ReasoningAdapter struct {
*BaseAdapter
format ReasoningFormat
format ReasoningFormat
supportsEffort bool // Whether the model supports reasoning_effort parameter
}
// NewReasoningAdapter creates a new reasoning adapter
func NewReasoningAdapter(format ReasoningFormat) *ReasoningAdapter {
return &ReasoningAdapter{
BaseAdapter: NewBaseAdapter("ReasoningAdapter"),
format: format,
supportsEffort := false
// Only OpenAI o1 and GPT-5 support reasoning_effort parameter
if format == ReasoningFormatOpenAI || format == ReasoningFormatGPT5 {
supportsEffort = true
}
return &ReasoningAdapter{
BaseAdapter: NewBaseAdapter("ReasoningAdapter"),
format: format,
supportsEffort: supportsEffort,
}
}
// PreprocessOptions handles reasoning_effort parameter
func (a *ReasoningAdapter) PreprocessOptions(options *context.CompletionOptions) (*context.CompletionOptions, error) {
if options == nil {
return options, nil
}
// If model doesn't support reasoning_effort, remove it
if !a.supportsEffort && options.ReasoningEffort != nil {
// Model doesn't support reasoning_effort, remove the parameter
newOptions := *options
newOptions.ReasoningEffort = nil
return &newOptions, nil
}
// If model supports reasoning_effort, keep it as-is (user can set "low", "medium", or "high")
return options, nil
}
// ProcessStreamChunk processes streaming chunks with reasoning content
@ -37,23 +66,27 @@ func (a *ReasoningAdapter) ProcessStreamChunk(chunkType context.StreamChunkType,
}
// TODO: Parse reasoning_content based on format
// - OpenAI o1: reasoning_content field in delta
// - DeepSeek R1: may have different format
// - Extract and emit as ChunkThinking
// - OpenAI o1: No visible reasoning in stream (reasoning happens internally)
// - GPT-5: No visible reasoning in stream (reasoning happens internally)
// - DeepSeek R1: May have <think>...</think> tags or reasoning_content field
return chunkType, data, nil
}
// PostprocessResponse extracts reasoning content from the final response
// PostprocessResponse extracts reasoning content and tokens from the final response
func (a *ReasoningAdapter) PostprocessResponse(response *context.CompletionResponse) (*context.CompletionResponse, error) {
if a.format == ReasoningFormatNone {
// No reasoning support
return response, nil
}
// TODO: Extract reasoning content from response
// - Set response.ReasoningContent if present
// - Separate thinking from final answer
// Reasoning tokens are already extracted in Usage.CompletionTokensDetails.ReasoningTokens
// by the OpenAI response parser, no additional processing needed for o1/GPT-5
// TODO: For DeepSeek R1, extract visible reasoning content
// - Parse <think>...</think> tags from content
// - Set response.ReasoningContent
// - Remove <think> tags from response.Content (keep only final answer)
return response, nil
}

1
agent/llm/adapters/toolcall.go
View file

@ -64,4 +64,3 @@ func (a *ToolCallAdapter) PostprocessResponse(response *context.CompletionRespon
// - Add to response.ToolCalls
return response, nil
}

6
agent/llm/providers/factory.go
View file

@ -77,9 +77,9 @@ func DetectAPIFormat(conn connector.Connector) string {
// contains checks if a string contains a substring (case-insensitive helper)
func contains(s, substr string) bool {
return len(s) >= len(substr) && (s == substr || len(s) > len(substr) &&
(s[:len(substr)] == substr || s[len(s)-len(substr):] == substr ||
findSubstring(s, substr)))
return len(s) >= len(substr) && (s == substr || len(s) > len(substr) &&
(s[:len(substr)] == substr || s[len(s)-len(substr):] == substr ||
findSubstring(s, substr)))
}
func findSubstring(s, substr string) bool {

383
agent/llm/providers/openai/deepseek_r1_test.go Normal file
View file

@ -0,0 +1,383 @@
package openai_test
import (
gocontext "context"
"strings"
"testing"
"github.com/yaoapp/gou/connector"
"github.com/yaoapp/gou/plan"
"github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/llm"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/openapi/oauth/types"
"github.com/yaoapp/yao/test"
)
// TestDeepSeekR1StreamBasic tests basic streaming completion with DeepSeek R1
func TestDeepSeekR1StreamBasic(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Create connector from real configuration
conn, err := connector.Select("deepseek.r1")
if err != nil {
t.Fatalf("Failed to select connector: %v", err)
}
// Create LLM instance with capabilities
trueVal := true
falseVal := false
options := &context.CompletionOptions{
Capabilities: &context.ModelCapabilities{
Streaming: &trueVal,
Reasoning: &trueVal, // DeepSeek R1 supports reasoning
ToolCalls: &falseVal, // R1 doesn't support native tool calls
Vision: &falseVal,
Audio: &falseVal,
Multimodal: &falseVal,
},
}
llmInstance, err := llm.New(conn, options)
if err != nil {
t.Fatalf("Failed to create LLM instance: %v", err)
}
// Prepare messages with reasoning prompt (simple question for faster reasoning)
messages := []context.Message{
{
Role: context.RoleUser,
Content: "What is 2 + 2?",
},
}
// Set max tokens (higher for reasoning models to allow full reasoning + answer)
maxTokens := 500
options.MaxTokens = &maxTokens
// Create context
ctx := newDeepSeekTestContext("test-deepseek-r1-basic", "deepseek.r1")
// Track streaming chunks
var reasoningChunks []string
var contentChunks []string
handler := func(chunkType context.StreamChunkType, data []byte) int {
dataStr := string(data)
t.Logf("Stream chunk [%s]: %s", chunkType, dataStr)
// Track different chunk types
if chunkType == context.ChunkThinking {
reasoningChunks = append(reasoningChunks, dataStr)
} else if chunkType == context.ChunkText {
contentChunks = append(contentChunks, dataStr)
}
return 0 // Continue
}
// Call Stream
response, err := llmInstance.Stream(ctx, messages, options, handler)
if err != nil {
t.Fatalf("Stream failed: %v", err)
}
// Validate response
if response == nil {
t.Fatal("Response is nil")
}
if response.ID == "" {
t.Error("Response ID is empty")
}
if response.Model == "" {
t.Error("Response Model is empty")
}
if response.Content == "" {
t.Error("Response content is empty")
}
if response.FinishReason == "" {
t.Error("FinishReason is empty")
}
// DeepSeek R1 should have reasoning content
if response.ReasoningContent == "" {
t.Error("Expected reasoning_content but got empty")
} else {
t.Logf("Reasoning content length: %d characters", len(response.ReasoningContent))
}
// Check reasoning tokens in usage
if response.Usage == nil {
t.Error("Response Usage is nil")
} else {
if response.Usage.TotalTokens == 0 {
t.Error("Response Usage.TotalTokens is 0")
}
if response.Usage.CompletionTokensDetails != nil {
if response.Usage.CompletionTokensDetails.ReasoningTokens == 0 {
t.Error("Expected reasoning_tokens > 0 for DeepSeek R1")
} else {
t.Logf("Reasoning tokens: %d", response.Usage.CompletionTokensDetails.ReasoningTokens)
}
}
t.Logf("Usage: prompt=%d, completion=%d, total=%d",
response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens)
}
// Should have received reasoning chunks
if len(reasoningChunks) == 0 {
t.Error("Expected reasoning chunks (ChunkThinking) but got none")
} else {
t.Logf("Received %d reasoning chunks", len(reasoningChunks))
}
// Should have received content chunks
if len(contentChunks) == 0 {
t.Error("Expected content chunks (ChunkText) but got none")
} else {
t.Logf("Received %d content chunks", len(contentChunks))
}
t.Logf("Final response: %+v", response)
}
// TestDeepSeekR1PostBasic tests basic non-streaming completion
func TestDeepSeekR1PostBasic(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Create connector
conn, err := connector.Select("deepseek.r1")
if err != nil {
t.Fatalf("Failed to select connector: %v", err)
}
// Create LLM instance
trueVal := true
falseVal := false
options := &context.CompletionOptions{
Capabilities: &context.ModelCapabilities{
Reasoning: &trueVal,
ToolCalls: &falseVal,
Vision: &falseVal,
Audio: &falseVal,
Multimodal: &falseVal,
},
}
llmInstance, err := llm.New(conn, options)
if err != nil {
t.Fatalf("Failed to create LLM instance: %v", err)
}
// Prepare messages (very simple question)
messages := []context.Message{
{
Role: context.RoleUser,
Content: "What is 1+1?",
},
}
// Set max tokens (enough for reasoning + answer)
maxTokens := 500
options.MaxTokens = &maxTokens
// Create context
ctx := newDeepSeekTestContext("test-deepseek-r1-post", "deepseek.r1")
// Call Post
response, err := llmInstance.Post(ctx, messages, options)
if err != nil {
t.Fatalf("Post failed: %v", err)
}
// Validate response
if response == nil {
t.Fatal("Response is nil")
}
if response.ID == "" {
t.Error("Response ID is empty")
}
if response.Model == "" {
t.Error("Response Model is empty")
}
if response.Content == "" {
t.Error("Response content is empty")
}
// DeepSeek R1 should have reasoning content
if response.ReasoningContent == "" {
t.Error("Expected reasoning_content but got empty")
} else {
t.Logf("Reasoning content: %s", response.ReasoningContent)
t.Logf("Final answer: %s", response.Content)
}
// Check usage
if response.Usage == nil {
t.Error("Response Usage is nil")
} else {
if response.Usage.TotalTokens == 0 {
t.Error("Response Usage.TotalTokens is 0")
}
if response.Usage.CompletionTokensDetails != nil && response.Usage.CompletionTokensDetails.ReasoningTokens > 0 {
t.Logf("Reasoning tokens: %d", response.Usage.CompletionTokensDetails.ReasoningTokens)
}
t.Logf("Usage: prompt=%d, completion=%d, total=%d",
response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens)
}
t.Logf("Response: %+v", response)
}
// TestDeepSeekR1LogicPuzzle tests DeepSeek R1's reasoning with a logic puzzle
func TestDeepSeekR1LogicPuzzle(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
conn, err := connector.Select("deepseek.r1")
if err != nil {
t.Fatalf("Failed to select connector: %v", err)
}
trueVal := true
falseVal := false
options := &context.CompletionOptions{
Capabilities: &context.ModelCapabilities{
Streaming: &trueVal,
Reasoning: &trueVal,
ToolCalls: &falseVal,
Vision: &falseVal,
Audio: &falseVal,
Multimodal: &falseVal,
},
}
maxTokens := 800
options.MaxTokens = &maxTokens
llmInstance, err := llm.New(conn, options)
if err != nil {
t.Fatalf("Failed to create LLM instance: %v", err)
}
// Use a simpler logic question
messages := []context.Message{
{
Role: context.RoleUser,
Content: "Is 5 greater than 3? Explain your reasoning.",
},
}
ctx := newDeepSeekTestContext("test-deepseek-r1-logic", "deepseek.r1")
// Track reasoning and content separately
var hasReasoning, hasContent bool
handler := func(chunkType context.StreamChunkType, data []byte) int {
if chunkType == context.ChunkThinking && len(data) > 0 {
hasReasoning = true
} else if chunkType == context.ChunkText && len(data) > 0 {
hasContent = true
}
return 0
}
response, err := llmInstance.Stream(ctx, messages, options, handler)
if err != nil {
t.Fatalf("Stream failed: %v", err)
}
if response == nil {
t.Fatal("Response is nil")
}
// Should have both reasoning and content
if !hasReasoning {
t.Error("Expected to receive reasoning chunks but didn't")
}
if !hasContent {
t.Error("Expected to receive content chunks but didn't")
}
// Validate reasoning content exists and is substantial
if response.ReasoningContent == "" {
t.Error("Expected reasoning_content but got empty")
} else if len(response.ReasoningContent) < 50 {
t.Errorf("Reasoning content too short (%d chars), expected detailed thinking", len(response.ReasoningContent))
} else {
t.Logf("✓ Reasoning content length: %d characters", len(response.ReasoningContent))
}
// Validate final answer
contentStr := ""
if response.Content != nil {
if str, ok := response.Content.(string); ok {
contentStr = str
}
}
if len(contentStr) == 0 {
t.Error("Content is empty")
} else {
// Should mention "Yes" or affirm that 5 > 3
if !strings.Contains(strings.ToLower(contentStr), "yes") && !strings.Contains(strings.ToLower(contentStr), "greater") {
t.Logf("Warning: Content might not contain expected answer. Content: %s", contentStr)
} else {
t.Logf("✓ Final answer: %s", contentStr)
}
}
// Check reasoning tokens
if response.Usage != nil && response.Usage.CompletionTokensDetails != nil {
reasoningTokens := response.Usage.CompletionTokensDetails.ReasoningTokens
if reasoningTokens == 0 {
t.Error("Expected reasoning_tokens > 0")
} else {
t.Logf("✓ Reasoning tokens: %d", reasoningTokens)
}
}
t.Log("Logic puzzle test completed successfully")
}
// ============================================================================
// Helper Functions
// ============================================================================
// newDeepSeekTestContext creates a real Context for testing DeepSeek provider
func newDeepSeekTestContext(chatID, connectorID string) *context.Context {
return &context.Context{
Context: gocontext.Background(),
Space: plan.NewMemorySharedSpace(),
ChatID: chatID,
AssistantID: "test-assistant",
Connector: connectorID,
Locale: "en-us",
Theme: "light",
Client: context.Client{
Type: "web",
UserAgent: "DeepSeekProviderTest/1.0",
IP: "127.0.0.1",
},
Referer: context.RefererAPI,
Accept: context.AcceptStandard,
Route: "/api/test",
Metadata: make(map[string]interface{}),
Authorized: &types.AuthorizedInfo{
Subject: "test-user",
ClientID: "test-client",
UserID: "test-user-123",
TeamID: "test-team-456",
TenantID: "test-tenant-789",
SessionID: "test-session-id",
Constraints: types.DataConstraints{
TeamOnly: true,
Extra: map[string]interface{}{
"test": "deepseek-provider",
},
},
},
}
}

405
agent/llm/providers/openai/deepseek_v3_test.go Normal file
View file

@ -0,0 +1,405 @@
package openai_test
import (
gocontext "context"
"testing"
"github.com/yaoapp/gou/connector"
"github.com/yaoapp/gou/plan"
"github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/llm"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/openapi/oauth/types"
"github.com/yaoapp/yao/test"
)
// TestDeepSeekV3StreamBasic tests basic streaming completion with DeepSeek V3
func TestDeepSeekV3StreamBasic(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
conn, err := connector.Select("deepseek.v3")
if err != nil {
t.Fatalf("Failed to select connector: %v", err)
}
trueVal := true
falseVal := false
options := &context.CompletionOptions{
Capabilities: &context.ModelCapabilities{
Streaming: &trueVal,
Reasoning: &falseVal, // V3 doesn't support reasoning
ToolCalls: &trueVal, // V3 supports tool calls
Vision: &falseVal,
Audio: &falseVal,
Multimodal: &falseVal,
},
}
llmInstance, err := llm.New(conn, options)
if err != nil {
t.Fatalf("Failed to create LLM instance: %v", err)
}
// Simple math question
messages := []context.Message{
{
Role: context.RoleUser,
Content: "What is 5 + 3?",
},
}
// Set max tokens
maxTokens := 100
options.MaxTokens = &maxTokens
ctx := newDeepSeekV3TestContext("test-deepseek-v3-basic", "deepseek.v3")
// Track streaming chunks
var contentChunks []string
handler := func(chunkType context.StreamChunkType, data []byte) int {
dataStr := string(data)
t.Logf("Stream chunk [%s]: %s", chunkType, dataStr)
if chunkType == context.ChunkText {
contentChunks = append(contentChunks, dataStr)
}
return 0 // Continue
}
// Call Stream
response, err := llmInstance.Stream(ctx, messages, options, handler)
if err != nil {
t.Fatalf("Stream failed: %v", err)
}
// Validate response
if response == nil {
t.Fatal("Response is nil")
}
if response.ID == "" {
t.Error("Response ID is empty")
}
if response.Model == "" {
t.Error("Response Model is empty")
}
// Should have content (V3 is not a reasoning model)
contentStr, ok := response.Content.(string)
if !ok || contentStr == "" {
t.Error("Expected content but got empty")
} else {
t.Logf("Response content: %s", contentStr)
}
// Should NOT have reasoning content (V3 doesn't support reasoning)
if response.ReasoningContent != "" {
t.Errorf("Expected no reasoning_content for V3, but got: %s", response.ReasoningContent)
}
// Check usage
if response.Usage == nil {
t.Error("Response Usage is nil")
} else {
t.Logf("Usage: prompt=%d, completion=%d, total=%d",
response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens)
// Should have 0 reasoning tokens
if response.Usage.CompletionTokensDetails != nil {
reasoningTokens := response.Usage.CompletionTokensDetails.ReasoningTokens
if reasoningTokens != 0 {
t.Errorf("Expected reasoning_tokens=0 for V3, got %d", reasoningTokens)
}
}
}
if len(contentChunks) == 0 {
t.Error("Expected content chunks but got none")
} else {
t.Logf("Received %d content chunks", len(contentChunks))
}
t.Logf("Final response: %+v", response)
}
// TestDeepSeekV3PostBasic tests basic non-streaming completion
func TestDeepSeekV3PostBasic(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
conn, err := connector.Select("deepseek.v3")
if err != nil {
t.Fatalf("Failed to select connector: %v", err)
}
trueVal := true
falseVal := false
options := &context.CompletionOptions{
Capabilities: &context.ModelCapabilities{
Reasoning: &falseVal,
ToolCalls: &trueVal,
Vision: &falseVal,
Audio: &falseVal,
Multimodal: &falseVal,
},
}
llmInstance, err := llm.New(conn, options)
if err != nil {
t.Fatalf("Failed to create LLM instance: %v", err)
}
// Simple question
messages := []context.Message{
{
Role: context.RoleUser,
Content: "What is 2 * 4?",
},
}
// Set max tokens
maxTokens := 100
options.MaxTokens = &maxTokens
ctx := newDeepSeekV3TestContext("test-deepseek-v3-post", "deepseek.v3")
// Call Post
response, err := llmInstance.Post(ctx, messages, options)
if err != nil {
t.Fatalf("Post failed: %v", err)
}
// Validate response
if response == nil {
t.Fatal("Response is nil")
}
if response.ID == "" {
t.Error("Response ID is empty")
}
if response.Model == "" {
t.Error("Response Model is empty")
}
// Should have content
contentStr, ok := response.Content.(string)
if !ok || contentStr == "" {
t.Error("Expected content but got empty")
} else {
t.Logf("Response content: %s", contentStr)
}
// Should NOT have reasoning content
if response.ReasoningContent != "" {
t.Errorf("V3 should not have reasoning_content, but got: %s", response.ReasoningContent)
}
// Check usage
if response.Usage == nil {
t.Error("Response Usage is nil")
} else {
t.Logf("Usage: prompt=%d, completion=%d, total=%d",
response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens)
// Should have 0 reasoning tokens
if response.Usage.CompletionTokensDetails != nil {
reasoningTokens := response.Usage.CompletionTokensDetails.ReasoningTokens
if reasoningTokens != 0 {
t.Errorf("Expected reasoning_tokens=0 for V3, got %d", reasoningTokens)
}
}
}
t.Logf("Response: %+v", response)
}
// TestDeepSeekV3WithToolCalls tests V3 with tool calls
func TestDeepSeekV3WithToolCalls(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
conn, err := connector.Select("deepseek.v3")
if err != nil {
t.Fatalf("Failed to select connector: %v", err)
}
trueVal := true
falseVal := false
options := &context.CompletionOptions{
Capabilities: &context.ModelCapabilities{
Reasoning: &falseVal,
ToolCalls: &trueVal,
},
}
// Define a weather tool
weatherTool := map[string]interface{}{
"type": "function",
"function": map[string]interface{}{
"name": "get_weather",
"description": "Get current weather for a location",
"parameters": map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"location": map[string]interface{}{
"type": "string",
"description": "City name",
},
"unit": map[string]interface{}{
"type": "string",
"enum": []string{"celsius", "fahrenheit"},
},
},
"required": []string{"location"},
},
},
}
options.Tools = []map[string]interface{}{weatherTool}
options.ToolChoice = "auto"
llmInstance, err := llm.New(conn, options)
if err != nil {
t.Fatalf("Failed to create LLM instance: %v", err)
}
messages := []context.Message{
{
Role: context.RoleUser,
Content: "What's the weather in Beijing?",
},
}
ctx := newDeepSeekV3TestContext("test-deepseek-v3-tools", "deepseek.v3")
response, err := llmInstance.Post(ctx, messages, options)
if err != nil {
t.Fatalf("Post with tool calls failed: %v", err)
}
if response == nil {
t.Fatal("Response is nil")
}
// Should have tool calls
if len(response.ToolCalls) == 0 {
t.Error("Expected tool calls but got none")
} else {
tc := response.ToolCalls[0]
t.Logf("✓ Tool call: %s(%s)", tc.Function.Name, tc.Function.Arguments)
if tc.Function.Name != "get_weather" {
t.Errorf("Expected tool name 'get_weather', got '%s'", tc.Function.Name)
}
}
if response.Usage != nil {
t.Logf("Usage: prompt=%d, completion=%d, total=%d",
response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens)
}
t.Logf("Response: %+v", response)
}
// TestDeepSeekV3NoReasoningEffort tests that V3 ignores reasoning_effort parameter
func TestDeepSeekV3NoReasoningEffort(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
conn, err := connector.Select("deepseek.v3")
if err != nil {
t.Fatalf("Failed to select connector: %v", err)
}
trueVal := true
falseVal := false
effort := "high"
options := &context.CompletionOptions{
Capabilities: &context.ModelCapabilities{
Reasoning: &falseVal, // V3 doesn't support reasoning
ToolCalls: &trueVal,
},
ReasoningEffort: &effort, // Should be ignored by adapter
}
llmInstance, err := llm.New(conn, options)
if err != nil {
t.Fatalf("Failed to create LLM instance: %v", err)
}
messages := []context.Message{
{
Role: context.RoleUser,
Content: "Say 'OK'",
},
}
maxTokens := 10
options.MaxTokens = &maxTokens
ctx := newDeepSeekV3TestContext("test-deepseek-v3-no-reasoning", "deepseek.v3")
// Should succeed (adapter removes reasoning_effort parameter)
response, err := llmInstance.Post(ctx, messages, options)
if err != nil {
t.Fatalf("Post failed: %v", err)
}
if response == nil {
t.Fatal("Response is nil")
}
// Should have 0 reasoning tokens
if response.Usage != nil && response.Usage.CompletionTokensDetails != nil {
reasoningTokens := response.Usage.CompletionTokensDetails.ReasoningTokens
if reasoningTokens != 0 {
t.Errorf("Expected reasoning_tokens=0 for V3, got %d", reasoningTokens)
} else {
t.Log("✓ V3 correctly shows reasoning_tokens=0")
}
}
t.Log("✓ ReasoningAdapter correctly removed reasoning_effort parameter for V3")
}
// ============================================================================
// Helper Functions
// ============================================================================
// newDeepSeekV3TestContext creates a real Context for testing DeepSeek V3 provider
func newDeepSeekV3TestContext(chatID, connectorID string) *context.Context {
return &context.Context{
Context: gocontext.Background(),
Space: plan.NewMemorySharedSpace(),
ChatID: chatID,
AssistantID: "test-assistant",
Connector: connectorID,
Locale: "en-us",
Theme: "light",
Client: context.Client{
Type: "web",
UserAgent: "DeepSeekV3ProviderTest/1.0",
IP: "127.0.0.1",
},
Referer: context.RefererAPI,
Accept: context.AcceptStandard,
Route: "/api/test",
Metadata: make(map[string]interface{}),
Authorized: &types.AuthorizedInfo{
Subject: "test-user",
ClientID: "test-client",
UserID: "test-user-123",
TeamID: "test-team-456",
TenantID: "test-tenant-789",
SessionID: "test-session-id",
Constraints: types.DataConstraints{
TeamOnly: true,
Extra: map[string]interface{}{
"test": "deepseek-v3-provider",
},
},
},
}
}

422
agent/llm/providers/openai/gpt5_test.go Normal file
View file

@ -0,0 +1,422 @@
package openai_test
import (
gocontext "context"
"testing"
"github.com/yaoapp/gou/connector"
"github.com/yaoapp/gou/plan"
"github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/llm"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/openapi/oauth/types"
"github.com/yaoapp/yao/test"
)
// TestGPT5StreamBasic tests basic streaming completion with GPT-5
func TestGPT5StreamBasic(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
conn, err := connector.Select("openai.gpt-5")
if err != nil {
t.Fatalf("Failed to select connector: %v", err)
}
trueVal := true
options := &context.CompletionOptions{
Capabilities: &context.ModelCapabilities{
Streaming: &trueVal,
Reasoning: &trueVal, // GPT-5 supports reasoning
ToolCalls: &trueVal,
Vision: &trueVal,
Multimodal: &trueVal,
},
}
llmInstance, err := llm.New(conn, options)
if err != nil {
t.Fatalf("Failed to create LLM instance: %v", err)
}
messages := []context.Message{
{
Role: context.RoleUser,
Content: "What is 1+1? Reply with just the number.",
},
}
maxTokens := 100
options.MaxCompletionTokens = &maxTokens
ctx := newGPT5TestContext("test-gpt5-basic", "openai.gpt-5")
var chunks []string
handler := func(chunkType context.StreamChunkType, data []byte) int {
chunks = append(chunks, string(data))
t.Logf("Stream chunk [%s]: %s", chunkType, string(data))
return 0
}
response, err := llmInstance.Stream(ctx, messages, options, handler)
if err != nil {
t.Fatalf("Stream failed: %v", err)
}
if response == nil {
t.Fatal("Response is nil")
}
// Basic validation
if response.ID == "" {
t.Error("Response ID is empty")
}
if response.Model == "" {
t.Error("Response Model is empty")
}
// GPT-5 may use all tokens for reasoning, so content could be empty
// Just log the content instead of failing
t.Logf("Response content: %v", response.Content)
t.Logf("Usage: prompt=%d, completion=%d, total=%d",
response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens)
if response.Usage != nil && response.Usage.CompletionTokensDetails != nil {
t.Logf("Reasoning tokens: %d", response.Usage.CompletionTokensDetails.ReasoningTokens)
}
t.Logf("Final response: %+v", response)
t.Logf("Total chunks received: %d", len(chunks))
}
// TestGPT5ReasoningEffort tests reasoning_effort parameter with different levels
func TestGPT5ReasoningEffort(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
conn, err := connector.Select("openai.gpt-5")
if err != nil {
t.Fatalf("Failed to select connector: %v", err)
}
// Test with different reasoning effort levels
effortLevels := []string{"low", "medium", "high"}
for _, effort := range effortLevels {
t.Run("effort_"+effort, func(t *testing.T) {
trueVal := true
options := &context.CompletionOptions{
Capabilities: &context.ModelCapabilities{
Reasoning: &trueVal,
ToolCalls: &trueVal,
},
ReasoningEffort: &effort,
}
llmInstance, err := llm.New(conn, options)
if err != nil {
t.Fatalf("Failed to create LLM instance: %v", err)
}
messages := []context.Message{
{
Role: context.RoleUser,
Content: "Solve: If all Bloops are Razzies and all Razzies are Lazzies, are all Bloops Lazzies?",
},
}
maxTokens := 1000
options.MaxCompletionTokens = &maxTokens
ctx := newGPT5TestContext("test-gpt5-reasoning-"+effort, "openai.gpt-5")
response, err := llmInstance.Post(ctx, messages, options)
if err != nil {
t.Fatalf("Post failed with effort=%s: %v", effort, err)
}
if response == nil {
t.Fatal("Response is nil")
}
// Check reasoning tokens
var reasoningTokens int
if response.Usage != nil && response.Usage.CompletionTokensDetails != nil {
reasoningTokens = response.Usage.CompletionTokensDetails.ReasoningTokens
}
t.Logf("Reasoning effort: %s", effort)
t.Logf("Reasoning tokens: %d", reasoningTokens)
t.Logf("Total tokens: %d", response.Usage.TotalTokens)
t.Logf("Content: %s", response.Content)
// GPT-5 reasoning is hidden (no reasoning_content field)
// But should have reasoning_tokens in usage
if effort != "low" {
if reasoningTokens == 0 {
t.Logf("Warning: Expected reasoning_tokens > 0 for effort='%s', got 0", effort)
}
}
})
}
}
// TestGPT5PostWithToolCalls tests GPT-5 with tool calls
func TestGPT5PostWithToolCalls(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
conn, err := connector.Select("openai.gpt-5")
if err != nil {
t.Fatalf("Failed to select connector: %v", err)
}
trueVal := true
options := &context.CompletionOptions{
Capabilities: &context.ModelCapabilities{
Reasoning: &trueVal,
ToolCalls: &trueVal,
},
}
// Define a calculation tool
calcTool := map[string]interface{}{
"type": "function",
"function": map[string]interface{}{
"name": "calculate",
"description": "Perform a mathematical calculation",
"parameters": map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"expression": map[string]interface{}{
"type": "string",
"description": "The mathematical expression to evaluate",
},
},
"required": []string{"expression"},
},
},
}
options.Tools = []map[string]interface{}{calcTool}
options.ToolChoice = "auto"
llmInstance, err := llm.New(conn, options)
if err != nil {
t.Fatalf("Failed to create LLM instance: %v", err)
}
messages := []context.Message{
{
Role: context.RoleUser,
Content: "Use the calculate function to compute 2 * 3",
},
}
ctx := newGPT5TestContext("test-gpt5-tools", "openai.gpt-5")
response, err := llmInstance.Post(ctx, messages, options)
if err != nil {
t.Fatalf("Post with tool calls failed: %v", err)
}
if response == nil {
t.Fatal("Response is nil")
}
// GPT-5 reasoning models may not always use tool calls
// Log what we got instead of failing
if len(response.ToolCalls) == 0 {
t.Logf("No tool calls returned. Content: %v", response.Content)
} else {
tc := response.ToolCalls[0]
t.Logf("✓ Tool call: %s(%s)", tc.Function.Name, tc.Function.Arguments)
if tc.Function.Name != "calculate" {
t.Logf("Warning: Expected tool name 'calculate', got '%s'", tc.Function.Name)
}
}
if response.Usage != nil {
t.Logf("Usage: prompt=%d, completion=%d, total=%d",
response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens)
if response.Usage.CompletionTokensDetails != nil {
t.Logf("Reasoning tokens: %d", response.Usage.CompletionTokensDetails.ReasoningTokens)
}
}
t.Logf("Response: %+v", response)
}
// TestGPT5Vision tests GPT-5 with image input
func TestGPT5Vision(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
conn, err := connector.Select("openai.gpt-5")
if err != nil {
t.Fatalf("Failed to select connector: %v", err)
}
trueVal := true
options := &context.CompletionOptions{
Capabilities: &context.ModelCapabilities{
Reasoning: &trueVal,
Vision: &trueVal,
Multimodal: &trueVal,
},
}
llmInstance, err := llm.New(conn, options)
if err != nil {
t.Fatalf("Failed to create LLM instance: %v", err)
}
// Message with image content
messages := []context.Message{
{
Role: context.RoleUser,
Content: []context.ContentPart{
{
Type: context.ContentText,
Text: "What is in this image? Describe briefly.",
},
{
Type: context.ContentImageURL,
ImageURL: &context.ImageURL{
URL: "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/320px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
},
},
},
},
}
maxTokens := 200
options.MaxCompletionTokens = &maxTokens
ctx := newGPT5TestContext("test-gpt5-vision", "openai.gpt-5")
response, err := llmInstance.Post(ctx, messages, options)
if err != nil {
t.Fatalf("Post with vision failed: %v", err)
}
if response == nil {
t.Fatal("Response is nil")
}
// Should have content describing the image
contentStr, ok := response.Content.(string)
if !ok || contentStr == "" {
t.Error("Expected text content describing the image")
} else {
t.Logf("Image description: %s", contentStr)
}
if response.Usage != nil {
t.Logf("Usage: prompt=%d, completion=%d, total=%d",
response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens)
}
}
// TestGPT5ReasoningEffortWithGPT4o tests that GPT-4o ignores reasoning_effort
func TestGPT5ReasoningEffortWithGPT4o(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Use GPT-4o which doesn't support reasoning
conn, err := connector.Select("openai.gpt-4o")
if err != nil {
t.Fatalf("Failed to select connector: %v", err)
}
trueVal := true
falseVal := false
effort := "high"
options := &context.CompletionOptions{
Capabilities: &context.ModelCapabilities{
Reasoning: &falseVal, // GPT-4o doesn't support reasoning
ToolCalls: &trueVal,
},
ReasoningEffort: &effort, // Should be ignored by adapter
}
llmInstance, err := llm.New(conn, options)
if err != nil {
t.Fatalf("Failed to create LLM instance: %v", err)
}
messages := []context.Message{
{
Role: context.RoleUser,
Content: "Say 'OK'",
},
}
maxTokens := 10
options.MaxCompletionTokens = &maxTokens
ctx := newGPT5TestContext("test-gpt4o-no-reasoning", "openai.gpt-4o")
// Should succeed (adapter removes reasoning_effort parameter)
response, err := llmInstance.Post(ctx, messages, options)
if err != nil {
t.Fatalf("Post failed: %v", err)
}
if response == nil {
t.Fatal("Response is nil")
}
// Should have 0 reasoning tokens (GPT-4o doesn't do reasoning)
if response.Usage != nil && response.Usage.CompletionTokensDetails != nil {
reasoningTokens := response.Usage.CompletionTokensDetails.ReasoningTokens
if reasoningTokens != 0 {
t.Errorf("Expected reasoning_tokens=0 for GPT-4o, got %d", reasoningTokens)
} else {
t.Log("✓ GPT-4o correctly shows reasoning_tokens=0")
}
}
t.Log("✓ ReasoningAdapter correctly removed reasoning_effort parameter for GPT-4o")
}
// ============================================================================
// Helper Functions
// ============================================================================
// newGPT5TestContext creates a real Context for testing GPT-5 provider
func newGPT5TestContext(chatID, connectorID string) *context.Context {
return &context.Context{
Context: gocontext.Background(),
Space: plan.NewMemorySharedSpace(),
ChatID: chatID,
AssistantID: "test-assistant",
Connector: connectorID,
Locale: "en-us",
Theme: "light",
Client: context.Client{
Type: "web",
UserAgent: "GPT5ProviderTest/1.0",
IP: "127.0.0.1",
},
Referer: context.RefererAPI,
Accept: context.AcceptStandard,
Route: "/api/test",
Metadata: make(map[string]interface{}),
Authorized: &types.AuthorizedInfo{
Subject: "test-user",
ClientID: "test-client",
UserID: "test-user-123",
TeamID: "test-team-456",
TenantID: "test-tenant-789",
SessionID: "test-session-id",
Constraints: types.DataConstraints{
TeamOnly: true,
Extra: map[string]interface{}{
"test": "gpt5-provider",
},
},
},
}
}

100
agent/llm/providers/openai/openai.go
View file

@ -141,11 +141,17 @@ func buildAdapters(cap *context.ModelCapabilities) []adapters.CapabilityAdapter
result = append(result, adapters.NewAudioAdapter(*cap.Audio))
}
// Reasoning adapter
if cap.Reasoning != nil && *cap.Reasoning {
// Detect reasoning format based on capabilities
format := detectReasoningFormat(cap)
result = append(result, adapters.NewReasoningAdapter(format))
// Reasoning adapter (always add to handle reasoning_effort parameter)
// Even if the model doesn't support reasoning, we need the adapter to strip reasoning_effort
if cap.Reasoning != nil {
if *cap.Reasoning {
// Detect reasoning format based on capabilities
format := detectReasoningFormat(cap)
result = append(result, adapters.NewReasoningAdapter(format))
} else {
// Model doesn't support reasoning, use None format to strip reasoning parameters
result = append(result, adapters.NewReasoningAdapter(adapters.ReasoningFormatNone))
}
}
return result
@ -282,8 +288,22 @@ func (p *Provider) streamWithRetry(ctx *context.Context, messages []context.Mess
}
}
// Preprocess options through adapters
processedOptions := options
for _, adapter := range p.adapters {
newOpts, err := adapter.PreprocessOptions(processedOptions)
if err != nil {
// Send error to handler
if handler != nil {
handler(context.ChunkError, []byte(fmt.Sprintf("adapter %s preprocessing failed: %v", adapter.Name(), err)))
}
return nil, fmt.Errorf("adapter %s preprocessing failed: %w", adapter.Name(), err)
}
processedOptions = newOpts
}
// Build request body
requestBody, err := p.buildRequestBody(messages, options, true)
requestBody, err := p.buildRequestBody(messages, processedOptions, true)
if err != nil {
// Send stream_end with error
if handler != nil {
@ -391,6 +411,20 @@ func (p *Provider) streamWithRetry(ctx *context.Context, messages []context.Mess
accumulator.role = delta.Role
}
// Handle reasoning content (DeepSeek R1)
if delta.ReasoningContent != "" {
// Start thinking group if not active
if !groupTracker.active || groupTracker.groupType != context.ChunkThinking {
groupTracker.startGroup(context.ChunkThinking, handler)
}
accumulator.reasoningContent += delta.ReasoningContent
if handler != nil {
handler(context.ChunkThinking, []byte(delta.ReasoningContent))
groupTracker.incrementChunk()
}
}
// Handle content
if delta.Content != "" {
// Start text group if not active
@ -637,15 +671,16 @@ func (p *Provider) streamWithRetry(ctx *context.Context, messages []context.Mess
// Build final response
response := &context.CompletionResponse{
ID: accumulator.id,
Object: "chat.completion",
Created: accumulator.created,
Model: accumulator.model,
Role: accumulator.role,
Content: accumulator.content,
Refusal: accumulator.refusal,
FinishReason: accumulator.finishReason,
Usage: accumulator.usage,
ID: accumulator.id,
Object: "chat.completion",
Created: accumulator.created,
Model: accumulator.model,
Role: accumulator.role,
Content: accumulator.content,
ReasoningContent: accumulator.reasoningContent,
Refusal: accumulator.refusal,
FinishReason: accumulator.finishReason,
Usage: accumulator.usage,
}
// Convert accumulated tool calls to ToolCall slice
@ -801,8 +836,18 @@ func (p *Provider) Post(ctx *context.Context, messages []context.Message, option
// postWithRetry performs a single POST request attempt
func (p *Provider) postWithRetry(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error) {
// Preprocess options through adapters
processedOptions := options
for _, adapter := range p.adapters {
newOpts, err := adapter.PreprocessOptions(processedOptions)
if err != nil {
return nil, fmt.Errorf("adapter %s preprocessing failed: %w", adapter.Name(), err)
}
processedOptions = newOpts
}
// Build request body
requestBody, err := p.buildRequestBody(messages, options, false)
requestBody, err := p.buildRequestBody(messages, processedOptions, false)
if err != nil {
return nil, fmt.Errorf("failed to build request body: %w", err)
}
@ -854,13 +899,29 @@ func (p *Provider) postWithRetry(ctx *context.Context, messages []context.Messag
}
choice := fullResp.Choices[0]
// Convert content interface{} to string
content := ""
if choice.Message.Content != nil {
switch v := choice.Message.Content.(type) {
case string:
content = v
default:
// For complex content (arrays), marshal to JSON
if contentBytes, err := jsoniter.Marshal(v); err == nil {
content = string(contentBytes)
}
}
}
response := &context.CompletionResponse{
ID: fullResp.ID,
Object: fullResp.Object,
Created: fullResp.Created,
Model: fullResp.Model,
Role: string(choice.Message.Role),
Content: choice.Message.Content,
Content: content,
ReasoningContent: choice.Message.ReasoningContent,
ToolCalls: choice.Message.ToolCalls,
FinishReason: choice.FinishReason,
Usage: fullResp.Usage,
@ -999,6 +1060,11 @@ func (p *Provider) buildRequestBody(messages []context.Message, options *context
body["tool_choice"] = options.ToolChoice
}
// Reasoning effort (o1 and GPT-5 models)
if options.ReasoningEffort != nil {
body["reasoning_effort"] = *options.ReasoningEffort
}
// For streaming, include usage info by default
if streaming {
if options.StreamOptions != nil {

40
agent/llm/providers/openai/types.go
View file

@ -25,10 +25,11 @@ type Delta struct {
// DeltaContent represents the content in a delta
type DeltaContent struct {
Role string `json:"role,omitempty"`
Content string `json:"content,omitempty"`
ToolCalls []ToolCallDelta `json:"tool_calls,omitempty"`
Refusal string `json:"refusal,omitempty"`
Role string `json:"role,omitempty"`
Content string `json:"content,omitempty"`
ReasoningContent string `json:"reasoning_content,omitempty"` // DeepSeek R1 reasoning
ToolCalls []ToolCallDelta `json:"tool_calls,omitempty"`
Refusal string `json:"refusal,omitempty"`
}
// ToolCallDelta represents a tool call delta in streaming
@ -52,9 +53,15 @@ type CompletionResponseFull struct {
Created int64 `json:"created"`
Model string `json:"model"`
Choices []struct {
Index int `json:"index"`
Message context.Message `json:"message"`
FinishReason string `json:"finish_reason"`
Index int `json:"index"`
Message struct {
Role context.MessageRole `json:"role"`
Content interface{} `json:"content,omitempty"` // string or array
ReasoningContent string `json:"reasoning_content,omitempty"` // DeepSeek R1 reasoning
ToolCalls []context.ToolCall `json:"tool_calls,omitempty"`
Refusal *string `json:"refusal,omitempty"`
} `json:"message"`
FinishReason string `json:"finish_reason"`
} `json:"choices"`
Usage *context.UsageInfo `json:"usage,omitempty"`
SystemFingerprint string `json:"system_fingerprint,omitempty"`
@ -62,15 +69,16 @@ type CompletionResponseFull struct {
// streamAccumulator accumulates streaming response data
type streamAccumulator struct {
id string
model string
created int64
role string
content string
refusal string
toolCalls map[int]*accumulatedToolCall
finishReason string
usage *context.UsageInfo
id string
model string
created int64
role string
content string
reasoningContent string // DeepSeek R1 reasoning content
refusal string
toolCalls map[int]*accumulatedToolCall
finishReason string
usage *context.UsageInfo
}
// accumulatedToolCall accumulates a single tool call