Merge pull request #1318 from trheyi/main
Refactor connector settings to model capabilities
This commit is contained in:
commit
edcd8d0a91
11 changed files with 481 additions and 64 deletions
|
|
@ -154,46 +154,46 @@ func (ast *Assistant) getConnectorCapabilities(connectorID string) *context.Mode
|
||||||
Streaming: &falseVal,
|
Streaming: &falseVal,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get connector setting from global settings
|
// Get model capabilities from global configuration
|
||||||
setting, exists := connectorSettings[connectorID]
|
modelCaps, exists := modelCapabilities[connectorID]
|
||||||
if !exists {
|
if !exists {
|
||||||
// Return default capabilities if connector not found in settings
|
// Return default capabilities if model not found in configuration
|
||||||
return capabilities
|
return capabilities
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update capabilities based on connector settings
|
// Update capabilities based on model configuration
|
||||||
if setting.Vision {
|
if modelCaps.Vision {
|
||||||
v := true
|
v := true
|
||||||
capabilities.Vision = &v
|
capabilities.Vision = &v
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle both Tools (deprecated) and ToolCalls
|
// Handle both Tools (deprecated) and ToolCalls
|
||||||
if setting.ToolCalls || setting.Tools {
|
if modelCaps.ToolCalls || modelCaps.Tools {
|
||||||
v := true
|
v := true
|
||||||
capabilities.ToolCalls = &v
|
capabilities.ToolCalls = &v
|
||||||
}
|
}
|
||||||
|
|
||||||
if setting.Audio {
|
if modelCaps.Audio {
|
||||||
v := true
|
v := true
|
||||||
capabilities.Audio = &v
|
capabilities.Audio = &v
|
||||||
}
|
}
|
||||||
|
|
||||||
if setting.Reasoning {
|
if modelCaps.Reasoning {
|
||||||
v := true
|
v := true
|
||||||
capabilities.Reasoning = &v
|
capabilities.Reasoning = &v
|
||||||
}
|
}
|
||||||
|
|
||||||
if setting.Streaming {
|
if modelCaps.Streaming {
|
||||||
v := true
|
v := true
|
||||||
capabilities.Streaming = &v
|
capabilities.Streaming = &v
|
||||||
}
|
}
|
||||||
|
|
||||||
if setting.JSON {
|
if modelCaps.JSON {
|
||||||
v := true
|
v := true
|
||||||
capabilities.JSON = &v
|
capabilities.JSON = &v
|
||||||
}
|
}
|
||||||
|
|
||||||
if setting.Multimodal {
|
if modelCaps.Multimodal {
|
||||||
v := true
|
v := true
|
||||||
capabilities.Multimodal = &v
|
capabilities.Multimodal = &v
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -770,7 +770,7 @@ func (ast *Assistant) withOptions(options map[string]interface{}) map[string]int
|
||||||
|
|
||||||
// Add tool_calls
|
// Add tool_calls
|
||||||
if ast.Tools != nil && ast.Tools.Tools != nil && len(ast.Tools.Tools) > 0 {
|
if ast.Tools != nil && ast.Tools.Tools != nil && len(ast.Tools.Tools) > 0 {
|
||||||
if settings, has := connectorSettings[ast.Connector]; has && settings.Tools {
|
if capabilities, has := modelCapabilities[ast.Connector]; has && capabilities.Tools {
|
||||||
options["tools"] = ast.Tools.Tools
|
options["tools"] = ast.Tools.Tools
|
||||||
if options["tool_choice"] == nil {
|
if options["tool_choice"] == nil {
|
||||||
options["tool_choice"] = "auto"
|
options["tool_choice"] = "auto"
|
||||||
|
|
@ -794,8 +794,8 @@ func (ast *Assistant) withPrompts(messages []chatMessage.Message) []chatMessage.
|
||||||
|
|
||||||
// Add tool_calls
|
// Add tool_calls
|
||||||
if ast.Tools != nil && ast.Tools.Tools != nil && len(ast.Tools.Tools) > 0 {
|
if ast.Tools != nil && ast.Tools.Tools != nil && len(ast.Tools.Tools) > 0 {
|
||||||
settings, has := connectorSettings[ast.Connector]
|
capabilities, has := modelCapabilities[ast.Connector]
|
||||||
if !has || !settings.Tools {
|
if !has || !capabilities.Tools {
|
||||||
// Convert store tools to runtime tools if not already done
|
// Convert store tools to runtime tools if not already done
|
||||||
if ast.runtimeTools == nil {
|
if ast.runtimeTools == nil {
|
||||||
runtimeTools, err := ToRuntimeTools(ast.Tools.Tools)
|
runtimeTools, err := ToRuntimeTools(ast.Tools.Tools)
|
||||||
|
|
|
||||||
|
|
@ -27,7 +27,7 @@ import (
|
||||||
var loaded = NewCache(200) // 200 is the default capacity
|
var loaded = NewCache(200) // 200 is the default capacity
|
||||||
var storage store.Store = nil
|
var storage store.Store = nil
|
||||||
var search interface{} = nil
|
var search interface{} = nil
|
||||||
var connectorSettings map[string]ConnectorSetting = map[string]ConnectorSetting{}
|
var modelCapabilities map[string]ModelCapabilities = map[string]ModelCapabilities{}
|
||||||
var vision *agentvision.Vision = nil
|
var vision *agentvision.Vision = nil
|
||||||
var defaultConnector string = "" // default connector
|
var defaultConnector string = "" // default connector
|
||||||
var globalUses *context.Uses = nil // global uses configuration from agent.yml
|
var globalUses *context.Uses = nil // global uses configuration from agent.yml
|
||||||
|
|
@ -137,9 +137,9 @@ func SetVision(v *agentvision.Vision) {
|
||||||
vision = v
|
vision = v
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetConnectorSettings set the connector settings
|
// SetModelCapabilities set the model capabilities configuration
|
||||||
func SetConnectorSettings(settings map[string]ConnectorSetting) {
|
func SetModelCapabilities(capabilities map[string]ModelCapabilities) {
|
||||||
connectorSettings = settings
|
modelCapabilities = capabilities
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetConnector set the connector
|
// SetConnector set the connector
|
||||||
|
|
|
||||||
|
|
@ -108,9 +108,9 @@ type Assistant struct {
|
||||||
runtimeTools []Tool // Converted tools for business logic (OpenAI format)
|
runtimeTools []Tool // Converted tools for business logic (OpenAI format)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ConnectorSetting the connector setting
|
// ModelCapabilities defines the capabilities of a language model
|
||||||
// Defines the capabilities of a connector/model
|
// This configuration is loaded from agent/models.yml
|
||||||
type ConnectorSetting struct {
|
type ModelCapabilities struct {
|
||||||
Vision bool `json:"vision,omitempty" yaml:"vision,omitempty"` // Supports vision/image input
|
Vision bool `json:"vision,omitempty" yaml:"vision,omitempty"` // Supports vision/image input
|
||||||
Tools bool `json:"tools,omitempty" yaml:"tools,omitempty"` // Supports tool/function calling (deprecated, use ToolCalls)
|
Tools bool `json:"tools,omitempty" yaml:"tools,omitempty"` // Supports tool/function calling (deprecated, use ToolCalls)
|
||||||
ToolCalls bool `json:"tool_calls,omitempty" yaml:"tool_calls,omitempty"` // Supports tool/function calling
|
ToolCalls bool `json:"tool_calls,omitempty" yaml:"tool_calls,omitempty"` // Supports tool/function calling
|
||||||
|
|
|
||||||
|
|
@ -122,13 +122,12 @@ func GetAssistantID(c *gin.Context, req *CompletionRequest) (string, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if model != "" {
|
if model != "" {
|
||||||
// Split by "-" and get the last field
|
// Parse model ID using the same logic as ParseModelID
|
||||||
parts := strings.Split(model, "-")
|
// Expected format: [prefix-]assistantName-model-yao_assistantID
|
||||||
lastField := strings.TrimSpace(parts[len(parts)-1])
|
// Find the last occurrence of "-yao_"
|
||||||
|
parts := strings.Split(model, "-yao_")
|
||||||
// Check if it has yao_ prefix
|
if len(parts) >= 2 {
|
||||||
if strings.HasPrefix(lastField, "yao_") {
|
assistantID := parts[len(parts)-1]
|
||||||
assistantID := strings.TrimPrefix(lastField, "yao_")
|
|
||||||
if assistantID != "" {
|
if assistantID != "" {
|
||||||
return assistantID, nil
|
return assistantID, nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -12,13 +12,14 @@ type Uses struct {
|
||||||
// ModelCapabilities defines the capabilities of a language model
|
// ModelCapabilities defines the capabilities of a language model
|
||||||
// Used by LLM to select appropriate provider and validate requests
|
// Used by LLM to select appropriate provider and validate requests
|
||||||
type ModelCapabilities struct {
|
type ModelCapabilities struct {
|
||||||
Vision *bool `json:"vision,omitempty"` // Supports vision/image input
|
Vision *bool `json:"vision,omitempty"` // Supports vision/image input
|
||||||
ToolCalls *bool `json:"tool_calls,omitempty"` // Supports tool/function calling
|
ToolCalls *bool `json:"tool_calls,omitempty"` // Supports tool/function calling
|
||||||
Audio *bool `json:"audio,omitempty"` // Supports audio input/output
|
Audio *bool `json:"audio,omitempty"` // Supports audio input/output
|
||||||
Reasoning *bool `json:"reasoning,omitempty"` // Supports reasoning/thinking mode (o1, DeepSeek R1)
|
Reasoning *bool `json:"reasoning,omitempty"` // Supports reasoning/thinking mode (o1, DeepSeek R1)
|
||||||
Streaming *bool `json:"streaming,omitempty"` // Supports streaming responses
|
Streaming *bool `json:"streaming,omitempty"` // Supports streaming responses
|
||||||
JSON *bool `json:"json,omitempty"` // Supports JSON mode
|
JSON *bool `json:"json,omitempty"` // Supports JSON mode
|
||||||
Multimodal *bool `json:"multimodal,omitempty"` // Supports multimodal input (text + images + audio)
|
Multimodal *bool `json:"multimodal,omitempty"` // Supports multimodal input (text + images + audio)
|
||||||
|
TemperatureAdjustable *bool `json:"temperature_adjustable,omitempty"` // Supports temperature adjustment (reasoning models typically don't)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CompletionOptions the completion request options
|
// CompletionOptions the completion request options
|
||||||
|
|
|
||||||
|
|
@ -16,45 +16,83 @@ const (
|
||||||
|
|
||||||
// ReasoningAdapter handles reasoning content capability
|
// ReasoningAdapter handles reasoning content capability
|
||||||
// - Manages reasoning_effort parameter (o1, GPT-5)
|
// - Manages reasoning_effort parameter (o1, GPT-5)
|
||||||
|
// - Manages temperature parameter constraints (reasoning models typically require temperature=1)
|
||||||
// - Extracts reasoning_tokens from usage
|
// - Extracts reasoning_tokens from usage
|
||||||
// - Parses visible reasoning content (DeepSeek R1)
|
// - Parses visible reasoning content (DeepSeek R1)
|
||||||
type ReasoningAdapter struct {
|
type ReasoningAdapter struct {
|
||||||
*BaseAdapter
|
*BaseAdapter
|
||||||
format ReasoningFormat
|
format ReasoningFormat
|
||||||
supportsEffort bool // Whether the model supports reasoning_effort parameter
|
supportsEffort bool // Whether the model supports reasoning_effort parameter
|
||||||
|
supportsTemperature bool // Whether the model supports temperature adjustment
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewReasoningAdapter creates a new reasoning adapter
|
// NewReasoningAdapter creates a new reasoning adapter
|
||||||
func NewReasoningAdapter(format ReasoningFormat) *ReasoningAdapter {
|
// If cap.TemperatureAdjustable is provided, it overrides the default behavior
|
||||||
|
func NewReasoningAdapter(format ReasoningFormat, cap *context.ModelCapabilities) *ReasoningAdapter {
|
||||||
supportsEffort := false
|
supportsEffort := false
|
||||||
|
supportsTemperature := true
|
||||||
|
|
||||||
// Only OpenAI o1 and GPT-5 support reasoning_effort parameter
|
// Set defaults based on reasoning format
|
||||||
if format == ReasoningFormatOpenAI || format == ReasoningFormatGPT5 {
|
switch format {
|
||||||
|
case ReasoningFormatOpenAI, ReasoningFormatGPT5:
|
||||||
|
// OpenAI o1 and GPT-5: support reasoning_effort, but NOT temperature adjustment
|
||||||
supportsEffort = true
|
supportsEffort = true
|
||||||
|
supportsTemperature = false
|
||||||
|
case ReasoningFormatDeepSeek:
|
||||||
|
// DeepSeek R1: no reasoning_effort, no temperature adjustment
|
||||||
|
supportsEffort = false
|
||||||
|
supportsTemperature = false
|
||||||
|
case ReasoningFormatNone:
|
||||||
|
// Non-reasoning models: no reasoning_effort, but support temperature
|
||||||
|
supportsEffort = false
|
||||||
|
supportsTemperature = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Override with explicit capability if provided
|
||||||
|
if cap != nil && cap.TemperatureAdjustable != nil {
|
||||||
|
supportsTemperature = *cap.TemperatureAdjustable
|
||||||
}
|
}
|
||||||
|
|
||||||
return &ReasoningAdapter{
|
return &ReasoningAdapter{
|
||||||
BaseAdapter: NewBaseAdapter("ReasoningAdapter"),
|
BaseAdapter: NewBaseAdapter("ReasoningAdapter"),
|
||||||
format: format,
|
format: format,
|
||||||
supportsEffort: supportsEffort,
|
supportsEffort: supportsEffort,
|
||||||
|
supportsTemperature: supportsTemperature,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// PreprocessOptions handles reasoning_effort parameter
|
// PreprocessOptions handles reasoning_effort and temperature parameters
|
||||||
func (a *ReasoningAdapter) PreprocessOptions(options *context.CompletionOptions) (*context.CompletionOptions, error) {
|
func (a *ReasoningAdapter) PreprocessOptions(options *context.CompletionOptions) (*context.CompletionOptions, error) {
|
||||||
if options == nil {
|
if options == nil {
|
||||||
return options, nil
|
return options, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// If model doesn't support reasoning_effort, remove it
|
newOptions := *options
|
||||||
if !a.supportsEffort && options.ReasoningEffort != nil {
|
modified := false
|
||||||
|
|
||||||
|
// 1. Handle reasoning_effort parameter
|
||||||
|
if !a.supportsEffort && newOptions.ReasoningEffort != nil {
|
||||||
// Model doesn't support reasoning_effort, remove the parameter
|
// Model doesn't support reasoning_effort, remove the parameter
|
||||||
newOptions := *options
|
|
||||||
newOptions.ReasoningEffort = nil
|
newOptions.ReasoningEffort = nil
|
||||||
|
modified = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Handle temperature parameter
|
||||||
|
if !a.supportsTemperature && newOptions.Temperature != nil {
|
||||||
|
currentTemp := *newOptions.Temperature
|
||||||
|
if currentTemp != 1.0 {
|
||||||
|
// Model doesn't support temperature adjustment, reset to default (1.0)
|
||||||
|
defaultTemp := 1.0
|
||||||
|
newOptions.Temperature = &defaultTemp
|
||||||
|
modified = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if modified {
|
||||||
return &newOptions, nil
|
return &newOptions, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// If model supports reasoning_effort, keep it as-is (user can set "low", "medium", or "high")
|
// No modifications needed
|
||||||
return options, nil
|
return options, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -141,16 +141,16 @@ func buildAdapters(cap *context.ModelCapabilities) []adapters.CapabilityAdapter
|
||||||
result = append(result, adapters.NewAudioAdapter(*cap.Audio))
|
result = append(result, adapters.NewAudioAdapter(*cap.Audio))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reasoning adapter (always add to handle reasoning_effort parameter)
|
// Reasoning adapter (always add to handle reasoning_effort and temperature parameters)
|
||||||
// Even if the model doesn't support reasoning, we need the adapter to strip reasoning_effort
|
// Even if the model doesn't support reasoning, we need the adapter to strip reasoning_effort
|
||||||
if cap.Reasoning != nil {
|
if cap.Reasoning != nil {
|
||||||
if *cap.Reasoning {
|
if *cap.Reasoning {
|
||||||
// Detect reasoning format based on capabilities
|
// Detect reasoning format based on capabilities
|
||||||
format := detectReasoningFormat(cap)
|
format := detectReasoningFormat(cap)
|
||||||
result = append(result, adapters.NewReasoningAdapter(format))
|
result = append(result, adapters.NewReasoningAdapter(format, cap))
|
||||||
} else {
|
} else {
|
||||||
// Model doesn't support reasoning, use None format to strip reasoning parameters
|
// Model doesn't support reasoning, use None format to strip reasoning parameters
|
||||||
result = append(result, adapters.NewReasoningAdapter(adapters.ReasoningFormatNone))
|
result = append(result, adapters.NewReasoningAdapter(adapters.ReasoningFormatNone, cap))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
379
agent/llm/providers/openai/temperature_test.go
Normal file
379
agent/llm/providers/openai/temperature_test.go
Normal file
|
|
@ -0,0 +1,379 @@
|
||||||
|
package openai_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
gocontext "context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/yaoapp/gou/connector"
|
||||||
|
"github.com/yaoapp/gou/plan"
|
||||||
|
"github.com/yaoapp/yao/agent/context"
|
||||||
|
"github.com/yaoapp/yao/agent/llm"
|
||||||
|
"github.com/yaoapp/yao/config"
|
||||||
|
"github.com/yaoapp/yao/openapi/oauth/types"
|
||||||
|
"github.com/yaoapp/yao/test"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestTemperatureGPT5AutoReset tests that GPT-5 automatically resets temperature to 1.0
|
||||||
|
func TestTemperatureGPT5AutoReset(t *testing.T) {
|
||||||
|
test.Prepare(t, config.Conf)
|
||||||
|
defer test.Clean()
|
||||||
|
|
||||||
|
conn, err := connector.Select("openai.gpt-5")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to select connector: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
trueVal := true
|
||||||
|
invalidTemp := 0.7 // GPT-5 doesn't support this
|
||||||
|
options := &context.CompletionOptions{
|
||||||
|
Capabilities: &context.ModelCapabilities{
|
||||||
|
Reasoning: &trueVal,
|
||||||
|
},
|
||||||
|
Temperature: &invalidTemp, // Should be reset to 1.0
|
||||||
|
}
|
||||||
|
|
||||||
|
llmInstance, err := llm.New(conn, options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create LLM instance: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
messages := []context.Message{
|
||||||
|
{
|
||||||
|
Role: context.RoleUser,
|
||||||
|
Content: "Say 'OK'",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
maxTokens := 10
|
||||||
|
options.MaxCompletionTokens = &maxTokens
|
||||||
|
|
||||||
|
ctx := newTemperatureTestContext("test-gpt5-temp", "openai.gpt-5")
|
||||||
|
|
||||||
|
// Should succeed (temperature automatically reset to 1.0)
|
||||||
|
response, err := llmInstance.Post(ctx, messages, options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Post failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if response == nil {
|
||||||
|
t.Fatal("Response is nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Log("✓ GPT-5 successfully handled invalid temperature by resetting to 1.0")
|
||||||
|
t.Logf("Response: %v", response.Content)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTemperatureDeepSeekR1AutoReset tests that DeepSeek R1 automatically resets temperature to 1.0
|
||||||
|
func TestTemperatureDeepSeekR1AutoReset(t *testing.T) {
|
||||||
|
test.Prepare(t, config.Conf)
|
||||||
|
defer test.Clean()
|
||||||
|
|
||||||
|
conn, err := connector.Select("deepseek.r1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to select connector: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
trueVal := true
|
||||||
|
invalidTemp := 0.5 // DeepSeek R1 doesn't support this
|
||||||
|
options := &context.CompletionOptions{
|
||||||
|
Capabilities: &context.ModelCapabilities{
|
||||||
|
Reasoning: &trueVal,
|
||||||
|
},
|
||||||
|
Temperature: &invalidTemp, // Should be reset to 1.0
|
||||||
|
}
|
||||||
|
|
||||||
|
llmInstance, err := llm.New(conn, options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create LLM instance: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
messages := []context.Message{
|
||||||
|
{
|
||||||
|
Role: context.RoleUser,
|
||||||
|
Content: "Say 'Hello'",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
maxTokens := 100
|
||||||
|
options.MaxCompletionTokens = &maxTokens
|
||||||
|
|
||||||
|
ctx := newTemperatureTestContext("test-deepseek-r1-temp", "deepseek.r1")
|
||||||
|
|
||||||
|
// Should succeed (temperature automatically reset to 1.0)
|
||||||
|
response, err := llmInstance.Post(ctx, messages, options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Post failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if response == nil {
|
||||||
|
t.Fatal("Response is nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Log("✓ DeepSeek R1 successfully handled invalid temperature by resetting to 1.0")
|
||||||
|
t.Logf("Response content: %v", response.Content)
|
||||||
|
if response.ReasoningContent != "" {
|
||||||
|
t.Logf("Reasoning content length: %d", len(response.ReasoningContent))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTemperatureGPT4oPreserved tests that GPT-4o preserves custom temperature
|
||||||
|
func TestTemperatureGPT4oPreserved(t *testing.T) {
|
||||||
|
test.Prepare(t, config.Conf)
|
||||||
|
defer test.Clean()
|
||||||
|
|
||||||
|
conn, err := connector.Select("openai.gpt-4o")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to select connector: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
trueVal := true
|
||||||
|
falseVal := false
|
||||||
|
customTemp := 0.3 // GPT-4o should preserve this
|
||||||
|
options := &context.CompletionOptions{
|
||||||
|
Capabilities: &context.ModelCapabilities{
|
||||||
|
Reasoning: &falseVal, // Not a reasoning model
|
||||||
|
ToolCalls: &trueVal,
|
||||||
|
},
|
||||||
|
Temperature: &customTemp, // Should be preserved
|
||||||
|
}
|
||||||
|
|
||||||
|
llmInstance, err := llm.New(conn, options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create LLM instance: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
messages := []context.Message{
|
||||||
|
{
|
||||||
|
Role: context.RoleUser,
|
||||||
|
Content: "Say 'OK'",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
maxTokens := 10
|
||||||
|
options.MaxCompletionTokens = &maxTokens
|
||||||
|
|
||||||
|
ctx := newTemperatureTestContext("test-gpt4o-temp", "openai.gpt-4o")
|
||||||
|
|
||||||
|
// Should succeed with custom temperature preserved
|
||||||
|
response, err := llmInstance.Post(ctx, messages, options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Post failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if response == nil {
|
||||||
|
t.Fatal("Response is nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Log("✓ GPT-4o successfully preserved custom temperature (0.3)")
|
||||||
|
t.Logf("Response: %v", response.Content)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTemperatureDeepSeekV3Preserved tests that DeepSeek V3 preserves custom temperature
|
||||||
|
func TestTemperatureDeepSeekV3Preserved(t *testing.T) {
|
||||||
|
test.Prepare(t, config.Conf)
|
||||||
|
defer test.Clean()
|
||||||
|
|
||||||
|
conn, err := connector.Select("deepseek.v3")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to select connector: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
trueVal := true
|
||||||
|
falseVal := false
|
||||||
|
customTemp := 0.8 // DeepSeek V3 should preserve this
|
||||||
|
options := &context.CompletionOptions{
|
||||||
|
Capabilities: &context.ModelCapabilities{
|
||||||
|
Reasoning: &falseVal, // Not a reasoning model
|
||||||
|
ToolCalls: &trueVal,
|
||||||
|
},
|
||||||
|
Temperature: &customTemp, // Should be preserved
|
||||||
|
}
|
||||||
|
|
||||||
|
llmInstance, err := llm.New(conn, options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create LLM instance: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
messages := []context.Message{
|
||||||
|
{
|
||||||
|
Role: context.RoleUser,
|
||||||
|
Content: "Say 'Hello World'",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
maxTokens := 20
|
||||||
|
options.MaxCompletionTokens = &maxTokens
|
||||||
|
|
||||||
|
ctx := newTemperatureTestContext("test-deepseek-v3-temp", "deepseek.v3")
|
||||||
|
|
||||||
|
// Should succeed with custom temperature preserved
|
||||||
|
response, err := llmInstance.Post(ctx, messages, options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Post failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if response == nil {
|
||||||
|
t.Fatal("Response is nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Log("✓ DeepSeek V3 successfully preserved custom temperature (0.8)")
|
||||||
|
t.Logf("Response: %v", response.Content)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTemperatureGPT5Default tests that GPT-5 with temperature=1.0 works fine
|
||||||
|
func TestTemperatureGPT5Default(t *testing.T) {
|
||||||
|
test.Prepare(t, config.Conf)
|
||||||
|
defer test.Clean()
|
||||||
|
|
||||||
|
conn, err := connector.Select("openai.gpt-5")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to select connector: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
trueVal := true
|
||||||
|
defaultTemp := 1.0 // GPT-5's valid temperature
|
||||||
|
options := &context.CompletionOptions{
|
||||||
|
Capabilities: &context.ModelCapabilities{
|
||||||
|
Reasoning: &trueVal,
|
||||||
|
},
|
||||||
|
Temperature: &defaultTemp, // Should work fine
|
||||||
|
}
|
||||||
|
|
||||||
|
llmInstance, err := llm.New(conn, options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create LLM instance: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
messages := []context.Message{
|
||||||
|
{
|
||||||
|
Role: context.RoleUser,
|
||||||
|
Content: "What is 2+2? Reply with just the number.",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
maxTokens := 10
|
||||||
|
options.MaxCompletionTokens = &maxTokens
|
||||||
|
|
||||||
|
ctx := newTemperatureTestContext("test-gpt5-temp-default", "openai.gpt-5")
|
||||||
|
|
||||||
|
// Should succeed with default temperature
|
||||||
|
response, err := llmInstance.Post(ctx, messages, options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Post failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if response == nil {
|
||||||
|
t.Fatal("Response is nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Log("✓ GPT-5 successfully handled default temperature (1.0)")
|
||||||
|
t.Logf("Response: %v", response.Content)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTemperatureNoTemperatureProvided tests that models work when no temperature is provided
|
||||||
|
func TestTemperatureNoTemperatureProvided(t *testing.T) {
|
||||||
|
test.Prepare(t, config.Conf)
|
||||||
|
defer test.Clean()
|
||||||
|
|
||||||
|
testCases := []struct {
|
||||||
|
name string
|
||||||
|
connector string
|
||||||
|
reasoning bool
|
||||||
|
}{
|
||||||
|
{"GPT-5 No Temp", "openai.gpt-5", true},
|
||||||
|
{"GPT-4o No Temp", "openai.gpt-4o", false},
|
||||||
|
{"DeepSeek R1 No Temp", "deepseek.r1", true},
|
||||||
|
{"DeepSeek V3 No Temp", "deepseek.v3", false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range testCases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
conn, err := connector.Select(tc.connector)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to select connector: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
trueVal := true
|
||||||
|
falseVal := false
|
||||||
|
options := &context.CompletionOptions{
|
||||||
|
Capabilities: &context.ModelCapabilities{
|
||||||
|
Reasoning: &falseVal,
|
||||||
|
ToolCalls: &trueVal,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if tc.reasoning {
|
||||||
|
options.Capabilities.Reasoning = &trueVal
|
||||||
|
}
|
||||||
|
// Temperature not set - should use API default
|
||||||
|
|
||||||
|
llmInstance, err := llm.New(conn, options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create LLM instance: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
messages := []context.Message{
|
||||||
|
{
|
||||||
|
Role: context.RoleUser,
|
||||||
|
Content: "Say 'OK'",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
maxTokens := 10
|
||||||
|
options.MaxCompletionTokens = &maxTokens
|
||||||
|
|
||||||
|
ctx := newTemperatureTestContext("test-no-temp-"+tc.connector, tc.connector)
|
||||||
|
|
||||||
|
response, err := llmInstance.Post(ctx, messages, options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Post failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if response == nil {
|
||||||
|
t.Fatal("Response is nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("✓ %s works fine without temperature parameter", tc.connector)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Helper Functions
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
// newTemperatureTestContext creates a real Context for testing temperature handling
|
||||||
|
func newTemperatureTestContext(chatID, connectorID string) *context.Context {
|
||||||
|
return &context.Context{
|
||||||
|
Context: gocontext.Background(),
|
||||||
|
Space: plan.NewMemorySharedSpace(),
|
||||||
|
ChatID: chatID,
|
||||||
|
AssistantID: "test-assistant",
|
||||||
|
Connector: connectorID,
|
||||||
|
Locale: "en-us",
|
||||||
|
Theme: "light",
|
||||||
|
Client: context.Client{
|
||||||
|
Type: "web",
|
||||||
|
UserAgent: "TemperatureTest/1.0",
|
||||||
|
IP: "127.0.0.1",
|
||||||
|
},
|
||||||
|
Referer: context.RefererAPI,
|
||||||
|
Accept: context.AcceptStandard,
|
||||||
|
Route: "/api/test",
|
||||||
|
Metadata: make(map[string]interface{}),
|
||||||
|
Authorized: &types.AuthorizedInfo{
|
||||||
|
Subject: "test-user",
|
||||||
|
ClientID: "test-client",
|
||||||
|
UserID: "test-user-123",
|
||||||
|
TeamID: "test-team-456",
|
||||||
|
TenantID: "test-tenant-789",
|
||||||
|
SessionID: "test-session-id",
|
||||||
|
Constraints: types.DataConstraints{
|
||||||
|
TeamOnly: true,
|
||||||
|
Extra: map[string]interface{}{
|
||||||
|
"test": "temperature",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@ -68,8 +68,8 @@ func Load(cfg config.Config) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize Connector settings
|
// Initialize model capabilities
|
||||||
err = initConnectorSettings()
|
err = initModelCapabilities()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
@ -107,26 +107,26 @@ func initGlobalI18n() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// initConnectors initialize the connectors
|
// initModelCapabilities initialize the model capabilities configuration
|
||||||
func initConnectorSettings() error {
|
func initModelCapabilities() error {
|
||||||
path := filepath.Join("agent", "connectors.yml")
|
path := filepath.Join("agent", "models.yml")
|
||||||
if exists, _ := application.App.Exists(path); !exists {
|
if exists, _ := application.App.Exists(path); !exists {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Open the connectors
|
// Read the model capabilities configuration
|
||||||
bytes, err := application.App.Read(path)
|
bytes, err := application.App.Read(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
var connectors map[string]assistant.ConnectorSetting = map[string]assistant.ConnectorSetting{}
|
var models map[string]assistant.ModelCapabilities = map[string]assistant.ModelCapabilities{}
|
||||||
err = application.Parse("connectors.yml", bytes, &connectors)
|
err = application.Parse("models.yml", bytes, &models)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
api.Agent.DSL.Connectors = connectors
|
api.Agent.DSL.Models = models
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -183,8 +183,8 @@ func initAssistant() error {
|
||||||
assistant.SetGlobalUses(globalUses)
|
assistant.SetGlobalUses(globalUses)
|
||||||
}
|
}
|
||||||
|
|
||||||
if api.Agent.DSL.Connectors != nil {
|
if api.Agent.DSL.Models != nil {
|
||||||
assistant.SetConnectorSettings(api.Agent.DSL.Connectors)
|
assistant.SetModelCapabilities(api.Agent.DSL.Models)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load Built-in Assistants
|
// Load Built-in Assistants
|
||||||
|
|
|
||||||
|
|
@ -20,9 +20,9 @@ type DSL struct {
|
||||||
// UploadSetting *Upload `json:"upload,omitempty" yaml:"upload,omitempty"` // Upload Settings
|
// UploadSetting *Upload `json:"upload,omitempty" yaml:"upload,omitempty"` // Upload Settings
|
||||||
// KnowledgeSetting *Knowledge `json:"knowledge,omitempty" yaml:"knowledge,omitempty"` // Knowledge base Settings
|
// KnowledgeSetting *Knowledge `json:"knowledge,omitempty" yaml:"knowledge,omitempty"` // Knowledge base Settings
|
||||||
|
|
||||||
// Global External Settings - connectors, tools, etc.
|
// Global External Settings - model capabilities, tools, etc.
|
||||||
// ===============================
|
// ===============================
|
||||||
Connectors map[string]assistant.ConnectorSetting `json:"connectors,omitempty" yaml:"connectors,omitempty"` // The connectors of the assistant
|
Models map[string]assistant.ModelCapabilities `json:"models,omitempty" yaml:"models,omitempty"` // The model capabilities configuration
|
||||||
|
|
||||||
// Agent API Settings
|
// Agent API Settings
|
||||||
// ===============================s
|
// ===============================s
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue