From 796f6437efa5b3147f7ec6ee070cec8340bede16 Mon Sep 17 00:00:00 2001
From: Max <shadow.iqka@qq.com>
Date: Mon, 17 Nov 2025 10:59:46 +0800
Subject: [PATCH] Update ModelCapabilities to support flexible vision input
 handling

- Changed the Vision field in ModelCapabilities from bool to interface{} to accommodate both boolean values and VisionFormat strings.
- Modified the getConnectorCapabilities function to directly assign the Vision capability based on the updated model configuration, enhancing compatibility with various vision input formats.
---
 agent/assistant/agent.go |  8 ++++----
 agent/assistant/types.go | 16 ++++++++--------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/agent/assistant/agent.go b/agent/assistant/agent.go
index 4691547b..ddc90b6a 100644
--- a/agent/assistant/agent.go
+++ b/agent/assistant/agent.go
@@ -147,7 +147,7 @@ func (ast *Assistant) getConnectorCapabilities(connectorID string) *context.Mode
 	// Initialize with default capabilities (all disabled)
 	falseVal := false
 	capabilities := &context.ModelCapabilities{
-		Vision:    &falseVal,
+		Vision:    falseVal,
 		ToolCalls: &falseVal,
 		Audio:     &falseVal,
 		Reasoning: &falseVal,
@@ -162,9 +162,9 @@ func (ast *Assistant) getConnectorCapabilities(connectorID string) *context.Mode
 	}
 
 	// Update capabilities based on model configuration
-	if modelCaps.Vision {
-		v := true
-		capabilities.Vision = &v
+	// Vision can be bool or string (VisionFormat)
+	if modelCaps.Vision != nil {
+		capabilities.Vision = modelCaps.Vision
 	}
 
 	// Handle both Tools (deprecated) and ToolCalls
diff --git a/agent/assistant/types.go b/agent/assistant/types.go
index a46bc0f1..8fb757c9 100644
--- a/agent/assistant/types.go
+++ b/agent/assistant/types.go
@@ -111,14 +111,14 @@ type Assistant struct {
 // ModelCapabilities defines the capabilities of a language model
 // This configuration is loaded from agent/models.yml
 type ModelCapabilities struct {
-	Vision     bool `json:"vision,omitempty" yaml:"vision,omitempty"`         // Supports vision/image input
-	Tools      bool `json:"tools,omitempty" yaml:"tools,omitempty"`           // Supports tool/function calling (deprecated, use ToolCalls)
-	ToolCalls  bool `json:"tool_calls,omitempty" yaml:"tool_calls,omitempty"` // Supports tool/function calling
-	Audio      bool `json:"audio,omitempty" yaml:"audio,omitempty"`           // Supports audio input/output
-	Reasoning  bool `json:"reasoning,omitempty" yaml:"reasoning,omitempty"`   // Supports reasoning/thinking mode (o1, DeepSeek R1)
-	Streaming  bool `json:"streaming,omitempty" yaml:"streaming,omitempty"`   // Supports streaming responses
-	JSON       bool `json:"json,omitempty" yaml:"json,omitempty"`             // Supports JSON mode
-	Multimodal bool `json:"multimodal,omitempty" yaml:"multimodal,omitempty"` // Supports multimodal input
+	Vision     interface{} `json:"vision,omitempty" yaml:"vision,omitempty"`         // Supports vision/image input: bool or VisionFormat string ("openai", "claude"/"base64", "default")
+	Tools      bool        `json:"tools,omitempty" yaml:"tools,omitempty"`           // Supports tool/function calling (deprecated, use ToolCalls)
+	ToolCalls  bool        `json:"tool_calls,omitempty" yaml:"tool_calls,omitempty"` // Supports tool/function calling
+	Audio      bool        `json:"audio,omitempty" yaml:"audio,omitempty"`           // Supports audio input/output
+	Reasoning  bool        `json:"reasoning,omitempty" yaml:"reasoning,omitempty"`   // Supports reasoning/thinking mode (o1, DeepSeek R1)
+	Streaming  bool        `json:"streaming,omitempty" yaml:"streaming,omitempty"`   // Supports streaming responses
+	JSON       bool        `json:"json,omitempty" yaml:"json,omitempty"`             // Supports JSON mode
+	Multimodal bool        `json:"multimodal,omitempty" yaml:"multimodal,omitempty"` // Supports multimodal input
 }
 
 // VisionCapableModels list of LLM models that support vision capabilities