Add Claude API configuration and enhance vision support in LLM adapters

- Integrated Claude API keys into GitHub workflows for both unit and PR tests.
- Introduced a new VisionFormat type and constants to manage image input formats.
- Updated ModelCapabilities to support vision input, allowing for flexible handling of image formats.
- Enhanced VisionAdapter to preprocess messages and convert image URLs to base64 format for Claude compatibility.
- Improved OpenAI provider to utilize vision support in message preprocessing, ensuring better integration with vision capabilities.

.github/workflows/pr-test.yml

@@ -49,6 +49,13 @@ env:
   DEEPSEEK_MODELS_V3: ${{ secrets.DEEPSEEK_MODELS_V3 }}
   DEEPSEEK_MODELS_V3_1: ${{ secrets.DEEPSEEK_MODELS_V3_1 }}
 
+  # Claude API Configuration
+  CLAUDE_API_KEY: ${{ secrets.CLAUDE_API_KEY }}
+  CLAUDE_PROXY: ${{ secrets.CLAUDE_PROXY }}
+  CLAUDE_API_HOST: ${{ secrets.CLAUDE_API_HOST }}
+  CLAUDE_SONNET_4: ${{ secrets.CLAUDE_SONNET_4 }}
+  CLAUDE_SONNET_4_THINKING: ${{ secrets.CLAUDE_SONNET_4_THINKING }}
+
   TAB_NAME: "::PET ADMIN"
   PAGE_SIZE: "20"
   PAGE_LINK: "https://yaoapps.com"

.github/workflows/unit-test.yml

@@ -53,6 +53,13 @@ env:
   DEEPSEEK_MODELS_V3: ${{ secrets.DEEPSEEK_MODELS_V3 }}
   DEEPSEEK_MODELS_V3_1: ${{ secrets.DEEPSEEK_MODELS_V3_1 }}
 
+  # Claude API Configuration
+  CLAUDE_API_KEY: ${{ secrets.CLAUDE_API_KEY }}
+  CLAUDE_PROXY: ${{ secrets.CLAUDE_PROXY }}
+  CLAUDE_API_HOST: ${{ secrets.CLAUDE_API_HOST }}
+  CLAUDE_SONNET_4: ${{ secrets.CLAUDE_SONNET_4 }}
+  CLAUDE_SONNET_4_THINKING: ${{ secrets.CLAUDE_SONNET_4_THINKING }}
+
   TAB_NAME: "::PET ADMIN"
   PAGE_SIZE: "20"
   PAGE_LINK: "https://yaoapps.com"

agent/context/types_llm.go

@@ -9,10 +9,27 @@ type Uses struct {
 	Fetch  string `json:"fetch,omitempty"`  // Fetch/retrieval tool. Format: "agent" or "mcp:server_id"
 }
 
+// VisionFormat specifies the vision input format
+type VisionFormat string
+
+// Vision format constants define how image inputs are processed
+const (
+	// VisionFormatNone indicates no vision support
+	VisionFormatNone VisionFormat = ""
+	// VisionFormatOpenAI indicates OpenAI format (image_url with URL)
+	VisionFormatOpenAI VisionFormat = "openai"
+	// VisionFormatClaude indicates Claude/Anthropic format (image with base64)
+	VisionFormatClaude VisionFormat = "claude"
+	// VisionFormatBase64 forces base64 conversion (alias for claude)
+	VisionFormatBase64 VisionFormat = "base64"
+	// VisionFormatDefault enables auto-detection of format
+	VisionFormatDefault VisionFormat = "default"
+)
+
 // ModelCapabilities defines the capabilities of a language model
 // Used by LLM to select appropriate provider and validate requests
 type ModelCapabilities struct {
-	Vision                *bool `json:"vision,omitempty"`                 // Supports vision/image input
+	Vision                interface{} `json:"vision,omitempty"`                 // Supports vision/image input: bool or VisionFormat string ("openai", "claude"/"base64", "default")
 	ToolCalls             *bool       `json:"tool_calls,omitempty"`             // Supports tool/function calling
 	Audio                 *bool       `json:"audio,omitempty"`                  // Supports audio input/output
 	Reasoning             *bool       `json:"reasoning,omitempty"`              // Supports reasoning/thinking mode (o1, DeepSeek R1)
@@ -22,6 +39,33 @@ type ModelCapabilities struct {
 	TemperatureAdjustable *bool       `json:"temperature_adjustable,omitempty"` // Supports temperature adjustment (reasoning models typically don't)
 }
 
+// GetVisionSupport returns whether vision is supported and the format
+func (m *ModelCapabilities) GetVisionSupport() (bool, VisionFormat) {
+	if m == nil || m.Vision == nil {
+		return false, VisionFormatNone
+	}
+
+	switch v := m.Vision.(type) {
+	case bool:
+		// Legacy bool format
+		return v, VisionFormatDefault
+	case string:
+		// String format
+		if v == "" || v == string(VisionFormatNone) {
+			return false, VisionFormatNone
+		}
+		return true, VisionFormat(v)
+	case VisionFormat:
+		// Direct VisionFormat type
+		if v == VisionFormatNone || v == "" {
+			return false, VisionFormatNone
+		}
+		return true, v
+	default:
+		return false, VisionFormatNone
+	}
+}
+
 // CompletionOptions the completion request options
 // These options are extracted from HookCreateResponse and Context, then passed to the LLM connector
 // Compatible with OpenAI Chat Completion API: https://platform.openai.com/docs/api-reference/chat/create

agent/llm/adapters/vision.go

@@ -1,6 +1,13 @@
 package adapters
 
 import (
+	"encoding/base64"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
 	"github.com/yaoapp/yao/agent/context"
 )
 
@@ -9,47 +16,64 @@ import (
 type VisionAdapter struct {
 	*BaseAdapter
 	nativeSupport bool
+	format        context.VisionFormat
 }
 
 // NewVisionAdapter creates a new vision adapter
-func NewVisionAdapter(nativeSupport bool) *VisionAdapter {
+func NewVisionAdapter(nativeSupport bool, format context.VisionFormat) *VisionAdapter {
 	return &VisionAdapter{
 		BaseAdapter:   NewBaseAdapter("VisionAdapter"),
 		nativeSupport: nativeSupport,
+		format:        format,
 	}
 }
 
 // PreprocessMessages removes or converts image content if not supported
 func (a *VisionAdapter) PreprocessMessages(messages []context.Message) ([]context.Message, error) {
-	if a.nativeSupport {
+	if !a.nativeSupport {
-		// Native support, no preprocessing needed
+		// No vision support, remove image content
+		return a.removeImageContent(messages), nil
+	}
+
+	// Check if we need to convert format
+	needsConversion := a.format == context.VisionFormatClaude || a.format == context.VisionFormatBase64
+
+	if !needsConversion {
+		// Native support with OpenAI format or default, no preprocessing needed
 		return messages, nil
 	}
 
-	// Process messages to remove image content
+	// Convert image_url format to Claude base64 format
+	return a.convertToBase64Format(messages)
+}
+
+// removeImageContent removes image content from messages
+func (a *VisionAdapter) removeImageContent(messages []context.Message) []context.Message {
 	processed := make([]context.Message, 0, len(messages))
 	for _, msg := range messages {
 		processedMsg := msg
 
-		// Handle multimodal content (array of ContentPart)
+		// Handle multimodal content (array of map)
-		if contentParts, ok := msg.Content.([]context.ContentPart); ok {
+		if contentParts, ok := msg.Content.([]map[string]interface{}); ok {
-			filteredParts := make([]context.ContentPart, 0)
+			filteredParts := make([]map[string]interface{}, 0)
 
 			for _, part := range contentParts {
-				// Skip image content if not supported
+				partType, _ := part["type"].(string)
-				if part.Type == context.ContentImageURL {
+				// Skip image content
-					// TODO: Optionally convert to text description
+				if partType != "image_url" && partType != "image" {
-					continue
-				}
 					filteredParts = append(filteredParts, part)
 				}
+			}
 
 			// If all parts were filtered out, add placeholder text
 			if len(filteredParts) == 0 {
 				processedMsg.Content = "[Image content not supported by this model]"
-			} else if len(filteredParts) == 1 && filteredParts[0].Type == context.ContentText {
+			} else if len(filteredParts) == 1 {
-				// Single text part, convert to string
+				if textVal, ok := filteredParts[0]["text"].(string); ok {
-				processedMsg.Content = filteredParts[0].Text
+					processedMsg.Content = textVal
+				} else {
+					processedMsg.Content = filteredParts
+				}
 			} else {
 				processedMsg.Content = filteredParts
 			}
@@ -58,5 +82,154 @@ func (a *VisionAdapter) PreprocessMessages(messages []context.Message) ([]contex
 		processed = append(processed, processedMsg)
 	}
 
+	return processed
+}
+
+// convertToBase64Format converts image_url format to Claude base64 format
+func (a *VisionAdapter) convertToBase64Format(messages []context.Message) ([]context.Message, error) {
+	processed := make([]context.Message, 0, len(messages))
+
+	for _, msg := range messages {
+		processedMsg := msg
+
+		// Handle multimodal content
+		if contentParts, ok := msg.Content.([]map[string]interface{}); ok {
+			convertedParts := make([]map[string]interface{}, 0)
+
+			for _, part := range contentParts {
+				partType, _ := part["type"].(string)
+
+				if partType == "image_url" {
+					// Convert to base64 format
+					convertedPart, err := a.convertImageURLToBase64(part)
+					if err != nil {
+						// If conversion fails, skip this image
+						continue
+					}
+					convertedParts = append(convertedParts, convertedPart)
+				} else {
+					// Keep non-image parts as-is
+					convertedParts = append(convertedParts, part)
+				}
+			}
+
+			processedMsg.Content = convertedParts
+		}
+
+		processed = append(processed, processedMsg)
+	}
+
 	return processed, nil
 }
+
+// convertImageURLToBase64 converts OpenAI image_url format to Claude base64 format
+func (a *VisionAdapter) convertImageURLToBase64(part map[string]interface{}) (map[string]interface{}, error) {
+	// Extract URL from image_url object
+	imageURLObj, ok := part["image_url"].(map[string]interface{})
+	if !ok {
+		return nil, fmt.Errorf("invalid image_url format")
+	}
+
+	url, ok := imageURLObj["url"].(string)
+	if !ok || url == "" {
+		return nil, fmt.Errorf("missing or invalid URL in image_url")
+	}
+
+	// Check if already base64 data URL
+	if strings.HasPrefix(url, "data:") {
+		// Extract media type and base64 data from data URL
+		// Format: data:image/jpeg;base64,<base64_data>
+		parts := strings.SplitN(url, ",", 2)
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("invalid data URL format")
+		}
+
+		// Extract media type from first part
+		mediaParts := strings.Split(parts[0], ";")
+		mediaType := strings.TrimPrefix(mediaParts[0], "data:")
+		base64Data := parts[1]
+
+		return map[string]interface{}{
+			"type": "image",
+			"source": map[string]interface{}{
+				"type":       "base64",
+				"media_type": mediaType,
+				"data":       base64Data,
+			},
+		}, nil
+	}
+
+	// Download image from URL and convert to base64
+	base64Data, mediaType, err := a.downloadAndEncodeImage(url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to download image: %w", err)
+	}
+
+	// Return Claude/Anthropic format
+	return map[string]interface{}{
+		"type": "image",
+		"source": map[string]interface{}{
+			"type":       "base64",
+			"media_type": mediaType,
+			"data":       base64Data,
+		},
+	}, nil
+}
+
+// downloadAndEncodeImage downloads an image from URL and returns base64 encoded data
+func (a *VisionAdapter) downloadAndEncodeImage(url string) (string, string, error) {
+	// Create HTTP client with timeout
+	client := &http.Client{
+		Timeout: 30 * time.Second,
+	}
+
+	// Download image
+	resp, err := client.Get(url)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to download image: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return "", "", fmt.Errorf("failed to download image: HTTP %d", resp.StatusCode)
+	}
+
+	// Read image data
+	imageData, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to read image data: %w", err)
+	}
+
+	// Detect media type from Content-Type header
+	mediaType := resp.Header.Get("Content-Type")
+
+	// Normalize media type (remove charset and other parameters)
+	if mediaType != "" {
+		// Split by semicolon to remove parameters like "; charset=utf-8"
+		if idx := strings.Index(mediaType, ";"); idx != -1 {
+			mediaType = strings.TrimSpace(mediaType[:idx])
+		}
+	}
+
+	if mediaType == "" {
+		// Fallback to detecting from URL extension or default to jpeg
+		urlLower := strings.ToLower(url)
+		if strings.HasSuffix(urlLower, ".png") {
+			mediaType = "image/png"
+		} else if strings.HasSuffix(urlLower, ".gif") {
+			mediaType = "image/gif"
+		} else if strings.HasSuffix(urlLower, ".webp") {
+			mediaType = "image/webp"
+		} else if strings.Contains(urlLower, ".jpg") || strings.Contains(urlLower, ".jpeg") {
+			mediaType = "image/jpeg"
+		} else {
+			// Default to jpeg
+			mediaType = "image/jpeg"
+		}
+	}
+
+	// Encode to base64
+	base64Data := base64.StdEncoding.EncodeToString(imageData)
+
+	return base64Data, mediaType, nil
+}

agent/llm/providers/base/base.go

@@ -71,7 +71,11 @@ func (p *Provider) PreprocessMessages(messages []context.Message) ([]context.Mes
 
 // SupportsVision check if this provider supports vision
 func (p *Provider) SupportsVision() bool {
-	return p.Capabilities != nil && p.Capabilities.Vision != nil && *p.Capabilities.Vision
+	if p.Capabilities == nil {
+		return false
+	}
+	supported, _ := p.Capabilities.GetVisionSupport()
+	return supported
 }
 
 // SupportsAudio check if this provider supports audio

agent/llm/providers/openai/claude_test.go

@@ -0,0 +1,593 @@
+package openai_test
+
+import (
+	gocontext "context"
+	"testing"
+
+	"github.com/yaoapp/gou/connector"
+	"github.com/yaoapp/gou/plan"
+	"github.com/yaoapp/yao/agent/context"
+	"github.com/yaoapp/yao/agent/llm"
+	"github.com/yaoapp/yao/config"
+	"github.com/yaoapp/yao/openapi/oauth/types"
+	"github.com/yaoapp/yao/test"
+)
+
+// newClaudeTestContext creates a real Context for testing Claude provider
+func newClaudeTestContext(chatID, connectorID string) *context.Context {
+	return &context.Context{
+		Context:     gocontext.Background(),
+		Space:       plan.NewMemorySharedSpace(),
+		ChatID:      chatID,
+		AssistantID: "test-assistant",
+		Connector:   connectorID,
+		Locale:      "en-us",
+		Theme:       "light",
+		Client: context.Client{
+			Type:      "web",
+			UserAgent: "ClaudeProviderTest/1.0",
+			IP:        "127.0.0.1",
+		},
+		Referer:  context.RefererAPI,
+		Accept:   context.AcceptStandard,
+		Route:    "/api/test",
+		Metadata: make(map[string]interface{}),
+		Authorized: &types.AuthorizedInfo{
+			Subject:   "test-user",
+			ClientID:  "test-client",
+			UserID:    "test-user-123",
+			TeamID:    "test-team-456",
+			TenantID:  "test-tenant-789",
+			SessionID: "test-session-id",
+			Constraints: types.DataConstraints{
+				TeamOnly: true,
+				Extra: map[string]interface{}{
+					"test": "claude-provider",
+				},
+			},
+		},
+	}
+}
+
+// TestClaudeSonnet4StreamBasic tests basic streaming completion with Claude Sonnet 4
+func TestClaudeSonnet4StreamBasic(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	conn, err := connector.Select("claude.sonnet-4_0")
+	if err != nil {
+		t.Fatalf("Failed to select connector: %v", err)
+	}
+
+	trueVal := true
+	falseVal := false
+	options := &context.CompletionOptions{
+		Capabilities: &context.ModelCapabilities{
+			Streaming:  &trueVal,
+			Reasoning:  &falseVal, // Claude Sonnet 4 (non-thinking) doesn't expose reasoning
+			ToolCalls:  &trueVal,
+			Vision:     &trueVal,
+			Multimodal: &trueVal,
+		},
+	}
+
+	llmInstance, err := llm.New(conn, options)
+	if err != nil {
+		t.Fatalf("Failed to create LLM instance: %v", err)
+	}
+
+	messages := []context.Message{
+		{
+			Role:    context.RoleUser,
+			Content: "What is 3+3? Reply with just the number.",
+		},
+	}
+
+	maxTokens := 100
+	options.MaxTokens = &maxTokens
+
+	ctx := newClaudeTestContext("test-claude-sonnet4-basic", "claude.sonnet-4_0")
+
+	var chunks []string
+	handler := func(chunkType context.StreamChunkType, data []byte) int {
+		chunks = append(chunks, string(data))
+		t.Logf("Stream chunk [%s]: %s", chunkType, string(data))
+		return 0
+	}
+
+	response, err := llmInstance.Stream(ctx, messages, options, handler)
+	if err != nil {
+		t.Fatalf("Stream failed: %v", err)
+	}
+
+	if response == nil {
+		t.Fatal("Response is nil")
+	}
+
+	// Basic validation
+	if response.ID == "" {
+		t.Error("Response ID is empty")
+	}
+	if response.Model == "" {
+		t.Error("Response Model is empty")
+	}
+
+	// Validate content
+	contentStr, ok := response.Content.(string)
+	if !ok {
+		t.Errorf("Content is not a string: %T", response.Content)
+	}
+	if len(contentStr) == 0 {
+		t.Error("Content is empty")
+	}
+
+	t.Logf("Response content: %v", response.Content)
+	t.Logf("Usage: prompt=%d, completion=%d, total=%d",
+		response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens)
+
+	t.Logf("Final response: %+v", response)
+	t.Logf("Total chunks received: %d", len(chunks))
+}
+
+// TestClaudeSonnet4PostBasic tests non-streaming completion
+func TestClaudeSonnet4PostBasic(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	conn, err := connector.Select("claude.sonnet-4_0")
+	if err != nil {
+		t.Fatalf("Failed to select connector: %v", err)
+	}
+
+	trueVal := true
+	falseVal := false
+	options := &context.CompletionOptions{
+		Capabilities: &context.ModelCapabilities{
+			Streaming:  &falseVal,
+			Reasoning:  &falseVal,
+			ToolCalls:  &trueVal,
+			Vision:     &trueVal,
+			Multimodal: &trueVal,
+		},
+	}
+
+	llmInstance, err := llm.New(conn, options)
+	if err != nil {
+		t.Fatalf("Failed to create LLM instance: %v", err)
+	}
+
+	messages := []context.Message{
+		{
+			Role:    context.RoleUser,
+			Content: "What is 4+4? Reply with just the number.",
+		},
+	}
+
+	maxTokens := 100
+	options.MaxTokens = &maxTokens
+
+	ctx := newClaudeTestContext("test-claude-sonnet4-post", "claude.sonnet-4_0")
+
+	response, err := llmInstance.Post(ctx, messages, options)
+	if err != nil {
+		t.Fatalf("Post failed: %v", err)
+	}
+
+	if response == nil {
+		t.Fatal("Response is nil")
+	}
+
+	// Validate content
+	contentStr, ok := response.Content.(string)
+	if !ok {
+		t.Fatalf("Content is not a string: %T", response.Content)
+	}
+
+	t.Logf("Response content: %s", contentStr)
+	t.Logf("Usage: prompt=%d, completion=%d, total=%d",
+		response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens)
+
+	// Basic content validation
+	if len(contentStr) == 0 {
+		t.Error("Content is empty")
+	}
+
+	t.Logf("Response: %+v", response)
+}
+
+// TestClaudeSonnet4WithToolCalls tests tool calling capability
+func TestClaudeSonnet4WithToolCalls(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	conn, err := connector.Select("claude.sonnet-4_0")
+	if err != nil {
+		t.Fatalf("Failed to select connector: %v", err)
+	}
+
+	trueVal := true
+	falseVal := false
+	options := &context.CompletionOptions{
+		Capabilities: &context.ModelCapabilities{
+			Streaming:  &falseVal,
+			Reasoning:  &falseVal,
+			ToolCalls:  &trueVal,
+			Vision:     &trueVal,
+			Multimodal: &trueVal,
+		},
+	}
+
+	// Define a simple tool with minimal parameters
+	simpleTool := map[string]interface{}{
+		"type": "function",
+		"function": map[string]interface{}{
+			"name":        "get_info",
+			"description": "Get information",
+			"parameters": map[string]interface{}{
+				"type": "object",
+				"properties": map[string]interface{}{
+					"query": map[string]interface{}{
+						"type":        "string",
+						"description": "Query string (single letter)",
+					},
+					"count": map[string]interface{}{
+						"type":        "number",
+						"description": "Count (single digit)",
+					},
+				},
+				"required": []string{"query", "count"},
+			},
+		},
+	}
+
+	options.Tools = []map[string]interface{}{simpleTool}
+	options.ToolChoice = "auto"
+
+	// Set lower max_tokens for faster response
+	maxTokens := 50
+	options.MaxTokens = &maxTokens
+
+	llmInstance, err := llm.New(conn, options)
+	if err != nil {
+		t.Fatalf("Failed to create LLM instance: %v", err)
+	}
+
+	messages := []context.Message{
+		{
+			Role:    context.RoleUser,
+			Content: "Call get_info with query='A' and count=1",
+		},
+	}
+
+	ctx := newClaudeTestContext("test-claude-sonnet4-tools", "claude.sonnet-4_0")
+
+	response, err := llmInstance.Post(ctx, messages, options)
+	if err != nil {
+		t.Fatalf("Post failed: %v", err)
+	}
+
+	if response == nil {
+		t.Fatal("Response is nil")
+	}
+
+	// Validate tool calls
+	if len(response.ToolCalls) == 0 {
+		t.Error("No tool calls in response")
+	} else {
+		tc := response.ToolCalls[0]
+		t.Logf("✓ Tool call: %s(%s)", tc.Function.Name, tc.Function.Arguments)
+
+		if tc.Function.Name != "get_info" {
+			t.Errorf("Expected tool name 'get_info', got '%s'", tc.Function.Name)
+		}
+	}
+
+	t.Logf("Usage: prompt=%d, completion=%d, total=%d",
+		response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens)
+
+	t.Logf("Response: %+v", response)
+}
+
+// TestClaudeSonnet4Vision tests vision capability with image input
+func TestClaudeSonnet4Vision(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	conn, err := connector.Select("claude.sonnet-4_0")
+	if err != nil {
+		t.Fatalf("Failed to select connector: %v", err)
+	}
+
+	trueVal := true
+	falseVal := false
+	options := &context.CompletionOptions{
+		Capabilities: &context.ModelCapabilities{
+			Streaming:  &falseVal,
+			Reasoning:  &falseVal,
+			ToolCalls:  &trueVal,
+			Vision:     &trueVal,
+			Multimodal: &trueVal,
+		},
+	}
+
+	llmInstance, err := llm.New(conn, options)
+	if err != nil {
+		t.Fatalf("Failed to create LLM instance: %v", err)
+	}
+
+	// Use a test image URL
+	imageURL := "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
+
+	messages := []context.Message{
+		{
+			Role: context.RoleUser,
+			Content: []map[string]interface{}{
+				{
+					"type": "text",
+					"text": "Describe this image in one sentence.",
+				},
+				{
+					"type": "image_url",
+					"image_url": map[string]string{
+						"url": imageURL,
+					},
+				},
+			},
+		},
+	}
+
+	maxTokens := 150
+	options.MaxTokens = &maxTokens
+
+	ctx := newClaudeTestContext("test-claude-sonnet4-vision", "claude.sonnet-4_0")
+
+	response, err := llmInstance.Post(ctx, messages, options)
+	if err != nil {
+		t.Fatalf("Post failed: %v", err)
+	}
+
+	if response == nil {
+		t.Fatal("Response is nil")
+	}
+
+	// Validate content
+	contentStr, ok := response.Content.(string)
+	if !ok {
+		t.Fatalf("Content is not a string: %T", response.Content)
+	}
+
+	if len(contentStr) == 0 {
+		t.Error("Image description is empty")
+	}
+
+	t.Logf("Image description: %s", contentStr)
+	t.Logf("Usage: prompt=%d, completion=%d, total=%d",
+		response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens)
+}
+
+// TestClaudeSonnet4ThinkingStream tests Claude Sonnet 4 Thinking with streaming
+func TestClaudeSonnet4ThinkingStream(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	conn, err := connector.Select("claude.sonnet-4_0-thinking")
+	if err != nil {
+		t.Fatalf("Failed to select connector: %v", err)
+	}
+
+	trueVal := true
+	falseVal := false
+	options := &context.CompletionOptions{
+		Capabilities: &context.ModelCapabilities{
+			Streaming:  &trueVal,
+			Reasoning:  &trueVal, // Claude Thinking mode exposes reasoning
+			ToolCalls:  &falseVal,
+			Vision:     &trueVal,
+			Multimodal: &trueVal,
+		},
+	}
+
+	llmInstance, err := llm.New(conn, options)
+	if err != nil {
+		t.Fatalf("Failed to create LLM instance: %v", err)
+	}
+
+	messages := []context.Message{
+		{
+			Role:    context.RoleUser,
+			Content: "If Sally has 3 apples and gives 2 to John, how many does she have left? Think through this step by step.",
+		},
+	}
+
+	maxTokens := 500
+	options.MaxTokens = &maxTokens
+
+	ctx := newClaudeTestContext("test-claude-thinking-stream", "claude.sonnet-4_0-thinking")
+
+	var thinkingChunks []string
+	var textChunks []string
+	handler := func(chunkType context.StreamChunkType, data []byte) int {
+		t.Logf("Stream chunk [%s]: %s", chunkType, string(data))
+		if chunkType == context.ChunkThinking {
+			thinkingChunks = append(thinkingChunks, string(data))
+		} else if chunkType == context.ChunkText {
+			textChunks = append(textChunks, string(data))
+		}
+		return 0
+	}
+
+	response, err := llmInstance.Stream(ctx, messages, options, handler)
+	if err != nil {
+		t.Fatalf("Stream failed: %v", err)
+	}
+
+	if response == nil {
+		t.Fatal("Response is nil")
+	}
+
+	// Validate response
+	contentStr, ok := response.Content.(string)
+	if !ok {
+		t.Errorf("Content is not a string: %T", response.Content)
+	}
+
+	t.Logf("Reasoning/Thinking content length: %d characters", len(response.ReasoningContent))
+	t.Logf("Response content: %v", contentStr)
+	t.Logf("Usage: prompt=%d, completion=%d, total=%d",
+		response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens)
+
+	if response.Usage != nil && response.Usage.CompletionTokensDetails != nil {
+		t.Logf("Reasoning tokens: %d", response.Usage.CompletionTokensDetails.ReasoningTokens)
+	}
+
+	t.Logf("Received %d thinking chunks", len(thinkingChunks))
+	t.Logf("Received %d text chunks", len(textChunks))
+	t.Logf("Final response: %+v", response)
+}
+
+// TestClaudeSonnet4ThinkingPost tests Claude Sonnet 4 Thinking in non-streaming mode
+func TestClaudeSonnet4ThinkingPost(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	conn, err := connector.Select("claude.sonnet-4_0-thinking")
+	if err != nil {
+		t.Fatalf("Failed to select connector: %v", err)
+	}
+
+	trueVal := true
+	falseVal := false
+	options := &context.CompletionOptions{
+		Capabilities: &context.ModelCapabilities{
+			Streaming:  &falseVal,
+			Reasoning:  &trueVal,
+			ToolCalls:  &falseVal,
+			Vision:     &trueVal,
+			Multimodal: &trueVal,
+		},
+	}
+
+	llmInstance, err := llm.New(conn, options)
+	if err != nil {
+		t.Fatalf("Failed to create LLM instance: %v", err)
+	}
+
+	messages := []context.Message{
+		{
+			Role:    context.RoleUser,
+			Content: "Is 7 greater than 5? Explain your reasoning.",
+		},
+	}
+
+	maxTokens := 500
+	options.MaxTokens = &maxTokens
+
+	ctx := newClaudeTestContext("test-claude-thinking-post", "claude.sonnet-4_0-thinking")
+
+	response, err := llmInstance.Post(ctx, messages, options)
+	if err != nil {
+		t.Fatalf("Post failed: %v", err)
+	}
+
+	if response == nil {
+		t.Fatal("Response is nil")
+	}
+
+	// Validate content
+	contentStr, ok := response.Content.(string)
+	if !ok {
+		t.Fatalf("Content is not a string: %T", response.Content)
+	}
+
+	t.Logf("Reasoning content: %s", response.ReasoningContent)
+	t.Logf("Final answer: %s", contentStr)
+
+	// Check for reasoning content
+	if len(response.ReasoningContent) > 0 {
+		t.Logf("✓ Reasoning content present: %d characters", len(response.ReasoningContent))
+	}
+
+	if response.Usage != nil && response.Usage.CompletionTokensDetails != nil && response.Usage.CompletionTokensDetails.ReasoningTokens > 0 {
+		t.Logf("✓ Reasoning tokens: %d", response.Usage.CompletionTokensDetails.ReasoningTokens)
+	}
+
+	t.Logf("Usage: prompt=%d, completion=%d, total=%d",
+		response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens)
+
+	t.Logf("Response: %+v", response)
+}
+
+// TestClaudeTemperatureHandling tests that Claude models handle temperature parameter correctly
+func TestClaudeTemperatureHandling(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	tests := []struct {
+		name        string
+		connector   string
+		temperature float64
+		reasoning   bool
+	}{
+		{
+			name:        "Sonnet 4 with temperature 0.7",
+			connector:   "claude.sonnet-4_0",
+			temperature: 0.7,
+			reasoning:   false,
+		},
+		{
+			name:        "Sonnet 4 Thinking with temperature 0.5",
+			connector:   "claude.sonnet-4_0-thinking",
+			temperature: 0.5,
+			reasoning:   true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			conn, err := connector.Select(tt.connector)
+			if err != nil {
+				t.Fatalf("Failed to select connector: %v", err)
+			}
+
+			trueVal := true
+			falseVal := false
+			options := &context.CompletionOptions{
+				Capabilities: &context.ModelCapabilities{
+					Streaming: &falseVal,
+					Reasoning: &tt.reasoning,
+					ToolCalls: &trueVal,
+					Vision:    &trueVal,
+				},
+			}
+
+			llmInstance, err := llm.New(conn, options)
+			if err != nil {
+				t.Fatalf("Failed to create LLM instance: %v", err)
+			}
+
+			messages := []context.Message{
+				{
+					Role:    context.RoleUser,
+					Content: "Say 'hello'.",
+				},
+			}
+
+			maxTokens := 50
+			options.MaxTokens = &maxTokens
+			options.Temperature = &tt.temperature
+
+			ctx := newClaudeTestContext("test-claude-temp-"+tt.connector, tt.connector)
+
+			response, err := llmInstance.Post(ctx, messages, options)
+			if err != nil {
+				t.Fatalf("Post failed: %v", err)
+			}
+
+			if response == nil {
+				t.Fatal("Response is nil")
+			}
+
+			t.Logf("✓ %s completed successfully with temperature=%.1f", tt.name, tt.temperature)
+		})
+	}
+}

agent/llm/providers/openai/deepseek_v3_test.go

@@ -234,32 +234,36 @@ func TestDeepSeekV3WithToolCalls(t *testing.T) {
 		},
 	}
 
-	// Define a weather tool
+	// Define a simple tool with minimal parameters
-	weatherTool := map[string]interface{}{
+	simpleTool := map[string]interface{}{
 		"type": "function",
 		"function": map[string]interface{}{
-			"name":        "get_weather",
+			"name":        "get_info",
-			"description": "Get current weather for a location",
+			"description": "Get information",
 			"parameters": map[string]interface{}{
 				"type": "object",
 				"properties": map[string]interface{}{
-					"location": map[string]interface{}{
+					"query": map[string]interface{}{
 						"type":        "string",
-						"description": "City name",
+						"description": "Query string (single letter)",
 					},
-					"unit": map[string]interface{}{
+					"count": map[string]interface{}{
-						"type": "string",
+						"type":        "number",
-						"enum": []string{"celsius", "fahrenheit"},
+						"description": "Count (single digit)",
 					},
 				},
-				"required": []string{"location"},
+				"required": []string{"query", "count"},
 			},
 		},
 	}
 
-	options.Tools = []map[string]interface{}{weatherTool}
+	options.Tools = []map[string]interface{}{simpleTool}
 	options.ToolChoice = "auto"
 
+	// Set lower max_tokens for faster response
+	maxTokens := 50
+	options.MaxTokens = &maxTokens
+
 	llmInstance, err := llm.New(conn, options)
 	if err != nil {
 		t.Fatalf("Failed to create LLM instance: %v", err)
@@ -268,7 +272,7 @@ func TestDeepSeekV3WithToolCalls(t *testing.T) {
 	messages := []context.Message{
 		{
 			Role:    context.RoleUser,
-			Content: "What's the weather in Beijing?",
+			Content: "Call get_info with query='A' and count=1",
 		},
 	}
 
@@ -290,8 +294,8 @@ func TestDeepSeekV3WithToolCalls(t *testing.T) {
 		tc := response.ToolCalls[0]
 		t.Logf("✓ Tool call: %s(%s)", tc.Function.Name, tc.Function.Arguments)
 
-		if tc.Function.Name != "get_weather" {
+		if tc.Function.Name != "get_info" {
-			t.Errorf("Expected tool name 'get_weather', got '%s'", tc.Function.Name)
+			t.Errorf("Expected tool name 'get_info', got '%s'", tc.Function.Name)
 		}
 	}
 
@@ -332,11 +336,11 @@ func TestDeepSeekV3NoReasoningEffort(t *testing.T) {
 	messages := []context.Message{
 		{
 			Role:    context.RoleUser,
-			Content: "Say 'OK'",
+			Content: "Reply with just: OK",
 		},
 	}
 
-	maxTokens := 10
+	maxTokens := 20
 	options.MaxTokens = &maxTokens
 
 	ctx := newDeepSeekV3TestContext("test-deepseek-v3-no-reasoning", "deepseek.v3")

agent/llm/providers/openai/openai.go

@@ -110,6 +110,19 @@ type Provider struct {
 	adapters []adapters.CapabilityAdapter
 }
 
+// buildAPIURL builds the complete API URL from host and endpoint
+// If host ends with /, it's used as-is (user has specified full path)
+// Otherwise, /v1 prefix is added automatically (standard for OpenAI-compatible APIs)
+func buildAPIURL(host, endpoint string) string {
+	// If host ends with /, use it as-is (user has specified full path like /v1/ or /api/)
+	// Otherwise, add /v1 prefix (standard for OpenAI-compatible APIs)
+	if !strings.HasSuffix(host, "/") {
+		endpoint = "/v1" + endpoint
+	}
+	host = strings.TrimSuffix(host, "/")
+	return host + endpoint
+}
+
 // New create a new OpenAI provider with capability adapters
 func New(conn connector.Connector, capabilities *context.ModelCapabilities) *Provider {
 	return &Provider{
@@ -132,8 +145,12 @@ func buildAdapters(cap *context.ModelCapabilities) []adapters.CapabilityAdapter
 	}
 
 	// Vision adapter
-	if cap.Vision != nil {
+	visionSupport, visionFormat := cap.GetVisionSupport()
-		result = append(result, adapters.NewVisionAdapter(*cap.Vision))
+	if visionSupport {
+		result = append(result, adapters.NewVisionAdapter(true, visionFormat))
+	} else if cap.Vision != nil {
+		// Vision explicitly disabled, add adapter to remove image content
+		result = append(result, adapters.NewVisionAdapter(false, context.VisionFormatNone))
 	}
 
 	// Audio adapter
@@ -288,22 +305,34 @@ func (p *Provider) streamWithRetry(ctx *context.Context, messages []context.Mess
 		}
 	}
 
-	// Preprocess options through adapters
+	// Preprocess messages and options through adapters
+	processedMessages := messages
 	processedOptions := options
 	for _, adapter := range p.adapters {
+		// Preprocess messages
+		newMessages, err := adapter.PreprocessMessages(processedMessages)
+		if err != nil {
+			if handler != nil {
+				handler(context.ChunkError, []byte(fmt.Sprintf("adapter %s message preprocessing failed: %v", adapter.Name(), err)))
+			}
+			return nil, fmt.Errorf("adapter %s message preprocessing failed: %w", adapter.Name(), err)
+		}
+		processedMessages = newMessages
+
+		// Preprocess options
 		newOpts, err := adapter.PreprocessOptions(processedOptions)
 		if err != nil {
 			// Send error to handler
 			if handler != nil {
-				handler(context.ChunkError, []byte(fmt.Sprintf("adapter %s preprocessing failed: %v", adapter.Name(), err)))
+				handler(context.ChunkError, []byte(fmt.Sprintf("adapter %s option preprocessing failed: %v", adapter.Name(), err)))
 			}
-			return nil, fmt.Errorf("adapter %s preprocessing failed: %w", adapter.Name(), err)
+			return nil, fmt.Errorf("adapter %s option preprocessing failed: %w", adapter.Name(), err)
 		}
 		processedOptions = newOpts
 	}
 
 	// Build request body
-	requestBody, err := p.buildRequestBody(messages, processedOptions, true)
+	requestBody, err := p.buildRequestBody(processedMessages, processedOptions, true)
 	if err != nil {
 		// Send stream_end with error
 		if handler != nil {
@@ -334,12 +363,7 @@ func (p *Provider) streamWithRetry(ctx *context.Context, messages []context.Mess
 	}
 
 	// Build URL
-	endpoint := "/chat/completions"
+	url := buildAPIURL(host, "/chat/completions")
-	if host == "https://api.openai.com" && !strings.HasPrefix(endpoint, "/v1") {
-		endpoint = "/v1" + endpoint
-	}
-	host = strings.TrimSuffix(host, "/")
-	url := host + endpoint
 
 	// Create HTTP request with proxy support
 	req := http.New(url).
@@ -836,18 +860,27 @@ func (p *Provider) Post(ctx *context.Context, messages []context.Message, option
 
 // postWithRetry performs a single POST request attempt
 func (p *Provider) postWithRetry(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error) {
-	// Preprocess options through adapters
+	// Preprocess messages and options through adapters
+	processedMessages := messages
 	processedOptions := options
 	for _, adapter := range p.adapters {
+		// Preprocess messages
+		newMessages, err := adapter.PreprocessMessages(processedMessages)
+		if err != nil {
+			return nil, fmt.Errorf("adapter %s message preprocessing failed: %w", adapter.Name(), err)
+		}
+		processedMessages = newMessages
+
+		// Preprocess options
 		newOpts, err := adapter.PreprocessOptions(processedOptions)
 		if err != nil {
-			return nil, fmt.Errorf("adapter %s preprocessing failed: %w", adapter.Name(), err)
+			return nil, fmt.Errorf("adapter %s option preprocessing failed: %w", adapter.Name(), err)
 		}
 		processedOptions = newOpts
 	}
 
 	// Build request body
-	requestBody, err := p.buildRequestBody(messages, processedOptions, false)
+	requestBody, err := p.buildRequestBody(processedMessages, processedOptions, false)
 	if err != nil {
 		return nil, fmt.Errorf("failed to build request body: %w", err)
 	}
@@ -865,12 +898,7 @@ func (p *Provider) postWithRetry(ctx *context.Context, messages []context.Messag
 	}
 
 	// Build URL
-	endpoint := "/chat/completions"
+	url := buildAPIURL(host, "/chat/completions")
-	if host == "https://api.openai.com" && !strings.HasPrefix(endpoint, "/v1") {
-		endpoint = "/v1" + endpoint
-	}
-	host = strings.TrimSuffix(host, "/")
-	url := host + endpoint
 
 	// Create HTTP request with proxy support
 	req := http.New(url).
@@ -880,7 +908,24 @@ func (p *Provider) postWithRetry(ctx *context.Context, messages []context.Messag
 	// Make request
 	resp := req.Post(requestBody)
 	if resp.Code != 200 {
-		return nil, fmt.Errorf("HTTP %d: %s", resp.Code, resp.Message)
+		// Try to get detailed error message from response
+		errorMsg := resp.Message
+		if resp.Data != nil {
+			if errorData, ok := resp.Data.(map[string]interface{}); ok {
+				if errObj, ok := errorData["error"]; ok {
+					if errMap, ok := errObj.(map[string]interface{}); ok {
+						if msg, ok := errMap["message"].(string); ok {
+							errorMsg = msg
+						}
+					}
+				}
+			}
+			// Log full response data for debugging
+			if respJSON, err := jsoniter.Marshal(resp.Data); err == nil {
+				log.Error("OpenAI API error response: %s", string(respJSON))
+			}
+		}
+		return nil, fmt.Errorf("HTTP %d: %s", resp.Code, errorMsg)
 	}
 
 	// Parse response