feat: add multimodal vision support across all channels

Convert image handling from file paths to base64 data URLs and pass them
through the message pipeline to the LLM using OpenAI's multi-part content
format (text + image_url). This enables vision capabilities for all
supported channels (Discord, Telegram, LINE, Slack, WebSocket).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

pkg/agent/context.go

@@ -220,10 +220,14 @@ func (cb *ContextBuilder) BuildMessages(history []providers.Message, summary str
 
 	messages = append(messages, history...)
 
-	messages = append(messages, providers.Message{
+	userMsg := providers.Message{
 		Role:    "user",
 		Content: currentMessage,
-	})
+	}
+	if len(media) > 0 {
+		userMsg.Media = media
+	}
+	messages = append(messages, userMsg)
 
 	return messages
 }

pkg/agent/loop.go

@@ -53,6 +53,7 @@ type processOptions struct {
 	Channel         string   // Target channel for tool execution
 	ChatID          string   // Target chat ID for tool execution
 	UserMessage     string   // User message content (may include prefix)
+	Media           []string // Base64 data URLs for images
 	DefaultResponse string   // Response when LLM returns empty
 	EnableSummary   bool     // Whether to trigger summarization
 	SendResponse    bool     // Whether to send response via bus
@@ -310,6 +311,7 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage)
 		Channel:         msg.Channel,
 		ChatID:          msg.ChatID,
 		UserMessage:     msg.Content,
+		Media:           msg.Media,
 		DefaultResponse: "I've completed processing but have no response to give.",
 		EnableSummary:   true,
 		SendResponse:    false,
@@ -396,7 +398,7 @@ func (al *AgentLoop) runAgentLoop(ctx context.Context, opts processOptions) (str
 		history,
 		summary,
 		opts.UserMessage,
-		nil,
+		opts.Media,
 		opts.Channel,
 		opts.ChatID,
 	)

pkg/channels/discord.go

@@ -353,12 +353,15 @@ func (c *DiscordChannel) handleMessage(s *discordgo.Session, m *discordgo.Messag
 					"url":      attachment.URL,
 					"filename": attachment.Filename,
 				})
-				mediaPaths = append(mediaPaths, attachment.URL)
-				content = appendContent(content, fmt.Sprintf("[attachment: %s]", attachment.URL))
 			}
 		} else {
-			mediaPaths = append(mediaPaths, attachment.URL)
-			content = appendContent(content, fmt.Sprintf("[attachment: %s]", attachment.URL))
+			localPath := c.downloadAttachment(attachment.URL, attachment.Filename)
+			if localPath != "" {
+				localFiles = append(localFiles, localPath)
+				if dataURL := utils.EncodeFileToDataURL(localPath); dataURL != "" {
+					mediaPaths = append(mediaPaths, dataURL)
+				}
+			}
 		}
 	}
 
@@ -366,10 +369,6 @@ func (c *DiscordChannel) handleMessage(s *discordgo.Session, m *discordgo.Messag
 		return
 	}
 
-	if content == "" {
-		content = "[media only]"
-	}
-
 	logger.DebugCF("discord", "Received message", map[string]any{
 		"sender_name": senderName,
 		"sender_id":   senderID,

pkg/channels/line.go

@@ -331,8 +331,9 @@ func (c *LINEChannel) processEvent(event lineEvent) {
 		localPath := c.downloadContent(msg.ID, "image.jpg")
 		if localPath != "" {
 			localFiles = append(localFiles, localPath)
-			mediaPaths = append(mediaPaths, localPath)
-			content = "[image]"
+			if dataURL := utils.EncodeFileToDataURL(localPath); dataURL != "" {
+				mediaPaths = append(mediaPaths, dataURL)
+			}
 		}
 	case "audio":
 		localPath := c.downloadContent(msg.ID, "audio.m4a")
@@ -356,7 +357,7 @@ func (c *LINEChannel) processEvent(event lineEvent) {
 		content = fmt.Sprintf("[%s]", msg.Type)
 	}
 
-	if strings.TrimSpace(content) == "" {
+	if strings.TrimSpace(content) == "" && len(mediaPaths) == 0 {
 		return
 	}
 

pkg/channels/slack.go

@@ -251,7 +251,6 @@ func (c *SlackChannel) handleMessageEvent(ev *slackevents.MessageEvent) {
 				continue
 			}
 			localFiles = append(localFiles, localPath)
-			mediaPaths = append(mediaPaths, localPath)
 
 			if utils.IsAudioFile(file.Name, file.Mimetype) && c.transcriber != nil && c.transcriber.IsAvailable() {
 				ctx, cancel := context.WithTimeout(c.ctx, 30*time.Second)
@@ -264,13 +263,15 @@ func (c *SlackChannel) handleMessageEvent(ev *slackevents.MessageEvent) {
 				} else {
 					content += fmt.Sprintf("\n[voice transcription: %s]", result.Text)
 				}
+			} else if dataURL := utils.EncodeFileToDataURL(localPath); dataURL != "" {
+				mediaPaths = append(mediaPaths, dataURL)
 			} else {
 				content += fmt.Sprintf("\n[file: %s]", file.Name)
 			}
 		}
 	}
 
-	if strings.TrimSpace(content) == "" {
+	if strings.TrimSpace(content) == "" && len(mediaPaths) == 0 {
 		return
 	}
 

pkg/channels/telegram.go

@@ -244,11 +244,9 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, message *telego.Mes
 		photoPath := c.downloadPhoto(ctx, photo.FileID)
 		if photoPath != "" {
 			localFiles = append(localFiles, photoPath)
-			mediaPaths = append(mediaPaths, photoPath)
-			if content != "" {
-				content += "\n"
+			if dataURL := utils.EncodeFileToDataURL(photoPath); dataURL != "" {
+				mediaPaths = append(mediaPaths, dataURL)
 			}
-			content += "[image: photo]"
 		}
 	}
 
@@ -311,7 +309,7 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, message *telego.Mes
 		}
 	}
 
-	if content == "" {
+	if content == "" && len(mediaPaths) == 0 {
 		content = "[empty message]"
 	}
 

pkg/channels/websocket.go

@@ -2,12 +2,9 @@ package channels
 
 import (
 	"context"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"os"
-	"path/filepath"
 	"sync"
 
 	"github.com/google/uuid"
@@ -235,19 +232,10 @@ func (c *WebSocketChannel) readPump(conn *websocket.Conn, clientID, chatID strin
 		content := incoming.Content
 		var media []string
 
-		// Save images to temp files (same pattern as Telegram).
-		for i, imgData := range incoming.Images {
-			path, err := c.saveImage(imgData)
-			if err != nil {
-				logger.ErrorCF("websocket", "Failed to save image", map[string]interface{}{
-					"client_id": clientID,
-					"index":     i,
-					"error":     err.Error(),
-				})
-				continue
-			}
-			media = append(media, path)
-			content += fmt.Sprintf("\n[image: photo_%d]", i)
+		// Convert base64 images directly to data URLs.
+		for _, imgData := range incoming.Images {
+			dataURL := "data:image/png;base64," + imgData
+			media = append(media, dataURL)
 		}
 
 		logger.DebugCF("websocket", "Received message", map[string]interface{}{
@@ -260,30 +248,3 @@ func (c *WebSocketChannel) readPump(conn *websocket.Conn, clientID, chatID strin
 	}
 }
 
-func (c *WebSocketChannel) saveImage(base64Data string) (string, error) {
-	data, err := base64.StdEncoding.DecodeString(base64Data)
-	if err != nil {
-		// Try URL-safe base64.
-		data, err = base64.URLEncoding.DecodeString(base64Data)
-		if err != nil {
-			return "", fmt.Errorf("failed to decode base64: %w", err)
-		}
-	}
-
-	tmpDir := filepath.Join(os.TempDir(), "picoclaw", "ws_images")
-	if err := os.MkdirAll(tmpDir, 0755); err != nil {
-		return "", fmt.Errorf("failed to create temp dir: %w", err)
-	}
-
-	f, err := os.CreateTemp(tmpDir, "ws_img_*.png")
-	if err != nil {
-		return "", fmt.Errorf("failed to create temp file: %w", err)
-	}
-	defer f.Close()
-
-	if _, err := f.Write(data); err != nil {
-		return "", fmt.Errorf("failed to write image: %w", err)
-	}
-
-	return f.Name(), nil
-}

pkg/providers/http_provider.go

@@ -65,7 +65,7 @@ func (p *HTTPProvider) Chat(ctx context.Context, messages []Message, tools []Too
 
 	requestBody := map[string]interface{}{
 		"model":    model,
-		"messages": messages,
+		"messages": p.buildAPIMessages(messages),
 	}
 
 	if len(tools) > 0 {
@@ -124,6 +124,51 @@ func (p *HTTPProvider) Chat(ctx context.Context, messages []Message, tools []Too
 	return p.parseResponse(body)
 }
 
+// buildAPIMessages converts internal Message slice to OpenAI API format.
+// If a user message has Media, content becomes an array of text + image_url objects.
+func (p *HTTPProvider) buildAPIMessages(messages []Message) []map[string]interface{} {
+	result := make([]map[string]interface{}, 0, len(messages))
+
+	for _, msg := range messages {
+		m := map[string]interface{}{
+			"role": msg.Role,
+		}
+
+		// Only user messages with media get the array-style content
+		if msg.Role == "user" && len(msg.Media) > 0 {
+			parts := make([]map[string]interface{}, 0, 1+len(msg.Media))
+			if msg.Content != "" {
+				parts = append(parts, map[string]interface{}{
+					"type": "text",
+					"text": msg.Content,
+				})
+			}
+			for _, dataURL := range msg.Media {
+				parts = append(parts, map[string]interface{}{
+					"type": "image_url",
+					"image_url": map[string]string{
+						"url": dataURL,
+					},
+				})
+			}
+			m["content"] = parts
+		} else {
+			m["content"] = msg.Content
+		}
+
+		if len(msg.ToolCalls) > 0 {
+			m["tool_calls"] = msg.ToolCalls
+		}
+		if msg.ToolCallID != "" {
+			m["tool_call_id"] = msg.ToolCallID
+		}
+
+		result = append(result, m)
+	}
+
+	return result
+}
+
 func (p *HTTPProvider) parseResponse(body []byte) (*LLMResponse, error) {
 	var apiResponse struct {
 		Choices []struct {

pkg/providers/types.go

@@ -31,6 +31,7 @@ type UsageInfo struct {
 type Message struct {
 	Role       string     `json:"role"`
 	Content    string     `json:"content"`
+	Media      []string   `json:"media,omitempty"`
 	ToolCalls  []ToolCall `json:"tool_calls,omitempty"`
 	ToolCallID string     `json:"tool_call_id,omitempty"`
 }

pkg/utils/media.go

@@ -1,6 +1,7 @@
 package utils
 
 import (
+	"encoding/base64"
 	"io"
 	"net/http"
 	"os"
@@ -12,6 +13,61 @@ import (
 	"github.com/sipeed/picoclaw/pkg/logger"
 )
 
+const maxImageFileSize = 50 * 1024 * 1024 // 50MB
+
+// EncodeFileToDataURL reads a local file and returns a base64 data URL string.
+// Supported: JPEG, PNG, WEBP, GIF. Max 50MB.
+// Returns empty string on error.
+func EncodeFileToDataURL(path string) string {
+	ext := strings.ToLower(filepath.Ext(path))
+	var mime string
+	switch ext {
+	case ".jpg", ".jpeg":
+		mime = "image/jpeg"
+	case ".png":
+		mime = "image/png"
+	case ".webp":
+		mime = "image/webp"
+	case ".gif":
+		mime = "image/gif"
+	default:
+		logger.WarnCF("media", "Unsupported image extension", map[string]interface{}{
+			"path": path,
+			"ext":  ext,
+		})
+		return ""
+	}
+
+	info, err := os.Stat(path)
+	if err != nil {
+		logger.ErrorCF("media", "Failed to stat image file", map[string]interface{}{
+			"path":  path,
+			"error": err.Error(),
+		})
+		return ""
+	}
+	if info.Size() > maxImageFileSize {
+		logger.WarnCF("media", "Image file too large, skipping", map[string]interface{}{
+			"path":     path,
+			"size_mb":  info.Size() / (1024 * 1024),
+			"max_mb":   maxImageFileSize / (1024 * 1024),
+		})
+		return ""
+	}
+
+	data, err := os.ReadFile(path)
+	if err != nil {
+		logger.ErrorCF("media", "Failed to read image file", map[string]interface{}{
+			"path":  path,
+			"error": err.Error(),
+		})
+		return ""
+	}
+
+	encoded := base64.StdEncoding.EncodeToString(data)
+	return "data:" + mime + ";base64," + encoded
+}
+
 // IsAudioFile checks if a file is an audio file based on its filename extension and content type.
 func IsAudioFile(filename, contentType string) bool {
 	audioExtensions := []string{".mp3", ".wav", ".ogg", ".m4a", ".flac", ".aac", ".wma"}