From f6d32bf951dc780872bef1a0a7bfab0bf1444656 Mon Sep 17 00:00:00 2001 From: KoheiYamashita Date: Wed, 18 Feb 2026 10:38:45 +0900 Subject: [PATCH] feat: add multimodal vision support across all channels Convert image handling from file paths to base64 data URLs and pass them through the message pipeline to the LLM using OpenAI's multi-part content format (text + image_url). This enables vision capabilities for all supported channels (Discord, Telegram, LINE, Slack, WebSocket). Co-Authored-By: Claude Opus 4.6 --- pkg/agent/context.go | 8 +++-- pkg/agent/loop.go | 20 ++++++------ pkg/channels/discord.go | 15 +++++---- pkg/channels/line.go | 7 +++-- pkg/channels/slack.go | 5 +-- pkg/channels/telegram.go | 8 ++--- pkg/channels/websocket.go | 47 +++------------------------- pkg/providers/http_provider.go | 47 +++++++++++++++++++++++++++- pkg/providers/types.go | 1 + pkg/utils/media.go | 56 ++++++++++++++++++++++++++++++++++ 10 files changed, 141 insertions(+), 73 deletions(-) diff --git a/pkg/agent/context.go b/pkg/agent/context.go index 4e72a08af..4231241a0 100644 --- a/pkg/agent/context.go +++ b/pkg/agent/context.go @@ -220,10 +220,14 @@ func (cb *ContextBuilder) BuildMessages(history []providers.Message, summary str messages = append(messages, history...) - messages = append(messages, providers.Message{ + userMsg := providers.Message{ Role: "user", Content: currentMessage, - }) + } + if len(media) > 0 { + userMsg.Media = media + } + messages = append(messages, userMsg) return messages } diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 208eb4a02..fe7c2515d 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -49,14 +49,15 @@ type AgentLoop struct { // processOptions configures how a message is processed type processOptions struct { - SessionKey string // Session identifier for history/context - Channel string // Target channel for tool execution - ChatID string // Target chat ID for tool execution - UserMessage string // User message content (may include prefix) - DefaultResponse string // Response when LLM returns empty - EnableSummary bool // Whether to trigger summarization - SendResponse bool // Whether to send response via bus - NoHistory bool // If true, don't load session history (for heartbeat) + SessionKey string // Session identifier for history/context + Channel string // Target channel for tool execution + ChatID string // Target chat ID for tool execution + UserMessage string // User message content (may include prefix) + Media []string // Base64 data URLs for images + DefaultResponse string // Response when LLM returns empty + EnableSummary bool // Whether to trigger summarization + SendResponse bool // Whether to send response via bus + NoHistory bool // If true, don't load session history (for heartbeat) } // createToolRegistry creates a tool registry with common tools. @@ -310,6 +311,7 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage) Channel: msg.Channel, ChatID: msg.ChatID, UserMessage: msg.Content, + Media: msg.Media, DefaultResponse: "I've completed processing but have no response to give.", EnableSummary: true, SendResponse: false, @@ -396,7 +398,7 @@ func (al *AgentLoop) runAgentLoop(ctx context.Context, opts processOptions) (str history, summary, opts.UserMessage, - nil, + opts.Media, opts.Channel, opts.ChatID, ) diff --git a/pkg/channels/discord.go b/pkg/channels/discord.go index 00aa8ab4d..3a97f9c33 100644 --- a/pkg/channels/discord.go +++ b/pkg/channels/discord.go @@ -353,12 +353,15 @@ func (c *DiscordChannel) handleMessage(s *discordgo.Session, m *discordgo.Messag "url": attachment.URL, "filename": attachment.Filename, }) - mediaPaths = append(mediaPaths, attachment.URL) - content = appendContent(content, fmt.Sprintf("[attachment: %s]", attachment.URL)) } } else { - mediaPaths = append(mediaPaths, attachment.URL) - content = appendContent(content, fmt.Sprintf("[attachment: %s]", attachment.URL)) + localPath := c.downloadAttachment(attachment.URL, attachment.Filename) + if localPath != "" { + localFiles = append(localFiles, localPath) + if dataURL := utils.EncodeFileToDataURL(localPath); dataURL != "" { + mediaPaths = append(mediaPaths, dataURL) + } + } } } @@ -366,10 +369,6 @@ func (c *DiscordChannel) handleMessage(s *discordgo.Session, m *discordgo.Messag return } - if content == "" { - content = "[media only]" - } - logger.DebugCF("discord", "Received message", map[string]any{ "sender_name": senderName, "sender_id": senderID, diff --git a/pkg/channels/line.go b/pkg/channels/line.go index ffb5533e8..c708ded7c 100644 --- a/pkg/channels/line.go +++ b/pkg/channels/line.go @@ -331,8 +331,9 @@ func (c *LINEChannel) processEvent(event lineEvent) { localPath := c.downloadContent(msg.ID, "image.jpg") if localPath != "" { localFiles = append(localFiles, localPath) - mediaPaths = append(mediaPaths, localPath) - content = "[image]" + if dataURL := utils.EncodeFileToDataURL(localPath); dataURL != "" { + mediaPaths = append(mediaPaths, dataURL) + } } case "audio": localPath := c.downloadContent(msg.ID, "audio.m4a") @@ -356,7 +357,7 @@ func (c *LINEChannel) processEvent(event lineEvent) { content = fmt.Sprintf("[%s]", msg.Type) } - if strings.TrimSpace(content) == "" { + if strings.TrimSpace(content) == "" && len(mediaPaths) == 0 { return } diff --git a/pkg/channels/slack.go b/pkg/channels/slack.go index 5387e9213..1bf367bc4 100644 --- a/pkg/channels/slack.go +++ b/pkg/channels/slack.go @@ -251,7 +251,6 @@ func (c *SlackChannel) handleMessageEvent(ev *slackevents.MessageEvent) { continue } localFiles = append(localFiles, localPath) - mediaPaths = append(mediaPaths, localPath) if utils.IsAudioFile(file.Name, file.Mimetype) && c.transcriber != nil && c.transcriber.IsAvailable() { ctx, cancel := context.WithTimeout(c.ctx, 30*time.Second) @@ -264,13 +263,15 @@ func (c *SlackChannel) handleMessageEvent(ev *slackevents.MessageEvent) { } else { content += fmt.Sprintf("\n[voice transcription: %s]", result.Text) } + } else if dataURL := utils.EncodeFileToDataURL(localPath); dataURL != "" { + mediaPaths = append(mediaPaths, dataURL) } else { content += fmt.Sprintf("\n[file: %s]", file.Name) } } } - if strings.TrimSpace(content) == "" { + if strings.TrimSpace(content) == "" && len(mediaPaths) == 0 { return } diff --git a/pkg/channels/telegram.go b/pkg/channels/telegram.go index 5601d508c..7b43c5df2 100644 --- a/pkg/channels/telegram.go +++ b/pkg/channels/telegram.go @@ -244,11 +244,9 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, message *telego.Mes photoPath := c.downloadPhoto(ctx, photo.FileID) if photoPath != "" { localFiles = append(localFiles, photoPath) - mediaPaths = append(mediaPaths, photoPath) - if content != "" { - content += "\n" + if dataURL := utils.EncodeFileToDataURL(photoPath); dataURL != "" { + mediaPaths = append(mediaPaths, dataURL) } - content += "[image: photo]" } } @@ -311,7 +309,7 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, message *telego.Mes } } - if content == "" { + if content == "" && len(mediaPaths) == 0 { content = "[empty message]" } diff --git a/pkg/channels/websocket.go b/pkg/channels/websocket.go index dab08a64f..1a50f6b16 100644 --- a/pkg/channels/websocket.go +++ b/pkg/channels/websocket.go @@ -2,12 +2,9 @@ package channels import ( "context" - "encoding/base64" "encoding/json" "fmt" "net/http" - "os" - "path/filepath" "sync" "github.com/google/uuid" @@ -235,19 +232,10 @@ func (c *WebSocketChannel) readPump(conn *websocket.Conn, clientID, chatID strin content := incoming.Content var media []string - // Save images to temp files (same pattern as Telegram). - for i, imgData := range incoming.Images { - path, err := c.saveImage(imgData) - if err != nil { - logger.ErrorCF("websocket", "Failed to save image", map[string]interface{}{ - "client_id": clientID, - "index": i, - "error": err.Error(), - }) - continue - } - media = append(media, path) - content += fmt.Sprintf("\n[image: photo_%d]", i) + // Convert base64 images directly to data URLs. + for _, imgData := range incoming.Images { + dataURL := "data:image/png;base64," + imgData + media = append(media, dataURL) } logger.DebugCF("websocket", "Received message", map[string]interface{}{ @@ -260,30 +248,3 @@ func (c *WebSocketChannel) readPump(conn *websocket.Conn, clientID, chatID strin } } -func (c *WebSocketChannel) saveImage(base64Data string) (string, error) { - data, err := base64.StdEncoding.DecodeString(base64Data) - if err != nil { - // Try URL-safe base64. - data, err = base64.URLEncoding.DecodeString(base64Data) - if err != nil { - return "", fmt.Errorf("failed to decode base64: %w", err) - } - } - - tmpDir := filepath.Join(os.TempDir(), "picoclaw", "ws_images") - if err := os.MkdirAll(tmpDir, 0755); err != nil { - return "", fmt.Errorf("failed to create temp dir: %w", err) - } - - f, err := os.CreateTemp(tmpDir, "ws_img_*.png") - if err != nil { - return "", fmt.Errorf("failed to create temp file: %w", err) - } - defer f.Close() - - if _, err := f.Write(data); err != nil { - return "", fmt.Errorf("failed to write image: %w", err) - } - - return f.Name(), nil -} diff --git a/pkg/providers/http_provider.go b/pkg/providers/http_provider.go index aba3fe2b8..866a099e5 100644 --- a/pkg/providers/http_provider.go +++ b/pkg/providers/http_provider.go @@ -65,7 +65,7 @@ func (p *HTTPProvider) Chat(ctx context.Context, messages []Message, tools []Too requestBody := map[string]interface{}{ "model": model, - "messages": messages, + "messages": p.buildAPIMessages(messages), } if len(tools) > 0 { @@ -124,6 +124,51 @@ func (p *HTTPProvider) Chat(ctx context.Context, messages []Message, tools []Too return p.parseResponse(body) } +// buildAPIMessages converts internal Message slice to OpenAI API format. +// If a user message has Media, content becomes an array of text + image_url objects. +func (p *HTTPProvider) buildAPIMessages(messages []Message) []map[string]interface{} { + result := make([]map[string]interface{}, 0, len(messages)) + + for _, msg := range messages { + m := map[string]interface{}{ + "role": msg.Role, + } + + // Only user messages with media get the array-style content + if msg.Role == "user" && len(msg.Media) > 0 { + parts := make([]map[string]interface{}, 0, 1+len(msg.Media)) + if msg.Content != "" { + parts = append(parts, map[string]interface{}{ + "type": "text", + "text": msg.Content, + }) + } + for _, dataURL := range msg.Media { + parts = append(parts, map[string]interface{}{ + "type": "image_url", + "image_url": map[string]string{ + "url": dataURL, + }, + }) + } + m["content"] = parts + } else { + m["content"] = msg.Content + } + + if len(msg.ToolCalls) > 0 { + m["tool_calls"] = msg.ToolCalls + } + if msg.ToolCallID != "" { + m["tool_call_id"] = msg.ToolCallID + } + + result = append(result, m) + } + + return result +} + func (p *HTTPProvider) parseResponse(body []byte) (*LLMResponse, error) { var apiResponse struct { Choices []struct { diff --git a/pkg/providers/types.go b/pkg/providers/types.go index 88b62e975..4b681e55b 100644 --- a/pkg/providers/types.go +++ b/pkg/providers/types.go @@ -31,6 +31,7 @@ type UsageInfo struct { type Message struct { Role string `json:"role"` Content string `json:"content"` + Media []string `json:"media,omitempty"` ToolCalls []ToolCall `json:"tool_calls,omitempty"` ToolCallID string `json:"tool_call_id,omitempty"` } diff --git a/pkg/utils/media.go b/pkg/utils/media.go index 2b184f2ec..15afa4b24 100644 --- a/pkg/utils/media.go +++ b/pkg/utils/media.go @@ -1,6 +1,7 @@ package utils import ( + "encoding/base64" "io" "net/http" "os" @@ -12,6 +13,61 @@ import ( "github.com/sipeed/picoclaw/pkg/logger" ) +const maxImageFileSize = 50 * 1024 * 1024 // 50MB + +// EncodeFileToDataURL reads a local file and returns a base64 data URL string. +// Supported: JPEG, PNG, WEBP, GIF. Max 50MB. +// Returns empty string on error. +func EncodeFileToDataURL(path string) string { + ext := strings.ToLower(filepath.Ext(path)) + var mime string + switch ext { + case ".jpg", ".jpeg": + mime = "image/jpeg" + case ".png": + mime = "image/png" + case ".webp": + mime = "image/webp" + case ".gif": + mime = "image/gif" + default: + logger.WarnCF("media", "Unsupported image extension", map[string]interface{}{ + "path": path, + "ext": ext, + }) + return "" + } + + info, err := os.Stat(path) + if err != nil { + logger.ErrorCF("media", "Failed to stat image file", map[string]interface{}{ + "path": path, + "error": err.Error(), + }) + return "" + } + if info.Size() > maxImageFileSize { + logger.WarnCF("media", "Image file too large, skipping", map[string]interface{}{ + "path": path, + "size_mb": info.Size() / (1024 * 1024), + "max_mb": maxImageFileSize / (1024 * 1024), + }) + return "" + } + + data, err := os.ReadFile(path) + if err != nil { + logger.ErrorCF("media", "Failed to read image file", map[string]interface{}{ + "path": path, + "error": err.Error(), + }) + return "" + } + + encoded := base64.StdEncoding.EncodeToString(data) + return "data:" + mime + ";base64," + encoded +} + // IsAudioFile checks if a file is an audio file based on its filename extension and content type. func IsAudioFile(filename, contentType string) bool { audioExtensions := []string{".mp3", ".wav", ".ogg", ".m4a", ".flac", ".aac", ".wma"}