diff --git a/go.mod b/go.mod index 98aecd6ab..8b7b75bbd 100644 --- a/go.mod +++ b/go.mod @@ -19,8 +19,6 @@ require ( golang.org/x/oauth2 v0.35.0 ) - - require ( github.com/andybalholm/brotli v1.2.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect @@ -28,9 +26,9 @@ require ( github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/github/copilot-sdk/go v0.1.23 - github.com/google/jsonschema-go v0.4.2 // indirect - github.com/go-resty/resty/v2 v2.17.1 // indirect + github.com/go-resty/resty/v2 v2.17.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/jsonschema-go v0.4.2 // indirect github.com/grbit/go-json v0.11.0 // indirect github.com/klauspost/compress v1.18.4 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect @@ -47,5 +45,4 @@ require ( golang.org/x/net v0.50.0 // indirect golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.41.0 // indirect - ) diff --git a/go.sum b/go.sum index 6a565b93e..84a94df31 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/github/copilot-sdk/go v0.1.23 h1:uExtO/inZQndCZMiSAA1hvXINiz9tqo/MZgQ github.com/github/copilot-sdk/go v0.1.23/go.mod h1:GdwwBfMbm9AABLEM3x5IZKw4ZfwCYxZ1BgyytmZenQ0= github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q= -github.com/go-resty/resty/v2 v2.17.1 h1:x3aMpHK1YM9e4va/TMDRlusDDoZiQ+ViDu/WpA6xTM4= -github.com/go-resty/resty/v2 v2.17.1/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA= +github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk= +github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= @@ -58,6 +58,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= diff --git a/pkg/agent/context.go b/pkg/agent/context.go index cf5ce2913..3c35ae402 100644 --- a/pkg/agent/context.go +++ b/pkg/agent/context.go @@ -1,6 +1,7 @@ package agent import ( + "encoding/base64" "fmt" "os" "path/filepath" @@ -12,6 +13,7 @@ import ( "github.com/sipeed/picoclaw/pkg/providers" "github.com/sipeed/picoclaw/pkg/skills" "github.com/sipeed/picoclaw/pkg/tools" + "github.com/sipeed/picoclaw/pkg/utils" ) type ContextBuilder struct { @@ -157,6 +159,65 @@ func (cb *ContextBuilder) LoadBootstrapFiles() string { return result } +// buildUserContent builds user message content with optional base64-encoded images. +// Returns either a string (text-only) or []interface{} (text + images). +func (cb *ContextBuilder) buildUserContent(text string, media []string) interface{} { + if len(media) == 0 { + return text + } + + // Build multipart content with images + content := []interface{}{} + + for _, mediaPath := range media { + mimeType := utils.GetMimeType(mediaPath) + + // Only process image files + if !strings.HasPrefix(mimeType, "image/") { + continue + } + + // Read and encode image + data, err := os.ReadFile(mediaPath) + if err != nil { + logger.ErrorCF("agent", "Failed to read media file", + map[string]interface{}{ + "path": mediaPath, + "error": err.Error(), + }) + continue + } + + b64 := base64.StdEncoding.EncodeToString(data) + imageURL := fmt.Sprintf("data:%s;base64,%s", mimeType, b64) + + content = append(content, map[string]interface{}{ + "type": "image_url", + "image_url": map[string]interface{}{ + "url": imageURL, + }, + }) + + // Clean up the temporary file after encoding + os.Remove(mediaPath) + } + + // Add text content + if text != "" { + content = append(content, map[string]interface{}{ + "type": "text", + "text": text, + }) + } + + // If no images were successfully processed, return text only + if len(content) == 0 || (len(content) == 1 && text != "") { + return text + } + + return content +} + func (cb *ContextBuilder) BuildMessages(history []providers.Message, summary string, currentMessage string, media []string, channel, chatID string) []providers.Message { messages := []providers.Message{} @@ -207,9 +268,11 @@ func (cb *ContextBuilder) BuildMessages(history []providers.Message, summary str messages = append(messages, history...) + // Build user content with optional media + userContent := cb.buildUserContent(currentMessage, media) messages = append(messages, providers.Message{ Role: "user", - Content: currentMessage, + Content: userContent, }) return messages diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index f3dd94090..f25b568ea 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -46,14 +46,15 @@ type AgentLoop struct { // processOptions configures how a message is processed type processOptions struct { - SessionKey string // Session identifier for history/context - Channel string // Target channel for tool execution - ChatID string // Target chat ID for tool execution - UserMessage string // User message content (may include prefix) - DefaultResponse string // Response when LLM returns empty - EnableSummary bool // Whether to trigger summarization - SendResponse bool // Whether to send response via bus - NoHistory bool // If true, don't load session history (for heartbeat) + SessionKey string // Session identifier for history/context + Channel string // Target channel for tool execution + ChatID string // Target chat ID for tool execution + UserMessage string // User message content (may include prefix) + Media []string // Media file paths (images, audio, etc.) + DefaultResponse string // Response when LLM returns empty + EnableSummary bool // Whether to trigger summarization + SendResponse bool // Whether to send response via bus + NoHistory bool // If true, don't load session history (for heartbeat) } // createToolRegistry creates a tool registry with common tools. @@ -256,6 +257,7 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage) "chat_id": msg.ChatID, "sender_id": msg.SenderID, "session_key": msg.SessionKey, + "media_count": len(msg.Media), }) // Route system messages to processSystemMessage @@ -269,6 +271,7 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage) Channel: msg.Channel, ChatID: msg.ChatID, UserMessage: msg.Content, + Media: msg.Media, DefaultResponse: "I've completed processing but have no response to give.", EnableSummary: true, SendResponse: false, @@ -355,7 +358,7 @@ func (al *AgentLoop) runAgentLoop(ctx context.Context, opts processOptions) (str history, summary, opts.UserMessage, - nil, + opts.Media, opts.Channel, opts.ChatID, ) @@ -425,6 +428,14 @@ func (al *AgentLoop) runLLMIteration(ctx context.Context, messages []providers.M // Build tool definitions providerToolDefs := al.tools.ToProviderDefs() + // Calculate system prompt length + systemPromptLen := 0 + if len(messages) > 0 { + if s, ok := messages[0].Content.(string); ok { + systemPromptLen = len(s) + } + } + // Log LLM request details logger.DebugCF("agent", "LLM request", map[string]interface{}{ @@ -434,7 +445,7 @@ func (al *AgentLoop) runLLMIteration(ctx context.Context, messages []providers.M "tools_count": len(providerToolDefs), "max_tokens": 8192, "temperature": 0.7, - "system_prompt_len": len(messages[0].Content), + "system_prompt_len": systemPromptLen, }) // Log full messages (detailed) @@ -640,9 +651,15 @@ func formatMessagesForLog(messages []providers.Message) string { } } } - if msg.Content != "" { - content := utils.Truncate(msg.Content, 200) - result += fmt.Sprintf(" Content: %s\n", content) + if msg.Content != nil { + contentStr := "" + if s, ok := msg.Content.(string); ok { + contentStr = s + } + if contentStr != "" { + content := utils.Truncate(contentStr, 200) + result += fmt.Sprintf(" Content: %s\n", content) + } } if msg.ToolCallID != "" { result += fmt.Sprintf(" ToolCallID: %s\n", msg.ToolCallID) @@ -698,7 +715,11 @@ func (al *AgentLoop) summarizeSession(sessionKey string) { continue } // Estimate tokens for this message - msgTokens := len(m.Content) / 4 + contentLen := 0 + if s, ok := m.Content.(string); ok { + contentLen = len(s) + } + msgTokens := contentLen / 4 if msgTokens > maxMessageTokens { omitted = true continue @@ -755,7 +776,11 @@ func (al *AgentLoop) summarizeBatch(ctx context.Context, batch []providers.Messa } prompt += "\nCONVERSATION:\n" for _, m := range batch { - prompt += fmt.Sprintf("%s: %s\n", m.Role, m.Content) + contentStr := "" + if s, ok := m.Content.(string); ok { + contentStr = s + } + prompt += fmt.Sprintf("%s: %s\n", m.Role, contentStr) } response, err := al.provider.Chat(ctx, []providers.Message{{Role: "user", Content: prompt}}, nil, al.model, map[string]interface{}{ @@ -775,7 +800,11 @@ func (al *AgentLoop) summarizeBatch(ctx context.Context, batch []providers.Messa func (al *AgentLoop) estimateTokens(messages []providers.Message) int { total := 0 for _, m := range messages { - total += utf8.RuneCountInString(m.Content) / 3 + contentStr := "" + if s, ok := m.Content.(string); ok { + contentStr = s + } + total += utf8.RuneCountInString(contentStr) / 3 } return total } diff --git a/pkg/channels/discord.go b/pkg/channels/discord.go index e65c99eec..ecb0db4b4 100644 --- a/pkg/channels/discord.go +++ b/pkg/channels/discord.go @@ -3,7 +3,6 @@ package channels import ( "context" "fmt" - "os" "time" "github.com/bwmarrin/discordgo" @@ -156,27 +155,16 @@ func (c *DiscordChannel) handleMessage(s *discordgo.Session, m *discordgo.Messag content := m.Content mediaPaths := make([]string, 0, len(m.Attachments)) - localFiles := make([]string, 0, len(m.Attachments)) - - // 确保临时文件在函数返回时被清理 - defer func() { - for _, file := range localFiles { - if err := os.Remove(file); err != nil { - logger.DebugCF("discord", "Failed to cleanup temp file", map[string]any{ - "file": file, - "error": err.Error(), - }) - } - } - }() + // Note: Files will be cleaned up by context.buildUserContent after base64 encoding for _, attachment := range m.Attachments { isAudio := utils.IsAudioFile(attachment.Filename, attachment.ContentType) + isImage := utils.IsImageFile(attachment.Filename, attachment.ContentType) if isAudio { localPath := c.downloadAttachment(attachment.URL, attachment.Filename) if localPath != "" { - localFiles = append(localFiles, localPath) + mediaPaths = append(mediaPaths, localPath) transcribedText := "" if c.transcriber != nil && c.transcriber.IsAvailable() { @@ -205,12 +193,24 @@ func (c *DiscordChannel) handleMessage(s *discordgo.Session, m *discordgo.Messag "url": attachment.URL, "filename": attachment.Filename, }) - mediaPaths = append(mediaPaths, attachment.URL) - content = appendContent(content, fmt.Sprintf("[attachment: %s]", attachment.URL)) + content = appendContent(content, fmt.Sprintf("[attachment: %s (download failed)]", attachment.Filename)) + } + } else if isImage { + // Download image for vision processing + localPath := c.downloadAttachment(attachment.URL, attachment.Filename) + if localPath != "" { + mediaPaths = append(mediaPaths, localPath) + content = appendContent(content, fmt.Sprintf("[image: %s]", attachment.Filename)) + } else { + logger.WarnCF("discord", "Failed to download image attachment", map[string]any{ + "url": attachment.URL, + "filename": attachment.Filename, + }) + content = appendContent(content, fmt.Sprintf("[image: %s (download failed)]", attachment.Filename)) } } else { - mediaPaths = append(mediaPaths, attachment.URL) - content = appendContent(content, fmt.Sprintf("[attachment: %s]", attachment.URL)) + // Non-image, non-audio attachment (documents, etc.) + content = appendContent(content, fmt.Sprintf("[attachment: %s]", attachment.Filename)) } } diff --git a/pkg/channels/line.go b/pkg/channels/line.go index ffb5533e8..ca3c79b82 100644 --- a/pkg/channels/line.go +++ b/pkg/channels/line.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "net/http" - "os" "strings" "sync" "time" @@ -307,18 +306,7 @@ func (c *LINEChannel) processEvent(event lineEvent) { var content string var mediaPaths []string - localFiles := []string{} - - defer func() { - for _, file := range localFiles { - if err := os.Remove(file); err != nil { - logger.DebugCF("line", "Failed to cleanup temp file", map[string]interface{}{ - "file": file, - "error": err.Error(), - }) - } - } - }() + // Note: Files will be cleaned up by context.buildUserContent after base64 encoding switch msg.Type { case "text": @@ -330,21 +318,18 @@ func (c *LINEChannel) processEvent(event lineEvent) { case "image": localPath := c.downloadContent(msg.ID, "image.jpg") if localPath != "" { - localFiles = append(localFiles, localPath) mediaPaths = append(mediaPaths, localPath) content = "[image]" } case "audio": localPath := c.downloadContent(msg.ID, "audio.m4a") if localPath != "" { - localFiles = append(localFiles, localPath) mediaPaths = append(mediaPaths, localPath) content = "[audio]" } case "video": localPath := c.downloadContent(msg.ID, "video.mp4") if localPath != "" { - localFiles = append(localFiles, localPath) mediaPaths = append(mediaPaths, localPath) content = "[video]" } diff --git a/pkg/channels/slack.go b/pkg/channels/slack.go index d86d08a9d..8bc7365dc 100644 --- a/pkg/channels/slack.go +++ b/pkg/channels/slack.go @@ -3,7 +3,6 @@ package channels import ( "context" "fmt" - "os" "strings" "sync" "time" @@ -230,19 +229,7 @@ func (c *SlackChannel) handleMessageEvent(ev *slackevents.MessageEvent) { content = c.stripBotMention(content) var mediaPaths []string - localFiles := []string{} // 跟踪需要清理的本地文件 - - // 确保临时文件在函数返回时被清理 - defer func() { - for _, file := range localFiles { - if err := os.Remove(file); err != nil { - logger.DebugCF("slack", "Failed to cleanup temp file", map[string]interface{}{ - "file": file, - "error": err.Error(), - }) - } - } - }() + // Note: Files will be cleaned up by context.buildUserContent after base64 encoding if ev.Message != nil && len(ev.Message.Files) > 0 { for _, file := range ev.Message.Files { @@ -250,7 +237,6 @@ func (c *SlackChannel) handleMessageEvent(ev *slackevents.MessageEvent) { if localPath == "" { continue } - localFiles = append(localFiles, localPath) mediaPaths = append(mediaPaths, localPath) if utils.IsAudioFile(file.Name, file.Mimetype) && c.transcriber != nil && c.transcriber.IsAvailable() { diff --git a/pkg/channels/telegram.go b/pkg/channels/telegram.go index b14b1632e..394790b8a 100644 --- a/pkg/channels/telegram.go +++ b/pkg/channels/telegram.go @@ -5,7 +5,6 @@ import ( "fmt" "net/http" "net/url" - "os" "regexp" "strings" "sync" @@ -197,19 +196,7 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, update telego.Updat content := "" mediaPaths := []string{} - localFiles := []string{} // 跟踪需要清理的本地文件 - - // 确保临时文件在函数返回时被清理 - defer func() { - for _, file := range localFiles { - if err := os.Remove(file); err != nil { - logger.DebugCF("telegram", "Failed to cleanup temp file", map[string]interface{}{ - "file": file, - "error": err.Error(), - }) - } - } - }() + // Note: Files will be cleaned up by context.buildUserContent after base64 encoding if message.Text != "" { content += message.Text @@ -226,7 +213,6 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, update telego.Updat photo := message.Photo[len(message.Photo)-1] photoPath := c.downloadPhoto(ctx, photo.FileID) if photoPath != "" { - localFiles = append(localFiles, photoPath) mediaPaths = append(mediaPaths, photoPath) if content != "" { content += "\n" @@ -238,7 +224,6 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, update telego.Updat if message.Voice != nil { voicePath := c.downloadFile(ctx, message.Voice.FileID, ".ogg") if voicePath != "" { - localFiles = append(localFiles, voicePath) mediaPaths = append(mediaPaths, voicePath) transcribedText := "" @@ -273,7 +258,6 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, update telego.Updat if message.Audio != nil { audioPath := c.downloadFile(ctx, message.Audio.FileID, ".mp3") if audioPath != "" { - localFiles = append(localFiles, audioPath) mediaPaths = append(mediaPaths, audioPath) if content != "" { content += "\n" @@ -285,7 +269,6 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, update telego.Updat if message.Document != nil { docPath := c.downloadFile(ctx, message.Document.FileID, "") if docPath != "" { - localFiles = append(localFiles, docPath) mediaPaths = append(mediaPaths, docPath) if content != "" { content += "\n" diff --git a/pkg/providers/claude_cli_provider.go b/pkg/providers/claude_cli_provider.go index a91795715..a1bfef0f8 100644 --- a/pkg/providers/claude_cli_provider.go +++ b/pkg/providers/claude_cli_provider.go @@ -71,11 +71,14 @@ func (p *ClaudeCliProvider) messagesToPrompt(messages []Message) string { case "system": // handled via --system-prompt flag case "user": - parts = append(parts, "User: "+msg.Content) + userText := ContentToString(msg.Content) + parts = append(parts, "User: "+userText) case "assistant": - parts = append(parts, "Assistant: "+msg.Content) + assistantText := ContentToString(msg.Content) + parts = append(parts, "Assistant: "+assistantText) case "tool": - parts = append(parts, fmt.Sprintf("[Tool Result for %s]: %s", msg.ToolCallID, msg.Content)) + toolText := ContentToString(msg.Content) + parts = append(parts, fmt.Sprintf("[Tool Result for %s]: %s", msg.ToolCallID, toolText)) } } @@ -93,7 +96,8 @@ func (p *ClaudeCliProvider) buildSystemPrompt(messages []Message, tools []ToolDe for _, msg := range messages { if msg.Role == "system" { - parts = append(parts, msg.Content) + systemText := ContentToString(msg.Content) + parts = append(parts, systemText) } } diff --git a/pkg/providers/claude_provider.go b/pkg/providers/claude_provider.go index ae6aca96d..24fcbd5ed 100644 --- a/pkg/providers/claude_provider.go +++ b/pkg/providers/claude_provider.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "github.com/anthropics/anthropic-sdk-go" "github.com/anthropics/anthropic-sdk-go/option" @@ -56,6 +57,80 @@ func (p *ClaudeProvider) GetDefaultModel() string { return "claude-sonnet-4-5-20250929" } +// buildContentBlocks converts Content (interface{}) to Claude content blocks. +// Handles both string content and multipart content (text + images). +func buildContentBlocks(content interface{}) []anthropic.ContentBlockParamUnion { + if content == nil { + return []anthropic.ContentBlockParamUnion{anthropic.NewTextBlock("")} + } + + // Try string content first + if s, ok := content.(string); ok { + return []anthropic.ContentBlockParamUnion{anthropic.NewTextBlock(s)} + } + + // Try multipart content ([]interface{}) + if parts, ok := content.([]interface{}); ok { + var blocks []anthropic.ContentBlockParamUnion + + for _, part := range parts { + partMap, ok := part.(map[string]interface{}) + if !ok { + continue + } + + partType, _ := partMap["type"].(string) + switch partType { + case "text": + if text, ok := partMap["text"].(string); ok { + blocks = append(blocks, anthropic.NewTextBlock(text)) + } + case "image_url": + if imageURL, ok := partMap["image_url"].(map[string]interface{}); ok { + if url, ok := imageURL["url"].(string); ok { + // Parse data URL: data:image/jpeg;base64, + if strings.HasPrefix(url, "data:") { + parts := strings.SplitN(url, ",", 2) + if len(parts) == 2 { + // Extract media type from data URL + mediaType := anthropic.Base64ImageSourceMediaTypeImageJPEG // default + if strings.Contains(parts[0], ";") { + mediaTypePart := strings.Split(parts[0], ";")[0] + if strings.HasPrefix(mediaTypePart, "data:") { + mimeType := mediaTypePart[5:] + switch mimeType { + case "image/png": + mediaType = anthropic.Base64ImageSourceMediaTypeImagePNG + case "image/gif": + mediaType = anthropic.Base64ImageSourceMediaTypeImageGIF + case "image/webp": + mediaType = anthropic.Base64ImageSourceMediaTypeImageWebP + default: + mediaType = anthropic.Base64ImageSourceMediaTypeImageJPEG + } + } + } + imageSource := anthropic.Base64ImageSourceParam{ + Data: parts[1], + MediaType: mediaType, + } + blocks = append(blocks, anthropic.NewImageBlock(imageSource)) + } + } + } + } + } + } + + if len(blocks) > 0 { + return blocks + } + } + + // Fallback to empty text block + return []anthropic.ContentBlockParamUnion{anthropic.NewTextBlock("")} +} + func buildClaudeParams(messages []Message, tools []ToolDefinition, model string, options map[string]interface{}) (anthropic.MessageNewParams, error) { var system []anthropic.TextBlockParam var anthropicMessages []anthropic.MessageParam @@ -63,35 +138,43 @@ func buildClaudeParams(messages []Message, tools []ToolDefinition, model string, for _, msg := range messages { switch msg.Role { case "system": - system = append(system, anthropic.TextBlockParam{Text: msg.Content}) + systemText := ContentToString(msg.Content) + system = append(system, anthropic.TextBlockParam{Text: systemText}) case "user": if msg.ToolCallID != "" { + // Tool result + resultText := ContentToString(msg.Content) anthropicMessages = append(anthropicMessages, - anthropic.NewUserMessage(anthropic.NewToolResultBlock(msg.ToolCallID, msg.Content, false)), + anthropic.NewUserMessage(anthropic.NewToolResultBlock(msg.ToolCallID, resultText, false)), ) } else { + // Regular user message (may include images) + blocks := buildContentBlocks(msg.Content) anthropicMessages = append(anthropicMessages, - anthropic.NewUserMessage(anthropic.NewTextBlock(msg.Content)), + anthropic.NewUserMessage(blocks...), ) } case "assistant": if len(msg.ToolCalls) > 0 { var blocks []anthropic.ContentBlockParamUnion - if msg.Content != "" { - blocks = append(blocks, anthropic.NewTextBlock(msg.Content)) + contentText := ContentToString(msg.Content) + if contentText != "" { + blocks = append(blocks, anthropic.NewTextBlock(contentText)) } for _, tc := range msg.ToolCalls { blocks = append(blocks, anthropic.NewToolUseBlock(tc.ID, tc.Arguments, tc.Name)) } anthropicMessages = append(anthropicMessages, anthropic.NewAssistantMessage(blocks...)) } else { + contentText := ContentToString(msg.Content) anthropicMessages = append(anthropicMessages, - anthropic.NewAssistantMessage(anthropic.NewTextBlock(msg.Content)), + anthropic.NewAssistantMessage(anthropic.NewTextBlock(contentText)), ) } case "tool": + resultText := ContentToString(msg.Content) anthropicMessages = append(anthropicMessages, - anthropic.NewUserMessage(anthropic.NewToolResultBlock(msg.ToolCallID, msg.Content, false)), + anthropic.NewUserMessage(anthropic.NewToolResultBlock(msg.ToolCallID, resultText, false)), ) } } diff --git a/pkg/providers/codex_provider.go b/pkg/providers/codex_provider.go index c0b10bd5b..40f1a6311 100644 --- a/pkg/providers/codex_provider.go +++ b/pkg/providers/codex_provider.go @@ -75,30 +75,33 @@ func buildCodexParams(messages []Message, tools []ToolDefinition, model string, for _, msg := range messages { switch msg.Role { case "system": - instructions = msg.Content + instructions = ContentToString(msg.Content) case "user": if msg.ToolCallID != "" { + resultText := ContentToString(msg.Content) inputItems = append(inputItems, responses.ResponseInputItemUnionParam{ OfFunctionCallOutput: &responses.ResponseInputItemFunctionCallOutputParam{ CallID: msg.ToolCallID, - Output: responses.ResponseInputItemFunctionCallOutputOutputUnionParam{OfString: openai.Opt(msg.Content)}, + Output: responses.ResponseInputItemFunctionCallOutputOutputUnionParam{OfString: openai.Opt(resultText)}, }, }) } else { + userText := ContentToString(msg.Content) inputItems = append(inputItems, responses.ResponseInputItemUnionParam{ OfMessage: &responses.EasyInputMessageParam{ Role: responses.EasyInputMessageRoleUser, - Content: responses.EasyInputMessageContentUnionParam{OfString: openai.Opt(msg.Content)}, + Content: responses.EasyInputMessageContentUnionParam{OfString: openai.Opt(userText)}, }, }) } case "assistant": if len(msg.ToolCalls) > 0 { - if msg.Content != "" { + assistantText := ContentToString(msg.Content) + if assistantText != "" { inputItems = append(inputItems, responses.ResponseInputItemUnionParam{ OfMessage: &responses.EasyInputMessageParam{ Role: responses.EasyInputMessageRoleAssistant, - Content: responses.EasyInputMessageContentUnionParam{OfString: openai.Opt(msg.Content)}, + Content: responses.EasyInputMessageContentUnionParam{OfString: openai.Opt(assistantText)}, }, }) } @@ -113,18 +116,20 @@ func buildCodexParams(messages []Message, tools []ToolDefinition, model string, }) } } else { + assistantText := ContentToString(msg.Content) inputItems = append(inputItems, responses.ResponseInputItemUnionParam{ OfMessage: &responses.EasyInputMessageParam{ Role: responses.EasyInputMessageRoleAssistant, - Content: responses.EasyInputMessageContentUnionParam{OfString: openai.Opt(msg.Content)}, + Content: responses.EasyInputMessageContentUnionParam{OfString: openai.Opt(assistantText)}, }, }) } case "tool": + resultText := ContentToString(msg.Content) inputItems = append(inputItems, responses.ResponseInputItemUnionParam{ OfFunctionCallOutput: &responses.ResponseInputItemFunctionCallOutputParam{ CallID: msg.ToolCallID, - Output: responses.ResponseInputItemFunctionCallOutputOutputUnionParam{OfString: openai.Opt(msg.Content)}, + Output: responses.ResponseInputItemFunctionCallOutputOutputUnionParam{OfString: openai.Opt(resultText)}, }, }) } diff --git a/pkg/providers/github_copilot_provider.go b/pkg/providers/github_copilot_provider.go index 5058819f5..f19fc5127 100644 --- a/pkg/providers/github_copilot_provider.go +++ b/pkg/providers/github_copilot_provider.go @@ -59,7 +59,7 @@ func (p *GitHubCopilotProvider) Chat(ctx context.Context, messages []Message, to for _, msg := range messages { out = append(out, tempMessage{ Role: msg.Role, - Content: msg.Content, + Content: ContentToString(msg.Content), }) } diff --git a/pkg/providers/types.go b/pkg/providers/types.go index 88b62e975..4c22f9878 100644 --- a/pkg/providers/types.go +++ b/pkg/providers/types.go @@ -1,6 +1,9 @@ package providers -import "context" +import ( + "context" + "strings" +) type ToolCall struct { ID string `json:"id"` @@ -29,10 +32,10 @@ type UsageInfo struct { } type Message struct { - Role string `json:"role"` - Content string `json:"content"` - ToolCalls []ToolCall `json:"tool_calls,omitempty"` - ToolCallID string `json:"tool_call_id,omitempty"` + Role string `json:"role"` + Content interface{} `json:"content"` // Can be string or []interface{} for multipart content (text + images) + ToolCalls []ToolCall `json:"tool_calls,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` } type LLMProvider interface { @@ -50,3 +53,32 @@ type ToolFunctionDefinition struct { Description string `json:"description"` Parameters map[string]interface{} `json:"parameters"` } + +// ContentToString extracts text from Content (interface{}). +// Handles both string content and multipart []interface{} content (text + images). +// For multipart content, extracts only the text parts and joins them with newlines. +func ContentToString(content interface{}) string { + if content == nil { + return "" + } + if s, ok := content.(string); ok { + return s + } + // If content is multipart ([]interface{}), extract text parts + if parts, ok := content.([]interface{}); ok { + var texts []string + for _, part := range parts { + partMap, ok := part.(map[string]interface{}) + if !ok { + continue + } + if partType, _ := partMap["type"].(string); partType == "text" { + if text, ok := partMap["text"].(string); ok { + texts = append(texts, text) + } + } + } + return strings.Join(texts, "\n") + } + return "" +} diff --git a/pkg/utils/media.go b/pkg/utils/media.go index 6345da8fc..2d9aff7ae 100644 --- a/pkg/utils/media.go +++ b/pkg/utils/media.go @@ -2,6 +2,7 @@ package utils import ( "io" + "mime" "net/http" "os" "path/filepath" @@ -12,6 +13,44 @@ import ( "github.com/sipeed/picoclaw/pkg/logger" ) +// GetMimeType returns the MIME type for a file based on its extension. +// If the MIME type cannot be determined, returns "application/octet-stream". +func GetMimeType(path string) string { + ext := strings.ToLower(filepath.Ext(path)) + if ext == "" { + return "application/octet-stream" + } + mimeType := mime.TypeByExtension(ext) + if mimeType == "" { + return "application/octet-stream" + } + // Remove charset parameter if present (e.g., "text/html; charset=utf-8" -> "text/html") + if idx := strings.Index(mimeType, ";"); idx > 0 { + mimeType = mimeType[:idx] + } + return mimeType +} + +// IsImageFile checks if a file is an image file based on its filename extension and content type. +func IsImageFile(filename, contentType string) bool { + imageExtensions := []string{".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg"} + imageTypes := []string{"image/"} + + for _, ext := range imageExtensions { + if strings.HasSuffix(strings.ToLower(filename), ext) { + return true + } + } + + for _, imageType := range imageTypes { + if strings.HasPrefix(strings.ToLower(contentType), imageType) { + return true + } + } + + return false +} + // IsAudioFile checks if a file is an audio file based on its filename extension and content type. func IsAudioFile(filename, contentType string) bool { audioExtensions := []string{".mp3", ".wav", ".ogg", ".m4a", ".flac", ".aac", ".wma"}