opencode/picoclaw
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: implement file/image support in chat channels (#61)

Browse source 
Implements file and image support for chat channels, following the
pattern from nanobot implementation. Images are downloaded to local
temp files, base64-encoded, and sent as multipart content to LLMs
with vision capabilities.

Changes:
- Modified Message.Content from string to interface{} to support
  multipart content (text + images)
- Added buildUserContent() method for base64 encoding images
- Updated all providers to handle multipart content
- Added IsImageFile() and improved GetMimeType() utilities
- Implemented image download in Discord, LINE, and Slack channels
- Fixed critical bug where media paths weren't reaching LLM
- Consolidated duplicate contentToString* functions
- Proper file cleanup after base64 encoding

Tested with Discord and Telegram channels and confirmed working.

Fixes #61

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
tttocklll 2026-02-15 17:20:40 +09:00
parent 9a3f3611c3
commit 77a4bbafe4
14 changed files with 324 additions and 116 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
go.mod
View file

@ -19,8 +19,6 @@ require (
golang.org/x/oauth2 v0.35.0
)
require (
github.com/andybalholm/brotli v1.2.0 // indirect
github.com/bytedance/gopkg v0.1.3 // indirect
@ -28,9 +26,9 @@ require (
github.com/bytedance/sonic/loader v0.5.0 // indirect
github.com/cloudwego/base64x v0.1.6 // indirect
github.com/github/copilot-sdk/go v0.1.23
github.com/google/jsonschema-go v0.4.2 // indirect
github.com/go-resty/resty/v2 v2.17.1 // indirect
github.com/go-resty/resty/v2 v2.17.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/google/jsonschema-go v0.4.2 // indirect
github.com/grbit/go-json v0.11.0 // indirect
github.com/klauspost/compress v1.18.4 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
@ -47,5 +45,4 @@ require (
golang.org/x/net v0.50.0 // indirect
golang.org/x/sync v0.19.0 // indirect
golang.org/x/sys v0.41.0 // indirect
)

6
go.sum
View file

@ -36,8 +36,8 @@ github.com/github/copilot-sdk/go v0.1.23 h1:uExtO/inZQndCZMiSAA1hvXINiz9tqo/MZgQ
github.com/github/copilot-sdk/go v0.1.23/go.mod h1:GdwwBfMbm9AABLEM3x5IZKw4ZfwCYxZ1BgyytmZenQ0=
github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w=
github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q=
github.com/go-resty/resty/v2 v2.17.1 h1:x3aMpHK1YM9e4va/TMDRlusDDoZiQ+ViDu/WpA6xTM4=
github.com/go-resty/resty/v2 v2.17.1/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA=
github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk=
github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
@ -58,6 +58,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8=
github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=

65
pkg/agent/context.go
View file

@ -1,6 +1,7 @@
package agent
import (
"encoding/base64"
"fmt"
"os"
"path/filepath"
@ -12,6 +13,7 @@ import (
"github.com/sipeed/picoclaw/pkg/providers"
"github.com/sipeed/picoclaw/pkg/skills"
"github.com/sipeed/picoclaw/pkg/tools"
"github.com/sipeed/picoclaw/pkg/utils"
)
type ContextBuilder struct {
@ -157,6 +159,65 @@ func (cb *ContextBuilder) LoadBootstrapFiles() string {
return result
}
// buildUserContent builds user message content with optional base64-encoded images.
// Returns either a string (text-only) or []interface{} (text + images).
func (cb *ContextBuilder) buildUserContent(text string, media []string) interface{} {
if len(media) == 0 {
return text
}
// Build multipart content with images
content := []interface{}{}
for _, mediaPath := range media {
mimeType := utils.GetMimeType(mediaPath)
// Only process image files
if !strings.HasPrefix(mimeType, "image/") {
continue
}
// Read and encode image
data, err := os.ReadFile(mediaPath)
if err != nil {
logger.ErrorCF("agent", "Failed to read media file",
map[string]interface{}{
"path": mediaPath,
"error": err.Error(),
})
continue
}
b64 := base64.StdEncoding.EncodeToString(data)
imageURL := fmt.Sprintf("data:%s;base64,%s", mimeType, b64)
content = append(content, map[string]interface{}{
"type": "image_url",
"image_url": map[string]interface{}{
"url": imageURL,
},
})
// Clean up the temporary file after encoding
os.Remove(mediaPath)
}
// Add text content
if text != "" {
content = append(content, map[string]interface{}{
"type": "text",
"text": text,
})
}
// If no images were successfully processed, return text only
if len(content) == 0 || (len(content) == 1 && text != "") {
return text
}
return content
}
func (cb *ContextBuilder) BuildMessages(history []providers.Message, summary string, currentMessage string, media []string, channel, chatID string) []providers.Message {
messages := []providers.Message{}
@ -207,9 +268,11 @@ func (cb *ContextBuilder) BuildMessages(history []providers.Message, summary str
messages = append(messages, history...)
// Build user content with optional media
userContent := cb.buildUserContent(currentMessage, media)
messages = append(messages, providers.Message{
Role: "user",
Content: currentMessage,
Content: userContent,
})
return messages

61
pkg/agent/loop.go
View file

@ -46,14 +46,15 @@ type AgentLoop struct {
// processOptions configures how a message is processed
type processOptions struct {
SessionKey string // Session identifier for history/context
Channel string // Target channel for tool execution
ChatID string // Target chat ID for tool execution
UserMessage string // User message content (may include prefix)
DefaultResponse string // Response when LLM returns empty
EnableSummary bool // Whether to trigger summarization
SendResponse bool // Whether to send response via bus
NoHistory bool // If true, don't load session history (for heartbeat)
SessionKey string // Session identifier for history/context
Channel string // Target channel for tool execution
ChatID string // Target chat ID for tool execution
UserMessage string // User message content (may include prefix)
Media []string // Media file paths (images, audio, etc.)
DefaultResponse string // Response when LLM returns empty
EnableSummary bool // Whether to trigger summarization
SendResponse bool // Whether to send response via bus
NoHistory bool // If true, don't load session history (for heartbeat)
}
// createToolRegistry creates a tool registry with common tools.
@ -256,6 +257,7 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage)
"chat_id": msg.ChatID,
"sender_id": msg.SenderID,
"session_key": msg.SessionKey,
"media_count": len(msg.Media),
})
// Route system messages to processSystemMessage
@ -269,6 +271,7 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage)
Channel: msg.Channel,
ChatID: msg.ChatID,
UserMessage: msg.Content,
Media: msg.Media,
DefaultResponse: "I've completed processing but have no response to give.",
EnableSummary: true,
SendResponse: false,
@ -355,7 +358,7 @@ func (al *AgentLoop) runAgentLoop(ctx context.Context, opts processOptions) (str
history,
summary,
opts.UserMessage,
nil,
opts.Media,
opts.Channel,
opts.ChatID,
)
@ -425,6 +428,14 @@ func (al *AgentLoop) runLLMIteration(ctx context.Context, messages []providers.M
// Build tool definitions
providerToolDefs := al.tools.ToProviderDefs()
// Calculate system prompt length
systemPromptLen := 0
if len(messages) > 0 {
if s, ok := messages[0].Content.(string); ok {
systemPromptLen = len(s)
}
}
// Log LLM request details
logger.DebugCF("agent", "LLM request",
map[string]interface{}{
@ -434,7 +445,7 @@ func (al *AgentLoop) runLLMIteration(ctx context.Context, messages []providers.M
"tools_count": len(providerToolDefs),
"max_tokens": 8192,
"temperature": 0.7,
"system_prompt_len": len(messages[0].Content),
"system_prompt_len": systemPromptLen,
})
// Log full messages (detailed)
@ -640,9 +651,15 @@ func formatMessagesForLog(messages []providers.Message) string {
}
}
}
if msg.Content != "" {
content := utils.Truncate(msg.Content, 200)
result += fmt.Sprintf(" Content: %s\n", content)
if msg.Content != nil {
contentStr := ""
if s, ok := msg.Content.(string); ok {
contentStr = s
}
if contentStr != "" {
content := utils.Truncate(contentStr, 200)
result += fmt.Sprintf(" Content: %s\n", content)
}
}
if msg.ToolCallID != "" {
result += fmt.Sprintf(" ToolCallID: %s\n", msg.ToolCallID)
@ -698,7 +715,11 @@ func (al *AgentLoop) summarizeSession(sessionKey string) {
continue
}
// Estimate tokens for this message
msgTokens := len(m.Content) / 4
contentLen := 0
if s, ok := m.Content.(string); ok {
contentLen = len(s)
}
msgTokens := contentLen / 4
if msgTokens > maxMessageTokens {
omitted = true
continue
@ -755,7 +776,11 @@ func (al *AgentLoop) summarizeBatch(ctx context.Context, batch []providers.Messa
}
prompt += "\nCONVERSATION:\n"
for _, m := range batch {
prompt += fmt.Sprintf("%s: %s\n", m.Role, m.Content)
contentStr := ""
if s, ok := m.Content.(string); ok {
contentStr = s
}
prompt += fmt.Sprintf("%s: %s\n", m.Role, contentStr)
}
response, err := al.provider.Chat(ctx, []providers.Message{{Role: "user", Content: prompt}}, nil, al.model, map[string]interface{}{
@ -775,7 +800,11 @@ func (al *AgentLoop) summarizeBatch(ctx context.Context, batch []providers.Messa
func (al *AgentLoop) estimateTokens(messages []providers.Message) int {
total := 0
for _, m := range messages {
total += utf8.RuneCountInString(m.Content) / 3
contentStr := ""
if s, ok := m.Content.(string); ok {
contentStr = s
}
total += utf8.RuneCountInString(contentStr) / 3
}
return total
}

38
pkg/channels/discord.go
View file

@ -3,7 +3,6 @@ package channels
import (
"context"
"fmt"
"os"
"time"
"github.com/bwmarrin/discordgo"
@ -156,27 +155,16 @@ func (c *DiscordChannel) handleMessage(s *discordgo.Session, m *discordgo.Messag
content := m.Content
mediaPaths := make([]string, 0, len(m.Attachments))
localFiles := make([]string, 0, len(m.Attachments))
// 确保临时文件在函数返回时被清理
defer func() {
for _, file := range localFiles {
if err := os.Remove(file); err != nil {
logger.DebugCF("discord", "Failed to cleanup temp file", map[string]any{
"file": file,
"error": err.Error(),
})
}
}
}()
// Note: Files will be cleaned up by context.buildUserContent after base64 encoding
for _, attachment := range m.Attachments {
isAudio := utils.IsAudioFile(attachment.Filename, attachment.ContentType)
isImage := utils.IsImageFile(attachment.Filename, attachment.ContentType)
if isAudio {
localPath := c.downloadAttachment(attachment.URL, attachment.Filename)
if localPath != "" {
localFiles = append(localFiles, localPath)
mediaPaths = append(mediaPaths, localPath)
transcribedText := ""
if c.transcriber != nil && c.transcriber.IsAvailable() {
@ -205,12 +193,24 @@ func (c *DiscordChannel) handleMessage(s *discordgo.Session, m *discordgo.Messag
"url": attachment.URL,
"filename": attachment.Filename,
})
mediaPaths = append(mediaPaths, attachment.URL)
content = appendContent(content, fmt.Sprintf("[attachment: %s]", attachment.URL))
content = appendContent(content, fmt.Sprintf("[attachment: %s (download failed)]", attachment.Filename))
}
} else if isImage {
// Download image for vision processing
localPath := c.downloadAttachment(attachment.URL, attachment.Filename)
if localPath != "" {
mediaPaths = append(mediaPaths, localPath)
content = appendContent(content, fmt.Sprintf("[image: %s]", attachment.Filename))
} else {
logger.WarnCF("discord", "Failed to download image attachment", map[string]any{
"url": attachment.URL,
"filename": attachment.Filename,
})
content = appendContent(content, fmt.Sprintf("[image: %s (download failed)]", attachment.Filename))
}
} else {
mediaPaths = append(mediaPaths, attachment.URL)
content = appendContent(content, fmt.Sprintf("[attachment: %s]", attachment.URL))
// Non-image, non-audio attachment (documents, etc.)
content = appendContent(content, fmt.Sprintf("[attachment: %s]", attachment.Filename))
}
}

17
pkg/channels/line.go
View file

@ -10,7 +10,6 @@ import (
"fmt"
"io"
"net/http"
"os"
"strings"
"sync"
"time"
@ -307,18 +306,7 @@ func (c *LINEChannel) processEvent(event lineEvent) {
var content string
var mediaPaths []string
localFiles := []string{}
defer func() {
for _, file := range localFiles {
if err := os.Remove(file); err != nil {
logger.DebugCF("line", "Failed to cleanup temp file", map[string]interface{}{
"file": file,
"error": err.Error(),
})
}
}
}()
// Note: Files will be cleaned up by context.buildUserContent after base64 encoding
switch msg.Type {
case "text":
@ -330,21 +318,18 @@ func (c *LINEChannel) processEvent(event lineEvent) {
case "image":
localPath := c.downloadContent(msg.ID, "image.jpg")
if localPath != "" {
localFiles = append(localFiles, localPath)
mediaPaths = append(mediaPaths, localPath)
content = "[image]"
}
case "audio":
localPath := c.downloadContent(msg.ID, "audio.m4a")
if localPath != "" {
localFiles = append(localFiles, localPath)
mediaPaths = append(mediaPaths, localPath)
content = "[audio]"
}
case "video":
localPath := c.downloadContent(msg.ID, "video.mp4")
if localPath != "" {
localFiles = append(localFiles, localPath)
mediaPaths = append(mediaPaths, localPath)
content = "[video]"
}

16
pkg/channels/slack.go
View file

@ -3,7 +3,6 @@ package channels
import (
"context"
"fmt"
"os"
"strings"
"sync"
"time"
@ -230,19 +229,7 @@ func (c *SlackChannel) handleMessageEvent(ev *slackevents.MessageEvent) {
content = c.stripBotMention(content)
var mediaPaths []string
localFiles := []string{} // 跟踪需要清理的本地文件
// 确保临时文件在函数返回时被清理
defer func() {
for _, file := range localFiles {
if err := os.Remove(file); err != nil {
logger.DebugCF("slack", "Failed to cleanup temp file", map[string]interface{}{
"file": file,
"error": err.Error(),
})
}
}
}()
// Note: Files will be cleaned up by context.buildUserContent after base64 encoding
if ev.Message != nil && len(ev.Message.Files) > 0 {
for _, file := range ev.Message.Files {
@ -250,7 +237,6 @@ func (c *SlackChannel) handleMessageEvent(ev *slackevents.MessageEvent) {
if localPath == "" {
continue
}
localFiles = append(localFiles, localPath)
mediaPaths = append(mediaPaths, localPath)
if utils.IsAudioFile(file.Name, file.Mimetype) && c.transcriber != nil && c.transcriber.IsAvailable() {

19
pkg/channels/telegram.go
View file

@ -5,7 +5,6 @@ import (
"fmt"
"net/http"
"net/url"
"os"
"regexp"
"strings"
"sync"
@ -197,19 +196,7 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, update telego.Updat
content := ""
mediaPaths := []string{}
localFiles := []string{} // 跟踪需要清理的本地文件
// 确保临时文件在函数返回时被清理
defer func() {
for _, file := range localFiles {
if err := os.Remove(file); err != nil {
logger.DebugCF("telegram", "Failed to cleanup temp file", map[string]interface{}{
"file": file,
"error": err.Error(),
})
}
}
}()
// Note: Files will be cleaned up by context.buildUserContent after base64 encoding
if message.Text != "" {
content += message.Text
@ -226,7 +213,6 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, update telego.Updat
photo := message.Photo[len(message.Photo)-1]
photoPath := c.downloadPhoto(ctx, photo.FileID)
if photoPath != "" {
localFiles = append(localFiles, photoPath)
mediaPaths = append(mediaPaths, photoPath)
if content != "" {
content += "\n"
@ -238,7 +224,6 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, update telego.Updat
if message.Voice != nil {
voicePath := c.downloadFile(ctx, message.Voice.FileID, ".ogg")
if voicePath != "" {
localFiles = append(localFiles, voicePath)
mediaPaths = append(mediaPaths, voicePath)
transcribedText := ""
@ -273,7 +258,6 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, update telego.Updat
if message.Audio != nil {
audioPath := c.downloadFile(ctx, message.Audio.FileID, ".mp3")
if audioPath != "" {
localFiles = append(localFiles, audioPath)
mediaPaths = append(mediaPaths, audioPath)
if content != "" {
content += "\n"
@ -285,7 +269,6 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, update telego.Updat
if message.Document != nil {
docPath := c.downloadFile(ctx, message.Document.FileID, "")
if docPath != "" {
localFiles = append(localFiles, docPath)
mediaPaths = append(mediaPaths, docPath)
if content != "" {
content += "\n"

12
pkg/providers/claude_cli_provider.go
View file

@ -71,11 +71,14 @@ func (p *ClaudeCliProvider) messagesToPrompt(messages []Message) string {
case "system":
// handled via --system-prompt flag
case "user":
parts = append(parts, "User: "+msg.Content)
userText := ContentToString(msg.Content)
parts = append(parts, "User: "+userText)
case "assistant":
parts = append(parts, "Assistant: "+msg.Content)
assistantText := ContentToString(msg.Content)
parts = append(parts, "Assistant: "+assistantText)
case "tool":
parts = append(parts, fmt.Sprintf("[Tool Result for %s]: %s", msg.ToolCallID, msg.Content))
toolText := ContentToString(msg.Content)
parts = append(parts, fmt.Sprintf("[Tool Result for %s]: %s", msg.ToolCallID, toolText))
}
}
@ -93,7 +96,8 @@ func (p *ClaudeCliProvider) buildSystemPrompt(messages []Message, tools []ToolDe
for _, msg := range messages {
if msg.Role == "system" {
parts = append(parts, msg.Content)
systemText := ContentToString(msg.Content)
parts = append(parts, systemText)
}
}

97
pkg/providers/claude_provider.go
View file

@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/anthropics/anthropic-sdk-go"
"github.com/anthropics/anthropic-sdk-go/option"
@ -56,6 +57,80 @@ func (p *ClaudeProvider) GetDefaultModel() string {
return "claude-sonnet-4-5-20250929"
}
// buildContentBlocks converts Content (interface{}) to Claude content blocks.
// Handles both string content and multipart content (text + images).
func buildContentBlocks(content interface{}) []anthropic.ContentBlockParamUnion {
if content == nil {
return []anthropic.ContentBlockParamUnion{anthropic.NewTextBlock("")}
}
// Try string content first
if s, ok := content.(string); ok {
return []anthropic.ContentBlockParamUnion{anthropic.NewTextBlock(s)}
}
// Try multipart content ([]interface{})
if parts, ok := content.([]interface{}); ok {
var blocks []anthropic.ContentBlockParamUnion
for _, part := range parts {
partMap, ok := part.(map[string]interface{})
if !ok {
continue
}
partType, _ := partMap["type"].(string)
switch partType {
case "text":
if text, ok := partMap["text"].(string); ok {
blocks = append(blocks, anthropic.NewTextBlock(text))
}
case "image_url":
if imageURL, ok := partMap["image_url"].(map[string]interface{}); ok {
if url, ok := imageURL["url"].(string); ok {
// Parse data URL: data:image/jpeg;base64,<data>
if strings.HasPrefix(url, "data:") {
parts := strings.SplitN(url, ",", 2)
if len(parts) == 2 {
// Extract media type from data URL
mediaType := anthropic.Base64ImageSourceMediaTypeImageJPEG // default
if strings.Contains(parts[0], ";") {
mediaTypePart := strings.Split(parts[0], ";")[0]
if strings.HasPrefix(mediaTypePart, "data:") {
mimeType := mediaTypePart[5:]
switch mimeType {
case "image/png":
mediaType = anthropic.Base64ImageSourceMediaTypeImagePNG
case "image/gif":
mediaType = anthropic.Base64ImageSourceMediaTypeImageGIF
case "image/webp":
mediaType = anthropic.Base64ImageSourceMediaTypeImageWebP
default:
mediaType = anthropic.Base64ImageSourceMediaTypeImageJPEG
}
}
}
imageSource := anthropic.Base64ImageSourceParam{
Data: parts[1],
MediaType: mediaType,
}
blocks = append(blocks, anthropic.NewImageBlock(imageSource))
}
}
}
}
}
}
if len(blocks) > 0 {
return blocks
}
}
// Fallback to empty text block
return []anthropic.ContentBlockParamUnion{anthropic.NewTextBlock("")}
}
func buildClaudeParams(messages []Message, tools []ToolDefinition, model string, options map[string]interface{}) (anthropic.MessageNewParams, error) {
var system []anthropic.TextBlockParam
var anthropicMessages []anthropic.MessageParam
@ -63,35 +138,43 @@ func buildClaudeParams(messages []Message, tools []ToolDefinition, model string,
for _, msg := range messages {
switch msg.Role {
case "system":
system = append(system, anthropic.TextBlockParam{Text: msg.Content})
systemText := ContentToString(msg.Content)
system = append(system, anthropic.TextBlockParam{Text: systemText})
case "user":
if msg.ToolCallID != "" {
// Tool result
resultText := ContentToString(msg.Content)
anthropicMessages = append(anthropicMessages,
anthropic.NewUserMessage(anthropic.NewToolResultBlock(msg.ToolCallID, msg.Content, false)),
anthropic.NewUserMessage(anthropic.NewToolResultBlock(msg.ToolCallID, resultText, false)),
)
} else {
// Regular user message (may include images)
blocks := buildContentBlocks(msg.Content)
anthropicMessages = append(anthropicMessages,
anthropic.NewUserMessage(anthropic.NewTextBlock(msg.Content)),
anthropic.NewUserMessage(blocks...),
)
}
case "assistant":
if len(msg.ToolCalls) > 0 {
var blocks []anthropic.ContentBlockParamUnion
if msg.Content != "" {
blocks = append(blocks, anthropic.NewTextBlock(msg.Content))
contentText := ContentToString(msg.Content)
if contentText != "" {
blocks = append(blocks, anthropic.NewTextBlock(contentText))
}
for _, tc := range msg.ToolCalls {
blocks = append(blocks, anthropic.NewToolUseBlock(tc.ID, tc.Arguments, tc.Name))
}
anthropicMessages = append(anthropicMessages, anthropic.NewAssistantMessage(blocks...))
} else {
contentText := ContentToString(msg.Content)
anthropicMessages = append(anthropicMessages,
anthropic.NewAssistantMessage(anthropic.NewTextBlock(msg.Content)),
anthropic.NewAssistantMessage(anthropic.NewTextBlock(contentText)),
)
}
case "tool":
resultText := ContentToString(msg.Content)
anthropicMessages = append(anthropicMessages,
anthropic.NewUserMessage(anthropic.NewToolResultBlock(msg.ToolCallID, msg.Content, false)),
anthropic.NewUserMessage(anthropic.NewToolResultBlock(msg.ToolCallID, resultText, false)),
)
}
}

19
pkg/providers/codex_provider.go
View file

@ -75,30 +75,33 @@ func buildCodexParams(messages []Message, tools []ToolDefinition, model string,
for _, msg := range messages {
switch msg.Role {
case "system":
instructions = msg.Content
instructions = ContentToString(msg.Content)
case "user":
if msg.ToolCallID != "" {
resultText := ContentToString(msg.Content)
inputItems = append(inputItems, responses.ResponseInputItemUnionParam{
OfFunctionCallOutput: &responses.ResponseInputItemFunctionCallOutputParam{
CallID: msg.ToolCallID,
Output: responses.ResponseInputItemFunctionCallOutputOutputUnionParam{OfString: openai.Opt(msg.Content)},
Output: responses.ResponseInputItemFunctionCallOutputOutputUnionParam{OfString: openai.Opt(resultText)},
},
})
} else {
userText := ContentToString(msg.Content)
inputItems = append(inputItems, responses.ResponseInputItemUnionParam{
OfMessage: &responses.EasyInputMessageParam{
Role: responses.EasyInputMessageRoleUser,
Content: responses.EasyInputMessageContentUnionParam{OfString: openai.Opt(msg.Content)},
Content: responses.EasyInputMessageContentUnionParam{OfString: openai.Opt(userText)},
},
})
}
case "assistant":
if len(msg.ToolCalls) > 0 {
if msg.Content != "" {
assistantText := ContentToString(msg.Content)
if assistantText != "" {
inputItems = append(inputItems, responses.ResponseInputItemUnionParam{
OfMessage: &responses.EasyInputMessageParam{
Role: responses.EasyInputMessageRoleAssistant,
Content: responses.EasyInputMessageContentUnionParam{OfString: openai.Opt(msg.Content)},
Content: responses.EasyInputMessageContentUnionParam{OfString: openai.Opt(assistantText)},
},
})
}
@ -113,18 +116,20 @@ func buildCodexParams(messages []Message, tools []ToolDefinition, model string,
})
}
} else {
assistantText := ContentToString(msg.Content)
inputItems = append(inputItems, responses.ResponseInputItemUnionParam{
OfMessage: &responses.EasyInputMessageParam{
Role: responses.EasyInputMessageRoleAssistant,
Content: responses.EasyInputMessageContentUnionParam{OfString: openai.Opt(msg.Content)},
Content: responses.EasyInputMessageContentUnionParam{OfString: openai.Opt(assistantText)},
},
})
}
case "tool":
resultText := ContentToString(msg.Content)
inputItems = append(inputItems, responses.ResponseInputItemUnionParam{
OfFunctionCallOutput: &responses.ResponseInputItemFunctionCallOutputParam{
CallID: msg.ToolCallID,
Output: responses.ResponseInputItemFunctionCallOutputOutputUnionParam{OfString: openai.Opt(msg.Content)},
Output: responses.ResponseInputItemFunctionCallOutputOutputUnionParam{OfString: openai.Opt(resultText)},
},
})
}

2
pkg/providers/github_copilot_provider.go
View file

@ -59,7 +59,7 @@ func (p *GitHubCopilotProvider) Chat(ctx context.Context, messages []Message, to
for _, msg := range messages {
out = append(out, tempMessage{
Role: msg.Role,
Content: msg.Content,
Content: ContentToString(msg.Content),
})
}

42
pkg/providers/types.go
View file

@ -1,6 +1,9 @@
package providers
import "context"
import (
"context"
"strings"
)
type ToolCall struct {
ID string `json:"id"`
@ -29,10 +32,10 @@ type UsageInfo struct {
}
type Message struct {
Role string `json:"role"`
Content string `json:"content"`
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
ToolCallID string `json:"tool_call_id,omitempty"`
Role string `json:"role"`
Content interface{} `json:"content"` // Can be string or []interface{} for multipart content (text + images)
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
ToolCallID string `json:"tool_call_id,omitempty"`
}
type LLMProvider interface {
@ -50,3 +53,32 @@ type ToolFunctionDefinition struct {
Description string `json:"description"`
Parameters map[string]interface{} `json:"parameters"`
}
// ContentToString extracts text from Content (interface{}).
// Handles both string content and multipart []interface{} content (text + images).
// For multipart content, extracts only the text parts and joins them with newlines.
func ContentToString(content interface{}) string {
if content == nil {
return ""
}
if s, ok := content.(string); ok {
return s
}
// If content is multipart ([]interface{}), extract text parts
if parts, ok := content.([]interface{}); ok {
var texts []string
for _, part := range parts {
partMap, ok := part.(map[string]interface{})
if !ok {
continue
}
if partType, _ := partMap["type"].(string); partType == "text" {
if text, ok := partMap["text"].(string); ok {
texts = append(texts, text)
}
}
}
return strings.Join(texts, "\n")
}
return ""
}

39
pkg/utils/media.go
View file

@ -2,6 +2,7 @@ package utils
import (
"io"
"mime"
"net/http"
"os"
"path/filepath"
@ -12,6 +13,44 @@ import (
"github.com/sipeed/picoclaw/pkg/logger"
)
// GetMimeType returns the MIME type for a file based on its extension.
// If the MIME type cannot be determined, returns "application/octet-stream".
func GetMimeType(path string) string {
ext := strings.ToLower(filepath.Ext(path))
if ext == "" {
return "application/octet-stream"
}
mimeType := mime.TypeByExtension(ext)
if mimeType == "" {
return "application/octet-stream"
}
// Remove charset parameter if present (e.g., "text/html; charset=utf-8" -> "text/html")
if idx := strings.Index(mimeType, ";"); idx > 0 {
mimeType = mimeType[:idx]
}
return mimeType
}
// IsImageFile checks if a file is an image file based on its filename extension and content type.
func IsImageFile(filename, contentType string) bool {
imageExtensions := []string{".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg"}
imageTypes := []string{"image/"}
for _, ext := range imageExtensions {
if strings.HasSuffix(strings.ToLower(filename), ext) {
return true
}
}
for _, imageType := range imageTypes {
if strings.HasPrefix(strings.ToLower(contentType), imageType) {
return true
}
}
return false
}
// IsAudioFile checks if a file is an audio file based on its filename extension and content type.
func IsAudioFile(filename, contentType string) bool {
audioExtensions := []string{".mp3", ".wav", ".ogg", ".m4a", ".flac", ".aac", ".wma"}