opencode/picoclaw
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add production media pipeline for vision, file delivery, and screenshots

Browse source
This commit is contained in:
paulchan 2026-02-14 16:55:53 +08:00
parent 5872e0f55e
commit cb22dd2a0a
17 changed files with 1209 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
config/config.example.json
View file

@ -103,10 +103,20 @@
},
"tools": {
"web": {
"search": {
"brave": {
"enabled": false,
"api_key": "YOUR_BRAVE_API_KEY",
"max_results": 5
},
"duckduckgo": {
"enabled": true,
"max_results": 5
}
},
"media": {
"max_inbound_image_bytes": 5242880,
"max_inbound_images": 3,
"max_outbound_file_bytes": 10485760
}
},
"heartbeat": {

110
pkg/agent/context.go
View file

@ -1,7 +1,10 @@
package agent
import (
"encoding/base64"
"fmt"
"net/http"
"mime"
"os"
"path/filepath"
"runtime"
@ -14,11 +17,18 @@ import (
"github.com/sipeed/picoclaw/pkg/tools"
)
const (
maxVisionImagesPerMessage = 3
maxVisionImageBytes = 5 * 1024 * 1024
)
type ContextBuilder struct {
workspace string
skillsLoader *skills.SkillsLoader
memory *MemoryStore
tools *tools.ToolRegistry // Direct reference to tool registry
maxImages int
maxImageSize int64
}
func getGlobalConfigDir() string {
@ -40,6 +50,8 @@ func NewContextBuilder(workspace string) *ContextBuilder {
workspace: workspace,
skillsLoader: skills.NewSkillsLoader(workspace, globalSkillsDir, builtinSkillsDir),
memory: NewMemoryStore(workspace),
maxImages: maxVisionImagesPerMessage,
maxImageSize: maxVisionImageBytes,
}
}
@ -48,6 +60,15 @@ func (cb *ContextBuilder) SetToolsRegistry(registry *tools.ToolRegistry) {
cb.tools = registry
}
func (cb *ContextBuilder) SetMediaLimits(maxImages int, maxImageSize int64) {
if maxImages > 0 {
cb.maxImages = maxImages
}
if maxImageSize > 0 {
cb.maxImageSize = maxImageSize
}
}
func (cb *ContextBuilder) getIdentity() string {
now := time.Now().Format("2006-01-02 15:04 (Monday)")
workspacePath, _ := filepath.Abs(filepath.Join(cb.workspace))
@ -207,14 +228,99 @@ func (cb *ContextBuilder) BuildMessages(history []providers.Message, summary str
messages = append(messages, history...)
messages = append(messages, providers.Message{
userMsg := providers.Message{
Role: "user",
Content: currentMessage,
})
}
visionMedia, skipped := cb.buildVisionMedia(media)
if len(visionMedia) > 0 {
userMsg.Media = visionMedia
}
if len(skipped) > 0 {
notice := "\n\n[media-note]\n" + strings.Join(skipped, "\n")
userMsg.Content += notice
}
messages = append(messages, userMsg)
return messages
}
func (cb *ContextBuilder) buildVisionMedia(paths []string) ([]providers.MediaItem, []string) {
if len(paths) == 0 {
return nil, nil
}
items := make([]providers.MediaItem, 0, len(paths))
skipped := make([]string, 0)
for _, p := range paths {
if len(items) >= cb.maxImages {
skipped = append(skipped, fmt.Sprintf("- skipped %q: image limit reached (%d)", p, cb.maxImages))
continue
}
item, reason, ok := loadImageAsDataURL(p, cb.maxImageSize)
if !ok {
if reason != "" {
skipped = append(skipped, fmt.Sprintf("- skipped %q: %s", p, reason))
}
continue
}
items = append(items, item)
}
return items, skipped
}
func loadImageAsDataURL(path string, maxImageSize int64) (providers.MediaItem, string, bool) {
info, err := os.Stat(path)
if err != nil {
return providers.MediaItem{}, "file not found or unreadable", false
}
if info.IsDir() {
return providers.MediaItem{}, "is a directory", false
}
if maxImageSize > 0 && info.Size() > maxImageSize {
return providers.MediaItem{}, fmt.Sprintf("too large (%d bytes > %d bytes)", info.Size(), maxImageSize), false
}
content, err := os.ReadFile(path)
if err != nil {
return providers.MediaItem{}, "failed to read file", false
}
mimeType := detectImageMIME(path, content)
if !strings.HasPrefix(mimeType, "image/") {
return providers.MediaItem{}, "", false
}
encoded := base64.StdEncoding.EncodeToString(content)
return providers.MediaItem{
Type: "image_url",
URL: fmt.Sprintf("data:%s;base64,%s", mimeType, encoded),
MIMEType: mimeType,
SourceRef: path,
}, "", true
}
func detectImageMIME(path string, content []byte) string {
if ext := strings.ToLower(filepath.Ext(path)); ext != "" {
if mt := mime.TypeByExtension(ext); mt != "" {
if semi := strings.Index(mt, ";"); semi > 0 {
return mt[:semi]
}
return mt
}
}
detected := http.DetectContentType(content)
if semi := strings.Index(detected, ";"); semi > 0 {
return detected[:semi]
}
return detected
}
func (cb *ContextBuilder) AddToolResult(messages []providers.Message, toolCallID, toolName, result string) []providers.Message {
messages = append(messages, providers.Message{
Role: "tool",

33
pkg/agent/loop.go
View file

@ -49,6 +49,7 @@ type processOptions struct {
Channel string // Target channel for tool execution
ChatID string // Target chat ID for tool execution
UserMessage string // User message content (may include prefix)
Media []string
DefaultResponse string // Response when LLM returns empty
EnableSummary bool // Whether to trigger summarization
SendResponse bool // Whether to send response via bus
@ -59,6 +60,10 @@ type processOptions struct {
// This is shared between main agent and subagents.
func createToolRegistry(workspace string, restrict bool, cfg *config.Config, msgBus *bus.MessageBus) *tools.ToolRegistry {
registry := tools.NewToolRegistry()
maxOutboundFileBytes := cfg.Tools.Media.MaxOutboundFileBytes
if maxOutboundFileBytes <= 0 {
maxOutboundFileBytes = 10 * 1024 * 1024
}
// File system tools
registry.Register(tools.NewReadFileTool(workspace, restrict))
@ -94,6 +99,20 @@ func createToolRegistry(workspace string, restrict bool, cfg *config.Config, msg
})
registry.Register(messageTool)
sendFileTool := tools.NewSendFileTool(workspace, restrict, maxOutboundFileBytes)
sendFileTool.SetSendCallback(func(msg bus.OutboundMessage) error {
msgBus.PublishOutbound(msg)
return nil
})
registry.Register(sendFileTool)
screenshotTool := tools.NewScreenshotTool(workspace, restrict, maxOutboundFileBytes)
screenshotTool.SetSendCallback(func(msg bus.OutboundMessage) error {
msgBus.PublishOutbound(msg)
return nil
})
registry.Register(screenshotTool)
return registry
}
@ -128,6 +147,7 @@ func NewAgentLoop(cfg *config.Config, msgBus *bus.MessageBus, provider providers
// Create context builder and set tools registry
contextBuilder := NewContextBuilder(workspace)
contextBuilder.SetToolsRegistry(toolsRegistry)
contextBuilder.SetMediaLimits(cfg.Tools.Media.MaxInboundImages, cfg.Tools.Media.MaxInboundImageBytes)
return &AgentLoop{
bus: msgBus,
@ -264,6 +284,7 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage)
Channel: msg.Channel,
ChatID: msg.ChatID,
UserMessage: msg.Content,
Media: msg.Media,
DefaultResponse: "I've completed processing but have no response to give.",
EnableSummary: true,
SendResponse: false,
@ -350,7 +371,7 @@ func (al *AgentLoop) runAgentLoop(ctx context.Context, opts processOptions) (str
history,
summary,
opts.UserMessage,
nil,
opts.Media,
opts.Channel,
opts.ChatID,
)
@ -581,6 +602,16 @@ func (al *AgentLoop) updateToolContexts(channel, chatID string) {
st.SetContext(channel, chatID)
}
}
if tool, ok := al.tools.Get("send_file"); ok {
if ft, ok := tool.(tools.ContextualTool); ok {
ft.SetContext(channel, chatID)
}
}
if tool, ok := al.tools.Get("screenshot"); ok {
if st, ok := tool.(tools.ContextualTool); ok {
st.SetContext(channel, chatID)
}
}
}
// maybeSummarize triggers summarization if the session history exceeds thresholds.

9
pkg/bus/types.go
View file

@ -10,10 +10,19 @@ type InboundMessage struct {
Metadata map[string]string `json:"metadata,omitempty"`
}
type Attachment struct {
Type string `json:"type"` // image | file | audio | video
Path string `json:"path,omitempty"`
URL string `json:"url,omitempty"`
FileName string `json:"file_name,omitempty"`
MIMEType string `json:"mime_type,omitempty"`
}
type OutboundMessage struct {
Channel string `json:"channel"`
ChatID string `json:"chat_id"`
Content string `json:"content"`
Attachments []Attachment `json:"attachments,omitempty"`
}
type MessageHandler func(InboundMessage) error

15
pkg/channels/line.go
View file

@ -10,7 +10,6 @@ import (
"fmt"
"io"
"net/http"
"os"
"strings"
"sync"
"time"
@ -309,17 +308,6 @@ func (c *LINEChannel) processEvent(event lineEvent) {
var mediaPaths []string
localFiles := []string{}
defer func() {
for _, file := range localFiles {
if err := os.Remove(file); err != nil {
logger.DebugCF("line", "Failed to cleanup temp file", map[string]interface{}{
"file": file,
"error": err.Error(),
})
}
}
}()
switch msg.Type {
case "text":
content = msg.Text
@ -378,6 +366,9 @@ func (c *LINEChannel) processEvent(event lineEvent) {
c.sendLoading(senderID)
c.HandleMessage(senderID, chatID, content, mediaPaths, metadata)
for _, file := range localFiles {
utils.ScheduleFileCleanup(file, 15*time.Minute, "line")
}
}
// isBotMentioned checks if the bot is mentioned in the message.

16
pkg/channels/slack.go
View file

@ -3,7 +3,6 @@ package channels
import (
"context"
"fmt"
"os"
"strings"
"sync"
"time"
@ -232,18 +231,6 @@ func (c *SlackChannel) handleMessageEvent(ev *slackevents.MessageEvent) {
var mediaPaths []string
localFiles := []string{} // 跟踪需要清理的本地文件
// 确保临时文件在函数返回时被清理
defer func() {
for _, file := range localFiles {
if err := os.Remove(file); err != nil {
logger.DebugCF("slack", "Failed to cleanup temp file", map[string]interface{}{
"file": file,
"error": err.Error(),
})
}
}
}()
if ev.Message != nil && len(ev.Message.Files) > 0 {
for _, file := range ev.Message.Files {
localPath := c.downloadSlackFile(file)
@ -289,6 +276,9 @@ func (c *SlackChannel) handleMessageEvent(ev *slackevents.MessageEvent) {
})
c.HandleMessage(senderID, chatID, content, mediaPaths, metadata)
for _, file := range localFiles {
utils.ScheduleFileCleanup(file, 15*time.Minute, "slack")
}
}
func (c *SlackChannel) handleAppMention(ev *slackevents.AppMentionEvent) {

193
pkg/channels/telegram.go
View file

@ -1,11 +1,16 @@
package channels
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
@ -24,6 +29,7 @@ import (
type TelegramChannel struct {
*BaseChannel
bot *telego.Bot
httpClient *http.Client
config config.TelegramConfig
chatIDs map[string]int64
transcriber *voice.GroqTranscriber
@ -43,17 +49,19 @@ func (c *thinkingCancel) Cancel() {
func NewTelegramChannel(cfg config.TelegramConfig, bus *bus.MessageBus) (*TelegramChannel, error) {
var opts []telego.BotOption
httpClient := &http.Client{}
if cfg.Proxy != "" {
proxyURL, parseErr := url.Parse(cfg.Proxy)
if parseErr != nil {
return nil, fmt.Errorf("invalid proxy URL %q: %w", cfg.Proxy, parseErr)
}
opts = append(opts, telego.WithHTTPClient(&http.Client{
httpClient = &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyURL(proxyURL),
},
}))
}
opts = append(opts, telego.WithHTTPClient(httpClient))
}
bot, err := telego.NewBot(cfg.Token, opts...)
@ -66,6 +74,7 @@ func NewTelegramChannel(cfg config.TelegramConfig, bus *bus.MessageBus) (*Telegr
return &TelegramChannel{
BaseChannel: base,
bot: bot,
httpClient: httpClient,
config: cfg,
chatIDs: make(map[string]int64),
transcriber: nil,
@ -136,25 +145,43 @@ func (c *TelegramChannel) Send(ctx context.Context, msg bus.OutboundMessage) err
}
c.stopThinking.Delete(msg.ChatID)
}
placeholderEdited := false
if len(msg.Attachments) > 0 {
placeholderEdited = c.editPlaceholder(ctx, msg.ChatID, chatID, "Attachment sent.")
htmlContent := markdownToTelegramHTML(msg.Content)
caption, truncated := truncateTelegramCaption(strings.TrimSpace(msg.Content))
for i, attachment := range msg.Attachments {
cap := ""
if i == 0 {
cap = caption
}
if err := c.sendAttachment(ctx, chatID, attachment, cap); err != nil {
return err
}
}
// Try to edit placeholder
if pID, ok := c.placeholders.Load(msg.ChatID); ok {
c.placeholders.Delete(msg.ChatID)
editMsg := tu.EditMessageText(tu.ID(chatID), pID.(int), htmlContent)
editMsg.ParseMode = telego.ModeHTML
if _, err = c.bot.EditMessageText(ctx, editMsg); err == nil {
if truncated {
if err := c.sendTextMessage(ctx, msg.ChatID, chatID, msg.Content, placeholderEdited); err != nil {
return err
}
}
return nil
}
// Fallback to new message if edit fails
return c.sendTextMessage(ctx, msg.ChatID, chatID, msg.Content, false)
}
func (c *TelegramChannel) sendTextMessage(ctx context.Context, chatIDKey string, chatID int64, content string, skipPlaceholder bool) error {
htmlContent := markdownToTelegramHTML(content)
if !skipPlaceholder && c.editPlaceholder(ctx, chatIDKey, chatID, htmlContent) {
return nil
}
tgMsg := tu.Message(tu.ID(chatID), htmlContent)
tgMsg.ParseMode = telego.ModeHTML
if _, err = c.bot.SendMessage(ctx, tgMsg); err != nil {
if _, err := c.bot.SendMessage(ctx, tgMsg); err != nil {
logger.ErrorCF("telegram", "HTML parse failed, falling back to plain text", map[string]interface{}{
"error": err.Error(),
})
@ -166,6 +193,129 @@ func (c *TelegramChannel) Send(ctx context.Context, msg bus.OutboundMessage) err
return nil
}
func (c *TelegramChannel) editPlaceholder(ctx context.Context, chatIDKey string, chatID int64, htmlContent string) bool {
if pID, ok := c.placeholders.Load(chatIDKey); ok {
c.placeholders.Delete(chatIDKey)
editMsg := tu.EditMessageText(tu.ID(chatID), pID.(int), htmlContent)
editMsg.ParseMode = telego.ModeHTML
if _, err := c.bot.EditMessageText(ctx, editMsg); err == nil {
return true
}
}
return false
}
func truncateTelegramCaption(content string) (string, bool) {
if content == "" {
return "", false
}
const maxCaptionLen = 1024
if len(content) <= maxCaptionLen {
return content, false
}
return content[:maxCaptionLen], true
}
func (c *TelegramChannel) sendAttachment(ctx context.Context, chatID int64, attachment bus.Attachment, caption string) error {
path := strings.TrimSpace(attachment.Path)
if path == "" {
return fmt.Errorf("telegram attachment requires local file path")
}
file, err := os.Open(path)
if err != nil {
return fmt.Errorf("open attachment %q: %w", path, err)
}
defer file.Close()
var endpoint string
var formField string
if isTelegramPhoto(attachment, path) {
endpoint = "sendPhoto"
formField = "photo"
} else {
endpoint = "sendDocument"
formField = "document"
}
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
if err := writer.WriteField("chat_id", fmt.Sprintf("%d", chatID)); err != nil {
return fmt.Errorf("write chat_id field: %w", err)
}
if caption != "" {
if err := writer.WriteField("caption", caption); err != nil {
return fmt.Errorf("write caption field: %w", err)
}
}
filename := attachment.FileName
if filename == "" {
filename = filepath.Base(path)
}
part, err := writer.CreateFormFile(formField, filename)
if err != nil {
return fmt.Errorf("create multipart form file: %w", err)
}
if _, err := io.Copy(part, file); err != nil {
return fmt.Errorf("copy file content: %w", err)
}
if err := writer.Close(); err != nil {
return fmt.Errorf("close multipart body: %w", err)
}
apiURL := fmt.Sprintf("https://api.telegram.org/bot%s/%s", c.config.Token, endpoint)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL, body)
if err != nil {
return fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", writer.FormDataContentType())
resp, err := c.httpClient.Do(req)
if err != nil {
return fmt.Errorf("send telegram attachment: %w", err)
}
defer resp.Body.Close()
respBody, _ := io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("telegram attachment API failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(respBody)))
}
var apiResp struct {
OK bool `json:"ok"`
Description string `json:"description"`
Result json.RawMessage `json:"result"`
}
if err := json.Unmarshal(respBody, &apiResp); err != nil {
return fmt.Errorf("parse telegram attachment response: %w", err)
}
if !apiResp.OK {
return fmt.Errorf("telegram attachment API error: %s", apiResp.Description)
}
return nil
}
func isTelegramPhoto(attachment bus.Attachment, path string) bool {
if strings.EqualFold(attachment.Type, "image") {
return true
}
if strings.HasPrefix(strings.ToLower(attachment.MIMEType), "image/") {
return true
}
switch strings.ToLower(filepath.Ext(path)) {
case ".jpg", ".jpeg", ".png", ".gif", ".webp":
return true
default:
return false
}
}
func (c *TelegramChannel) handleMessage(ctx context.Context, update telego.Update) {
message := update.Message
if message == nil {
@ -197,19 +347,7 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, update telego.Updat
content := ""
mediaPaths := []string{}
localFiles := []string{} // 跟踪需要清理的本地文件
// 确保临时文件在函数返回时被清理
defer func() {
for _, file := range localFiles {
if err := os.Remove(file); err != nil {
logger.DebugCF("telegram", "Failed to cleanup temp file", map[string]interface{}{
"file": file,
"error": err.Error(),
})
}
}
}()
localFiles := []string{}
if message.Text != "" {
content += message.Text
@ -362,6 +500,11 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, update telego.Updat
}
c.HandleMessage(senderID, fmt.Sprintf("%d", chatID), content, mediaPaths, metadata)
for _, file := range localFiles {
// Delay cleanup so downstream processing (vision, file tools) can access the file.
utils.ScheduleFileCleanup(file, 15*time.Minute, "telegram")
}
}
func (c *TelegramChannel) downloadPhoto(ctx context.Context, fileID string) string {

12
pkg/config/config.go
View file

@ -191,8 +191,15 @@ type WebToolsConfig struct {
DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"`
}
type MediaToolsConfig struct {
MaxInboundImageBytes int64 `json:"max_inbound_image_bytes" env:"PICOCLAW_TOOLS_MEDIA_MAX_INBOUND_IMAGE_BYTES"`
MaxInboundImages int `json:"max_inbound_images" env:"PICOCLAW_TOOLS_MEDIA_MAX_INBOUND_IMAGES"`
MaxOutboundFileBytes int64 `json:"max_outbound_file_bytes" env:"PICOCLAW_TOOLS_MEDIA_MAX_OUTBOUND_FILE_BYTES"`
}
type ToolsConfig struct {
Web WebToolsConfig `json:"web"`
Media MediaToolsConfig `json:"media"`
}
func DefaultConfig() *Config {
@ -294,6 +301,11 @@ func DefaultConfig() *Config {
MaxResults: 5,
},
},
Media: MediaToolsConfig{
MaxInboundImageBytes: 5 * 1024 * 1024,
MaxInboundImages: 3,
MaxOutboundFileBytes: 10 * 1024 * 1024,
},
},
Heartbeat: HeartbeatConfig{
Enabled: true,

10
pkg/config/config_test.go
View file

@ -145,6 +145,16 @@ func TestDefaultConfig_WebTools(t *testing.T) {
if cfg.Tools.Web.DuckDuckGo.MaxResults != 5 {
t.Error("Expected DuckDuckGo MaxResults 5, got ", cfg.Tools.Web.DuckDuckGo.MaxResults)
}
if cfg.Tools.Media.MaxInboundImageBytes <= 0 {
t.Error("Expected positive MaxInboundImageBytes")
}
if cfg.Tools.Media.MaxInboundImages <= 0 {
t.Error("Expected positive MaxInboundImages")
}
if cfg.Tools.Media.MaxOutboundFileBytes <= 0 {
t.Error("Expected positive MaxOutboundFileBytes")
}
}
// TestConfig_Complete verifies all config fields are set

112
pkg/providers/http_provider.go
View file

@ -62,7 +62,7 @@ func (p *HTTPProvider) Chat(ctx context.Context, messages []Message, tools []Too
requestBody := map[string]interface{}{
"model": model,
"messages": messages,
"messages": buildOpenAICompatMessages(messages),
}
if len(tools) > 0 {
@ -126,7 +126,7 @@ func (p *HTTPProvider) parseResponse(body []byte) (*LLMResponse, error) {
var apiResponse struct {
Choices []struct {
Message struct {
Content string `json:"content"`
Content json.RawMessage `json:"content"`
ToolCalls []struct {
ID string `json:"id"`
Type string `json:"type"`
@ -153,6 +153,7 @@ func (p *HTTPProvider) parseResponse(body []byte) (*LLMResponse, error) {
}
choice := apiResponse.Choices[0]
content := parseOpenAIContent(choice.Message.Content)
toolCalls := make([]ToolCall, 0, len(choice.Message.ToolCalls))
for _, tc := range choice.Message.ToolCalls {
@ -185,13 +186,118 @@ func (p *HTTPProvider) parseResponse(body []byte) (*LLMResponse, error) {
}
return &LLMResponse{
Content: choice.Message.Content,
Content: content,
ToolCalls: toolCalls,
FinishReason: choice.FinishReason,
Usage: apiResponse.Usage,
}, nil
}
func buildOpenAICompatMessages(messages []Message) []map[string]interface{} {
out := make([]map[string]interface{}, 0, len(messages))
for _, msg := range messages {
out = append(out, buildOpenAICompatMessage(msg))
}
return out
}
func buildOpenAICompatMessage(msg Message) map[string]interface{} {
result := map[string]interface{}{
"role": msg.Role,
}
switch msg.Role {
case "user":
contentParts := make([]map[string]interface{}, 0, 1+len(msg.Media))
if strings.TrimSpace(msg.Content) != "" {
contentParts = append(contentParts, map[string]interface{}{
"type": "text",
"text": msg.Content,
})
}
for _, media := range msg.Media {
if media.Type != "image_url" || media.URL == "" {
continue
}
contentParts = append(contentParts, map[string]interface{}{
"type": "image_url",
"image_url": map[string]interface{}{
"url": media.URL,
},
})
}
if len(contentParts) == 0 {
result["content"] = msg.Content
} else {
result["content"] = contentParts
}
case "assistant":
result["content"] = msg.Content
if len(msg.ToolCalls) > 0 {
toolCalls := make([]map[string]interface{}, 0, len(msg.ToolCalls))
for _, tc := range msg.ToolCalls {
name := tc.Name
args := ""
if tc.Function != nil {
if name == "" {
name = tc.Function.Name
}
args = tc.Function.Arguments
}
if args == "" && len(tc.Arguments) > 0 {
argsJSON, _ := json.Marshal(tc.Arguments)
args = string(argsJSON)
}
toolCalls = append(toolCalls, map[string]interface{}{
"id": tc.ID,
"type": "function",
"function": map[string]interface{}{
"name": name,
"arguments": args,
},
})
}
result["tool_calls"] = toolCalls
}
case "tool":
result["content"] = msg.Content
if msg.ToolCallID != "" {
result["tool_call_id"] = msg.ToolCallID
}
default:
result["content"] = msg.Content
}
return result
}
func parseOpenAIContent(raw json.RawMessage) string {
if len(raw) == 0 {
return ""
}
var text string
if err := json.Unmarshal(raw, &text); err == nil {
return text
}
var parts []struct {
Type string `json:"type"`
Text string `json:"text"`
}
if err := json.Unmarshal(raw, &parts); err != nil {
return ""
}
lines := make([]string, 0, len(parts))
for _, p := range parts {
if p.Type == "text" && p.Text != "" {
lines = append(lines, p.Text)
}
}
return strings.Join(lines, "\n")
}
func (p *HTTPProvider) GetDefaultModel() string {
return ""
}

54
pkg/providers/http_provider_test.go Normal file
View file

@ -0,0 +1,54 @@
package providers
import (
"encoding/json"
"strings"
"testing"
)
func TestBuildOpenAICompatMessages_WithImageMedia(t *testing.T) {
messages := []Message{
{
Role: "user",
Content: "Describe this image",
Media: []MediaItem{
{Type: "image_url", URL: "data:image/png;base64,abc"},
},
},
}
out := buildOpenAICompatMessages(messages)
if len(out) != 1 {
t.Fatalf("expected 1 message, got %d", len(out))
}
content, ok := out[0]["content"].([]map[string]interface{})
if !ok {
t.Fatalf("expected content parts array")
}
if len(content) != 2 {
t.Fatalf("expected 2 content parts, got %d", len(content))
}
if content[1]["type"] != "image_url" {
t.Fatalf("expected image_url part")
}
}
func TestParseOpenAIContent_TextArray(t *testing.T) {
raw := json.RawMessage(`[
{"type":"text","text":"line1"},
{"type":"text","text":"line2"}
]`)
got := parseOpenAIContent(raw)
if !strings.Contains(got, "line1") || !strings.Contains(got, "line2") {
t.Fatalf("unexpected parsed content: %q", got)
}
}
func TestParseOpenAIContent_String(t *testing.T) {
raw := json.RawMessage(`"hello"`)
got := parseOpenAIContent(raw)
if got != "hello" {
t.Fatalf("expected hello, got %q", got)
}
}

8
pkg/providers/types.go
View file

@ -31,10 +31,18 @@ type UsageInfo struct {
type Message struct {
Role string `json:"role"`
Content string `json:"content"`
Media []MediaItem `json:"media,omitempty"`
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
ToolCallID string `json:"tool_call_id,omitempty"`
}
type MediaItem struct {
Type string `json:"type"` // currently supports "image_url"
URL string `json:"url,omitempty"`
MIMEType string `json:"mime_type,omitempty"`
SourceRef string `json:"source_ref,omitempty"`
}
type LLMProvider interface {
Chat(ctx context.Context, messages []Message, tools []ToolDefinition, model string, options map[string]interface{}) (*LLMResponse, error)
GetDefaultModel() string

294
pkg/tools/screenshot.go Normal file
View file

@ -0,0 +1,294 @@
package tools
import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/sipeed/picoclaw/pkg/bus"
"github.com/sipeed/picoclaw/pkg/utils"
)
type ScreenshotTool struct {
workspace string
restrict bool
maxFileBytes int64
timeout time.Duration
sendCallback SendFileCallback
defaultChannel string
defaultChatID string
lookPath func(string) (string, error)
runCommand func(ctx context.Context, command string, args ...string) error
now func() time.Time
}
func NewScreenshotTool(workspace string, restrict bool, maxFileBytes int64) *ScreenshotTool {
return &ScreenshotTool{
workspace: workspace,
restrict: restrict,
maxFileBytes: maxFileBytes,
timeout: 45 * time.Second,
lookPath: exec.LookPath,
runCommand: func(ctx context.Context, command string, args ...string) error {
cmd := exec.CommandContext(ctx, command, args...)
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("%w: %s", err, strings.TrimSpace(string(output)))
}
return nil
},
now: time.Now,
}
}
func (t *ScreenshotTool) Name() string {
return "screenshot"
}
func (t *ScreenshotTool) Description() string {
return "Capture a screenshot (web URL or desktop) and optionally send it to user."
}
func (t *ScreenshotTool) Parameters() map[string]interface{} {
return map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"url": map[string]interface{}{
"type": "string",
"description": "Web page URL to capture. If omitted, captures local desktop screen.",
},
"path": map[string]interface{}{
"type": "string",
"description": "Optional output image path. Defaults to workspace/tmp/screenshots/*.png",
},
"caption": map[string]interface{}{
"type": "string",
"description": "Optional caption when sending screenshot.",
},
"send": map[string]interface{}{
"type": "boolean",
"description": "Whether to send the screenshot to chat. Default true.",
},
"keep_local": map[string]interface{}{
"type": "boolean",
"description": "Whether to keep local screenshot file after sending. Default false.",
},
"width": map[string]interface{}{
"type": "integer",
"description": "Browser viewport width for URL screenshot. Default 1366.",
},
"height": map[string]interface{}{
"type": "integer",
"description": "Browser viewport height for URL screenshot. Default 768.",
},
"channel": map[string]interface{}{
"type": "string",
"description": "Optional target channel override.",
},
"chat_id": map[string]interface{}{
"type": "string",
"description": "Optional target chat id override.",
},
},
}
}
func (t *ScreenshotTool) SetContext(channel, chatID string) {
t.defaultChannel = channel
t.defaultChatID = chatID
}
func (t *ScreenshotTool) SetSendCallback(callback SendFileCallback) {
t.sendCallback = callback
}
func (t *ScreenshotTool) Execute(ctx context.Context, args map[string]interface{}) *ToolResult {
outputPath, err := t.resolveOutputPath(args)
if err != nil {
return ErrorResult(err.Error())
}
width := intFromArgs(args, "width", 1366)
height := intFromArgs(args, "height", 768)
targetURL, _ := args["url"].(string)
cmdCtx, cancel := context.WithTimeout(ctx, t.timeout)
defer cancel()
if strings.TrimSpace(targetURL) != "" {
if err := t.captureURL(cmdCtx, targetURL, outputPath, width, height); err != nil {
return ErrorResult(fmt.Sprintf("capture URL screenshot failed: %v", err))
}
} else {
if err := t.captureDesktop(cmdCtx, outputPath); err != nil {
return ErrorResult(fmt.Sprintf("capture desktop screenshot failed: %v", err))
}
}
info, err := os.Stat(outputPath)
if err != nil {
return ErrorResult(fmt.Sprintf("screenshot not created: %v", err))
}
if info.Size() == 0 {
return ErrorResult("screenshot is empty")
}
if t.maxFileBytes > 0 && info.Size() > t.maxFileBytes {
return ErrorResult(fmt.Sprintf("screenshot too large: %d bytes (limit %d bytes)", info.Size(), t.maxFileBytes))
}
send := boolFromArgs(args, "send", true)
keepLocal := boolFromArgs(args, "keep_local", false)
caption, _ := args["caption"].(string)
if !send {
return UserResult(fmt.Sprintf("Screenshot saved: %s", outputPath))
}
channel, _ := args["channel"].(string)
chatID, _ := args["chat_id"].(string)
if channel == "" {
channel = t.defaultChannel
}
if chatID == "" {
chatID = t.defaultChatID
}
if channel == "" || chatID == "" {
return ErrorResult("No target channel/chat specified")
}
if t.sendCallback == nil {
return ErrorResult("Screenshot sending not configured")
}
if err := t.sendCallback(bus.OutboundMessage{
Channel: channel,
ChatID: chatID,
Content: caption,
Attachments: []bus.Attachment{
{
Type: "image",
Path: outputPath,
FileName: filepath.Base(outputPath),
MIMEType: "image/png",
},
},
}); err != nil {
return ErrorResult(fmt.Sprintf("sending screenshot: %v", err))
}
if !keepLocal {
utils.ScheduleFileCleanup(outputPath, 15*time.Minute, "screenshot")
}
return SilentResult(fmt.Sprintf("Screenshot sent to %s:%s", channel, chatID))
}
func (t *ScreenshotTool) resolveOutputPath(args map[string]interface{}) (string, error) {
rawPath, _ := args["path"].(string)
if strings.TrimSpace(rawPath) == "" {
rawPath = filepath.Join("tmp", "screenshots", t.now().Format("20060102_150405")+".png")
}
resolvedPath, err := validatePath(rawPath, t.workspace, t.restrict)
if err != nil {
return "", err
}
if ext := strings.ToLower(filepath.Ext(resolvedPath)); ext == "" {
resolvedPath += ".png"
}
if err := os.MkdirAll(filepath.Dir(resolvedPath), 0755); err != nil {
return "", fmt.Errorf("failed to create screenshot directory: %w", err)
}
return resolvedPath, nil
}
func (t *ScreenshotTool) captureURL(ctx context.Context, targetURL, outputPath string, width, height int) error {
browsers := []string{"chromium-browser", "chromium", "google-chrome", "google-chrome-stable"}
var browser string
for _, candidate := range browsers {
if _, err := t.lookPath(candidate); err == nil {
browser = candidate
break
}
}
if browser == "" {
return fmt.Errorf("no headless browser found (tried: %s)", strings.Join(browsers, ", "))
}
args := []string{
"--headless",
"--disable-gpu",
"--hide-scrollbars",
"--no-sandbox",
fmt.Sprintf("--window-size=%d,%d", width, height),
fmt.Sprintf("--screenshot=%s", outputPath),
targetURL,
}
return t.runCommand(ctx, browser, args...)
}
func (t *ScreenshotTool) captureDesktop(ctx context.Context, outputPath string) error {
type command struct {
name string
args []string
}
candidates := []command{
{name: "scrot", args: []string{outputPath}},
{name: "grim", args: []string{outputPath}},
{name: "import", args: []string{"-window", "root", outputPath}},
}
var available []command
for _, candidate := range candidates {
if _, err := t.lookPath(candidate.name); err == nil {
available = append(available, candidate)
}
}
if len(available) == 0 {
return fmt.Errorf("no desktop screenshot command found (tried: scrot, grim, import)")
}
var lastErr error
for _, cmd := range available {
if err := t.runCommand(ctx, cmd.name, cmd.args...); err == nil {
return nil
} else {
lastErr = err
}
}
return lastErr
}
func intFromArgs(args map[string]interface{}, key string, fallback int) int {
v, ok := args[key]
if !ok {
return fallback
}
switch n := v.(type) {
case int:
if n > 0 {
return n
}
case float64:
if int(n) > 0 {
return int(n)
}
}
return fallback
}
func boolFromArgs(args map[string]interface{}, key string, fallback bool) bool {
v, ok := args[key]
if !ok {
return fallback
}
if b, ok := v.(bool); ok {
return b
}
return fallback
}

104
pkg/tools/screenshot_test.go Normal file
View file

@ -0,0 +1,104 @@
package tools
import (
"context"
"os"
"path/filepath"
"testing"
"time"
"github.com/sipeed/picoclaw/pkg/bus"
)
func TestScreenshotTool_Execute_URL_Send(t *testing.T) {
tmpDir := t.TempDir()
tool := NewScreenshotTool(tmpDir, true, 1024*1024)
tool.SetContext("telegram", "999")
tool.lookPath = func(s string) (string, error) { return "/usr/bin/" + s, nil }
tool.runCommand = func(ctx context.Context, command string, args ...string) error {
var output string
for _, arg := range args {
if len(arg) > len("--screenshot=") && arg[:13] == "--screenshot=" {
output = arg[13:]
}
}
if output == "" {
t.Fatalf("missing screenshot output arg")
}
return os.WriteFile(output, []byte("fake-image"), 0644)
}
var sent bus.OutboundMessage
tool.SetSendCallback(func(msg bus.OutboundMessage) error {
sent = msg
return nil
})
result := tool.Execute(context.Background(), map[string]interface{}{
"url": "https://example.com",
"caption": "cap",
})
if result.IsError {
t.Fatalf("expected success, got error: %s", result.ForLLM)
}
if !result.Silent {
t.Fatalf("expected silent result")
}
if len(sent.Attachments) != 1 {
t.Fatalf("expected one attachment, got %d", len(sent.Attachments))
}
if sent.Attachments[0].Type != "image" {
t.Fatalf("expected image attachment")
}
}
func TestScreenshotTool_Execute_NoSendReturnsPath(t *testing.T) {
tmpDir := t.TempDir()
tool := NewScreenshotTool(tmpDir, true, 1024*1024)
tool.lookPath = func(s string) (string, error) { return "/usr/bin/" + s, nil }
tool.runCommand = func(ctx context.Context, command string, args ...string) error {
output := filepath.Join(tmpDir, "tmp", "screenshots", "out.png")
for _, arg := range args {
if len(arg) > len("--screenshot=") && arg[:13] == "--screenshot=" {
output = arg[13:]
}
}
return os.WriteFile(output, []byte("fake-image"), 0644)
}
tool.now = func() time.Time { return time.Date(2026, 2, 14, 10, 0, 0, 0, time.UTC) }
result := tool.Execute(context.Background(), map[string]interface{}{
"url": "https://example.com",
"send": false,
})
if result.IsError {
t.Fatalf("expected success, got %s", result.ForLLM)
}
if result.ForUser == "" {
t.Fatalf("expected user-visible path")
}
}
func TestScreenshotTool_Execute_NoTargetWhenSend(t *testing.T) {
tmpDir := t.TempDir()
tool := NewScreenshotTool(tmpDir, true, 1024*1024)
tool.lookPath = func(s string) (string, error) { return "/usr/bin/" + s, nil }
tool.runCommand = func(ctx context.Context, command string, args ...string) error {
for _, arg := range args {
if len(arg) > len("--screenshot=") && arg[:13] == "--screenshot=" {
return os.WriteFile(arg[13:], []byte("fake-image"), 0644)
}
}
return nil
}
tool.SetSendCallback(func(msg bus.OutboundMessage) error { return nil })
result := tool.Execute(context.Background(), map[string]interface{}{
"url": "https://example.com",
})
if !result.IsError {
t.Fatalf("expected error without target context")
}
}

165
pkg/tools/send_file.go Normal file
View file

@ -0,0 +1,165 @@
package tools
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/sipeed/picoclaw/pkg/bus"
)
type SendFileCallback func(msg bus.OutboundMessage) error
type SendFileTool struct {
workspace string
restrict bool
maxFileBytes int64
sendCallback SendFileCallback
defaultChannel string
defaultChatID string
}
func NewSendFileTool(workspace string, restrict bool, maxFileBytes int64) *SendFileTool {
return &SendFileTool{
workspace: workspace,
restrict: restrict,
maxFileBytes: maxFileBytes,
}
}
func (t *SendFileTool) Name() string {
return "send_file"
}
func (t *SendFileTool) Description() string {
return "Send a local file to the user on the current chat channel."
}
func (t *SendFileTool) Parameters() map[string]interface{} {
return map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"path": map[string]interface{}{
"type": "string",
"description": "Path to the local file. Relative paths are resolved from workspace.",
},
"caption": map[string]interface{}{
"type": "string",
"description": "Optional text to include with the file.",
},
"attachment_type": map[string]interface{}{
"type": "string",
"description": "Optional attachment type: image or file.",
},
"filename": map[string]interface{}{
"type": "string",
"description": "Optional custom filename shown to user.",
},
"mime_type": map[string]interface{}{
"type": "string",
"description": "Optional mime type hint, for example image/png.",
},
"channel": map[string]interface{}{
"type": "string",
"description": "Optional target channel override.",
},
"chat_id": map[string]interface{}{
"type": "string",
"description": "Optional target chat id override.",
},
},
"required": []string{"path"},
}
}
func (t *SendFileTool) SetContext(channel, chatID string) {
t.defaultChannel = channel
t.defaultChatID = chatID
}
func (t *SendFileTool) SetSendCallback(callback SendFileCallback) {
t.sendCallback = callback
}
func (t *SendFileTool) Execute(ctx context.Context, args map[string]interface{}) *ToolResult {
path, ok := args["path"].(string)
if !ok || strings.TrimSpace(path) == "" {
return ErrorResult("path is required")
}
channel, _ := args["channel"].(string)
chatID, _ := args["chat_id"].(string)
if channel == "" {
channel = t.defaultChannel
}
if chatID == "" {
chatID = t.defaultChatID
}
if channel == "" || chatID == "" {
return ErrorResult("No target channel/chat specified")
}
if t.sendCallback == nil {
return ErrorResult("File sending not configured")
}
resolvedPath, err := validatePath(path, t.workspace, t.restrict)
if err != nil {
return ErrorResult(err.Error())
}
info, err := os.Stat(resolvedPath)
if err != nil {
return ErrorResult(fmt.Sprintf("failed to stat file: %v", err))
}
if info.IsDir() {
return ErrorResult("path points to a directory, expected a file")
}
if t.maxFileBytes > 0 && info.Size() > t.maxFileBytes {
return ErrorResult(fmt.Sprintf("file too large: %d bytes (limit %d bytes)", info.Size(), t.maxFileBytes))
}
attachmentType, _ := args["attachment_type"].(string)
if attachmentType == "" {
attachmentType = detectAttachmentType(resolvedPath)
}
if attachmentType != "image" && attachmentType != "file" {
attachmentType = "file"
}
fileName, _ := args["filename"].(string)
if fileName == "" {
fileName = filepath.Base(resolvedPath)
}
mimeType, _ := args["mime_type"].(string)
caption, _ := args["caption"].(string)
outbound := bus.OutboundMessage{
Channel: channel,
ChatID: chatID,
Content: caption,
Attachments: []bus.Attachment{
{
Type: attachmentType,
Path: resolvedPath,
FileName: fileName,
MIMEType: mimeType,
},
},
}
if err := t.sendCallback(outbound); err != nil {
return ErrorResult(fmt.Sprintf("sending file: %v", err))
}
return SilentResult(fmt.Sprintf("File sent to %s:%s (%s)", channel, chatID, fileName))
}
func detectAttachmentType(path string) string {
switch strings.ToLower(filepath.Ext(path)) {
case ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp":
return "image"
default:
return "file"
}
}

86
pkg/tools/send_file_test.go Normal file
View file

@ -0,0 +1,86 @@
package tools
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/sipeed/picoclaw/pkg/bus"
)
func TestSendFileTool_Execute_Success(t *testing.T) {
tmpDir := t.TempDir()
filePath := filepath.Join(tmpDir, "test.png")
if err := os.WriteFile(filePath, []byte("png-data"), 0644); err != nil {
t.Fatalf("write file: %v", err)
}
tool := NewSendFileTool(tmpDir, true, 1024*1024)
tool.SetContext("telegram", "123")
var sent bus.OutboundMessage
tool.SetSendCallback(func(msg bus.OutboundMessage) error {
sent = msg
return nil
})
result := tool.Execute(context.Background(), map[string]interface{}{
"path": "test.png",
"caption": "hello",
})
if result.IsError {
t.Fatalf("expected success, got error: %s", result.ForLLM)
}
if !result.Silent {
t.Fatalf("expected silent result")
}
if sent.Channel != "telegram" || sent.ChatID != "123" {
t.Fatalf("unexpected target: %s:%s", sent.Channel, sent.ChatID)
}
if len(sent.Attachments) != 1 {
t.Fatalf("expected one attachment, got %d", len(sent.Attachments))
}
if sent.Attachments[0].Type != "image" {
t.Fatalf("expected image attachment, got %s", sent.Attachments[0].Type)
}
}
func TestSendFileTool_Execute_NoTarget(t *testing.T) {
tmpDir := t.TempDir()
filePath := filepath.Join(tmpDir, "test.txt")
if err := os.WriteFile(filePath, []byte("hello"), 0644); err != nil {
t.Fatalf("write file: %v", err)
}
tool := NewSendFileTool(tmpDir, true, 1024*1024)
tool.SetSendCallback(func(msg bus.OutboundMessage) error { return nil })
result := tool.Execute(context.Background(), map[string]interface{}{
"path": filePath,
})
if !result.IsError {
t.Fatalf("expected error when no channel/chat context")
}
}
func TestSendFileTool_Execute_TooLarge(t *testing.T) {
tmpDir := t.TempDir()
filePath := filepath.Join(tmpDir, "big.bin")
if err := os.WriteFile(filePath, make([]byte, 64), 0644); err != nil {
t.Fatalf("write file: %v", err)
}
tool := NewSendFileTool(tmpDir, true, 32)
tool.SetContext("telegram", "123")
tool.SetSendCallback(func(msg bus.OutboundMessage) error { return nil })
result := tool.Execute(context.Background(), map[string]interface{}{
"path": filePath,
})
if !result.IsError {
t.Fatalf("expected too large error")
}
}

27
pkg/utils/media.go
View file

@ -141,3 +141,30 @@ func DownloadFileSimple(url, filename string) string {
LoggerPrefix: "media",
})
}
// ScheduleFileCleanup removes a file after delay in a background goroutine.
// Use this for transient media files that must survive past the current stack frame.
func ScheduleFileCleanup(path string, delay time.Duration, loggerPrefix string) {
if path == "" {
return
}
if delay <= 0 {
delay = 10 * time.Minute
}
if loggerPrefix == "" {
loggerPrefix = "media"
}
go func() {
timer := time.NewTimer(delay)
defer timer.Stop()
<-timer.C
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
logger.DebugCF(loggerPrefix, "Failed to cleanup temp file", map[string]interface{}{
"path": path,
"error": err.Error(),
})
}
}()
}