diff --git a/docs/hooks/plugin-tool-injection.md b/docs/hooks/plugin-tool-injection.md index 6e0f78529..b95fab3e6 100644 --- a/docs/hooks/plugin-tool-injection.md +++ b/docs/hooks/plugin-tool-injection.md @@ -273,7 +273,9 @@ Tool definition follows OpenAI function calling format: "for_llm": "Content returned to LLM", "for_user": "Optional, content sent to user", "silent": false, - "is_error": false + "is_error": false, + "media": ["Optional, media reference list"], + "response_handled": false } } ``` @@ -284,6 +286,85 @@ Tool definition follows OpenAI function calling format: | `for_user` | Optional, sent directly to user | | `silent` | When true, not sent to user | | `is_error` | When true, indicates execution failure | +| `media` | Optional, media file references (images, files, etc.) | +| `response_handled` | When true, indicates user request is handled, turn will end | + +--- + +## Media File Handling + +The `respond` action supports returning media files (images, files, etc.). There are two processing modes: + +### 1. Automatic Delivery (`response_handled=true`) + +When `response_handled=true`, media files are automatically sent to the user and the turn ends: + +```json +{ + "action": "respond", + "result": { + "for_llm": "Image sent to user", + "for_user": "", + "media": ["media://abc123"], + "response_handled": true + } +} +``` + +Use cases: +- Image generation plugin directly returning results +- File download plugin sending files to user + +### 2. LLM Visible (`response_handled=false`) + +When `response_handled=false`, media references are passed to the LLM, which can see the content in the next request: + +```json +{ + "action": "respond", + "result": { + "for_llm": "Image loaded, path: /tmp/image.png [file:/tmp/image.png]", + "media": ["media://abc123"] + } +} +``` + +After seeing the content, the LLM can decide: +- Use `send_file` tool to send to user +- Analyze image content and reply to user +- Other processing approaches + +### Media Reference Format + +Media references use the `media://` protocol: + +``` +media:// +``` + +These references are managed by PicoClaw's MediaStore and can be: +- Sent to user via channel +- Converted to base64 in LLM vision requests + +### Alternative: Use Existing Tools + +If the plugin generates files, you can return the file path and let the LLM call `send_file` or similar tools: + +```json +{ + "action": "respond", + "result": { + "for_llm": "Image generated, saved at /tmp/generated_image.png. Use send_file tool to send to user.", + "for_user": "", + "silent": false + } +} +``` + +This approach: +- More decoupled, LLM decides when to send +- Leverages existing tool mechanisms +- Supports batch sending, delayed sending, etc. --- diff --git a/docs/hooks/plugin-tool-injection.zh.md b/docs/hooks/plugin-tool-injection.zh.md index 3ecc32322..8eebda7e3 100644 --- a/docs/hooks/plugin-tool-injection.zh.md +++ b/docs/hooks/plugin-tool-injection.zh.md @@ -273,7 +273,9 @@ if __name__ == "__main__": "for_llm": "返回给 LLM 的内容", "for_user": "可选，发送给用户的内容", "silent": false, - "is_error": false + "is_error": false, + "media": ["可选，媒体引用列表"], + "response_handled": false } } ``` @@ -284,6 +286,85 @@ if __name__ == "__main__": | `for_user` | 可选，直接发送给用户 | | `silent` | 为 true 时不发送给用户 | | `is_error` | 为 true 时表示执行失败 | +| `media` | 可选，媒体文件引用列表（如图片、文件） | +| `response_handled` | 为 true 时表示已处理用户请求，轮次将结束 | + +--- + +## 媒体文件处理 + +`respond` action 支持返回媒体文件（图片、文件等）。有两种处理方式： + +### 1. 自动发送（`response_handled=true`） + +当 `response_handled=true` 时，媒体文件会自动发送给用户，轮次结束： + +```json +{ + "action": "respond", + "result": { + "for_llm": "图片已发送给用户", + "for_user": "", + "media": ["media://abc123"], + "response_handled": true + } +} +``` + +适用场景： +- 图像生成插件直接返回结果 +- 文件下载插件发送文件给用户 + +### 2. LLM 可见（`response_handled=false`） + +当 `response_handled=false` 时，媒体引用会传递给 LLM，LLM 可以在下一轮请求中看到内容： + +```json +{ + "action": "respond", + "result": { + "for_llm": "图片已加载，路径：/tmp/image.png [file:/tmp/image.png]", + "media": ["media://abc123"] + } +} +``` + +LLM 看到内容后，可以自主决定： +- 使用 `send_file` 工具发送给用户 +- 分析图片内容并回复用户 +- 其他处理方式 + +### 媒体引用格式 + +媒体引用使用 `media://` 协议： + +``` +media:// +``` + +这些引用由 PicoClaw 的 MediaStore 管理，可以： +- 通过 channel 发送给用户 +- 在 LLM vision 请求中转换为 base64 + +### 替代方案：使用现有工具 + +如果插件生成文件，可以返回文件路径让 LLM 调用 `send_file` 等工具： + +```json +{ + "action": "respond", + "result": { + "for_llm": "图片已生成，保存在 /tmp/generated_image.png。使用 send_file 工具发送给用户。", + "for_user": "", + "silent": false + } +} +``` + +这种方式： +- 更解耦，LLM 自主决策发送时机 +- 利用现有工具机制 +- 支持批量发送、延迟发送等场景 --- diff --git a/pkg/agent/hooks.go b/pkg/agent/hooks.go index 0d30a0d39..c0dfa8399 100644 --- a/pkg/agent/hooks.go +++ b/pkg/agent/hooks.go @@ -133,7 +133,7 @@ type ToolCallHookRequest struct { Arguments map[string]any `json:"arguments,omitempty"` Channel string `json:"channel,omitempty"` ChatID string `json:"chat_id,omitempty"` - HookResult *tools.ToolResult `json:"hook_result,omitempty"` // Result returned directly by hook (for respond action) + HookResult *tools.ToolResult `json:"hook_result,omitempty"` // Result returned directly by hook (for respond action). Media is supported - see Media handling section in docs. } func (r *ToolCallHookRequest) Clone() *ToolCallHookRequest { diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index c65fc1685..f11bc5399 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -2407,6 +2407,41 @@ turnLoop: }) } + // Handle media from hook result (same as normal tool execution) + if len(hookResult.Media) > 0 && hookResult.ResponseHandled { + parts := make([]bus.MediaPart, 0, len(hookResult.Media)) + for _, ref := range hookResult.Media { + part := bus.MediaPart{Ref: ref} + if al.mediaStore != nil { + if _, meta, err := al.mediaStore.ResolveWithMeta(ref); err == nil { + part.Filename = meta.Filename + part.ContentType = meta.ContentType + part.Type = inferMediaType(meta.Filename, meta.ContentType) + } + } + parts = append(parts, part) + } + outboundMedia := bus.OutboundMediaMessage{ + Channel: ts.channel, + ChatID: ts.chatID, + Parts: parts, + } + if al.channelManager != nil && ts.channel != "" && !constants.IsInternalChannel(ts.channel) { + if err := al.channelManager.SendMedia(ctx, outboundMedia); err != nil { + logger.WarnCF("agent", "Failed to deliver hook media", + map[string]any{ + "agent_id": ts.agent.ID, + "tool": toolName, + "channel": ts.channel, + "chat_id": ts.chatID, + "error": err.Error(), + }) + } + } else if al.bus != nil { + al.bus.PublishOutboundMedia(ctx, outboundMedia) + } + } + // Track response handling status (same as normal tool execution) if !hookResult.ResponseHandled { allResponsesHandled = false @@ -2423,6 +2458,19 @@ turnLoop: Content: contentForLLM, ToolCallID: tc.ID, } + + // Handle media for LLM vision (same as normal tool execution) + if len(hookResult.Media) > 0 && !hookResult.ResponseHandled { + hookResult.ArtifactTags = buildArtifactTags(al.mediaStore, hookResult.Media) + // Recalculate contentForLLM after adding ArtifactTags + contentForLLM = hookResult.ContentForLLM() + if al.cfg.Tools.IsFilterSensitiveDataEnabled() { + contentForLLM = al.cfg.FilterSensitiveData(contentForLLM) + } + toolResultMsg.Content = contentForLLM + toolResultMsg.Media = append(toolResultMsg.Media, hookResult.Media...) + } + messages = append(messages, toolResultMsg) if !ts.opts.NoHistory { ts.agent.Sessions.AddFullMessage(ts.sessionKey, toolResultMsg) @@ -2432,7 +2480,13 @@ turnLoop: // Skip subsequent tool execution flow continue } - // If no HookResult, fall back to continue + // If no HookResult, fall back to continue with warning + logger.WarnCF("agent", "Hook returned respond action but no HookResult provided", + map[string]any{ + "agent_id": ts.agent.ID, + "tool": toolName, + "action": "respond", + }) case HookActionDenyTool: allResponsesHandled = false denyContent := hookDeniedToolContent("Tool execution denied by hook", decision.Reason)