From e37aef3df541ba9f2a65dabf24e7d26d0cd5eddd Mon Sep 17 00:00:00 2001 From: Fushan Wen Date: Thu, 2 Apr 2026 14:24:45 +0800 Subject: [PATCH] feat: add model input type support for local image handling - Add Input field to ModelConfig for specifying supported input types (text, image) - Add SupportedInput field to AgentInstance, loaded from model config - Modify resolveMediaRefs to handle images based on model capabilities: - If model supports image: encode to base64 data URL - If model doesn't support image: inject local path for tool processing - Update findMatches to support lookup by both model_name and model field - Add injectPathTags to replace [image] tags with actual file paths - Add tests for GetModelConfig by model field lookup --- docs/configuration.md | 15 ++++++++---- docs/fr/configuration.md | 4 ++-- docs/fr/providers.md | 10 +++++--- docs/ja/configuration.md | 4 ++-- docs/ja/providers.md | 10 +++++--- docs/providers.md | 10 +++++--- docs/pt-br/configuration.md | 4 ++-- docs/pt-br/providers.md | 10 +++++--- docs/vi/configuration.md | 4 ++-- docs/vi/providers.md | 10 +++++--- docs/zh/configuration.md | 16 +++++++++---- docs/zh/providers.md | 10 +++++--- pkg/agent/instance.go | 13 ++++++++++ pkg/agent/loop.go | 13 ++++++---- pkg/agent/loop_media.go | 21 +++++++++++++---- pkg/agent/loop_test.go | 24 +++++++++---------- pkg/config/config.go | 25 +++++++++++++++++++- pkg/config/model_config_test.go | 42 +++++++++++++++++++++++++++++++++ 18 files changed, 187 insertions(+), 58 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 7a5902f58..9afe82166 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -650,7 +650,8 @@ For complete documentation, see [`security_configuration.md`](security_configura { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", - "api_keys": ["sk-your-openai-key"] + "api_keys": ["sk-your-openai-key"], + "input": ["text", "image"] }, { "model_name": "claude-sonnet-4.6", @@ -793,7 +794,8 @@ PicoClaw sends OpenAI-compatible requests to LM Studio, and strips the `lmstudio { "model_name": "my-custom-model", "model": "openai/custom-model", - "api_base": "https://my-proxy.com/v1" + "api_base": "https://my-proxy.com/v1", + "input": ["text"] // api_key: set in .security.yml } ``` @@ -824,7 +826,8 @@ model_list: { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", - "api_base": "https://api.openai.com/v1" + "api_base": "https://api.openai.com/v1", + "input": ["text", "image"] // api_keys loaded from .security.yml } ] @@ -840,13 +843,15 @@ model_list: "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api1.example.com/v1", - "api_keys": ["sk-key1"] + "api_keys": ["sk-key1"], + "input": ["text", "image"] }, { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api2.example.com/v1", - "api_keys": ["sk-key2"] + "api_keys": ["sk-key2"], + "input": ["text", "image"] } ] } diff --git a/docs/fr/configuration.md b/docs/fr/configuration.md index 7a57cceae..22a685fc6 100644 --- a/docs/fr/configuration.md +++ b/docs/fr/configuration.md @@ -334,8 +334,8 @@ Configurez plusieurs endpoints pour le même nom de modèle — PicoClaw effectu ```json { "model_list": [ - { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api1.example.com/v1", "api_keys": ["sk-key1"] }, - { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api2.example.com/v1", "api_keys": ["sk-key2"] } + { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api1.example.com/v1", "api_keys": ["sk-key1"], "input": ["text", "image"] }, + { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api2.example.com/v1", "api_keys": ["sk-key2"], "input": ["text", "image"] } ] } ``` diff --git a/docs/fr/providers.md b/docs/fr/providers.md index 3305ec5ee..f7c4fd88b 100644 --- a/docs/fr/providers.md +++ b/docs/fr/providers.md @@ -107,6 +107,7 @@ Cette conception permet également le **support multi-agents** avec une sélecti | `model` | string | Oui | Identifiant fournisseur/modèle (ex : `openai/gpt-5.4`, `azure/gpt-5.4`, `anthropic/claude-sonnet-4.6`) | | `api_keys` | string[] | Oui* | Clé(s) API pour l'authentification. Plusieurs clés permettent la rotation par requête. Non requis pour les fournisseurs locaux (Ollama, LM Studio, VLLM) | | `api_base` | string | Non | Remplace l'URL de base API par défaut | +| `input` | string[] | Non | Types d'entrée supportés par le modèle. Valeurs valides : `"text"`, `"image"`. Par défaut : `["text"]`. Lorsqu'un modèle ne supporte pas l'entrée d'images, les images sont téléchargées localement pour un traitement par outils au lieu d'être envoyées à l'API du modèle | | `proxy` | string | Non | URL du proxy HTTP pour cette entrée de modèle | | `user_agent` | string | Non | En-tête `User-Agent` personnalisé pour les requêtes API (supporté par les providers OpenAI-compatible, Anthropic et Azure) | | `request_timeout` | int | Non | Délai d'expiration de la requête en secondes (la valeur par défaut varie selon le provider) | @@ -125,7 +126,8 @@ Cette conception permet également le **support multi-agents** avec une sélecti { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", - "api_keys": ["sk-..."] + "api_keys": ["sk-..."], + "input": ["text", "image"] } ``` @@ -237,13 +239,15 @@ Configurez plusieurs endpoints pour le même nom de modèle — PicoClaw effectu "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api1.example.com/v1", - "api_keys": ["sk-key1"] + "api_keys": ["sk-key1"], + "input": ["text", "image"] }, { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api2.example.com/v1", - "api_keys": ["sk-key2"] + "api_keys": ["sk-key2"], + "input": ["text", "image"] } ] } diff --git a/docs/ja/configuration.md b/docs/ja/configuration.md index 6d6290e8a..1e1d05b3a 100644 --- a/docs/ja/configuration.md +++ b/docs/ja/configuration.md @@ -335,8 +335,8 @@ HEARTBEAT_OK を返信 ユーザーが直接結果を受信 ```json { "model_list": [ - { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api1.example.com/v1", "api_keys": ["sk-key1"] }, - { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api2.example.com/v1", "api_keys": ["sk-key2"] } + { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api1.example.com/v1", "api_keys": ["sk-key1"], "input": ["text", "image"] }, + { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api2.example.com/v1", "api_keys": ["sk-key2"], "input": ["text", "image"] } ] } ``` diff --git a/docs/ja/providers.md b/docs/ja/providers.md index 878530966..8238cb1d2 100644 --- a/docs/ja/providers.md +++ b/docs/ja/providers.md @@ -107,6 +107,7 @@ | `model` | string | はい | ベンダー/モデル識別子（例：`openai/gpt-5.4`、`azure/gpt-5.4`、`anthropic/claude-sonnet-4.6`） | | `api_keys` | string[] | はい* | 認証キー。複数キーでリクエストごとのローテーションが可能。ローカル provider（Ollama、LM Studio、VLLM）には不要 | | `api_base` | string | いいえ | デフォルトの API エンドポイント URL を上書き | +| `input` | string[] | いいえ | モデルがサポートする入力タイプ。有効な値：`"text"`、`"image"`。デフォルト：`["text"]`。モデルが画像入力をサポートしない場合、画像はモデル API に送信されず、ローカルにダウンロードされてツール処理されます | | `proxy` | string | いいえ | このモデルエントリの HTTP プロキシ URL | | `user_agent` | string | いいえ | カスタム `User-Agent` リクエストヘッダー（OpenAI 互換、Anthropic、Azure provider で対応） | | `request_timeout` | int | いいえ | リクエストタイムアウト（秒）。デフォルト値は provider により異なる | @@ -125,7 +126,8 @@ { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", - "api_keys": ["sk-..."] + "api_keys": ["sk-..."], + "input": ["text", "image"] } ``` @@ -248,13 +250,15 @@ PicoClaw はリクエスト送信前に外側の `litellm/` プレフィック "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api1.example.com/v1", - "api_keys": ["sk-key1"] + "api_keys": ["sk-key1"], + "input": ["text", "image"] }, { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api2.example.com/v1", - "api_keys": ["sk-key2"] + "api_keys": ["sk-key2"], + "input": ["text", "image"] } ] } diff --git a/docs/providers.md b/docs/providers.md index 9bb95446c..6e6ff3323 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -116,6 +116,7 @@ This design also enables **multi-agent support** with flexible provider selectio | `model` | string | Yes | Vendor/model identifier (e.g., `openai/gpt-5.4`, `azure/gpt-5.4`, `anthropic/claude-sonnet-4.6`) | | `api_keys` | string[] | Yes* | API key(s) for authentication. Multiple keys enable per-request rotation. Not required for local providers (Ollama, LM Studio, VLLM) | | `api_base` | string | No | Override the default API endpoint URL | +| `input` | string[] | No | Input types supported by the model. Valid values: `"text"`, `"image"`. Default: `["text"]`. When a model doesn't support image input, images are downloaded locally for tool processing instead of being sent to the model API | | `proxy` | string | No | HTTP proxy URL for this model entry | | `user_agent` | string | No | Custom `User-Agent` header sent with API requests (supported by OpenAI-compatible, Anthropic, and Azure providers) | | `request_timeout` | int | No | Request timeout in seconds (default varies by provider) | @@ -161,7 +162,8 @@ If `voice.model_name` is not configured, PicoClaw will continue to fall back to { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", - "api_keys": ["sk-..."] + "api_keys": ["sk-..."], + "input": ["text", "image"] } ``` @@ -311,13 +313,15 @@ Configure multiple endpoints for the same model name—PicoClaw will automatical "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api1.example.com/v1", - "api_keys": ["sk-key1"] + "api_keys": ["sk-key1"], + "input": ["text", "image"] }, { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api2.example.com/v1", - "api_keys": ["sk-key2"] + "api_keys": ["sk-key2"], + "input": ["text", "image"] } ] } diff --git a/docs/pt-br/configuration.md b/docs/pt-br/configuration.md index 27cd6d21f..5cafeff39 100644 --- a/docs/pt-br/configuration.md +++ b/docs/pt-br/configuration.md @@ -335,8 +335,8 @@ Configure múltiplos endpoints para o mesmo nome de modelo — PicoClaw fará ro ```json { "model_list": [ - { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api1.example.com/v1", "api_keys": ["sk-key1"] }, - { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api2.example.com/v1", "api_keys": ["sk-key2"] } + { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api1.example.com/v1", "api_keys": ["sk-key1"], "input": ["text", "image"] }, + { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api2.example.com/v1", "api_keys": ["sk-key2"], "input": ["text", "image"] } ] } ``` diff --git a/docs/pt-br/providers.md b/docs/pt-br/providers.md index 103490dc7..83d5af061 100644 --- a/docs/pt-br/providers.md +++ b/docs/pt-br/providers.md @@ -107,6 +107,7 @@ Este design também permite **suporte multi-agente** com seleção flexível de | `model` | string | Sim | Identificador fornecedor/modelo (ex: `openai/gpt-5.4`, `azure/gpt-5.4`, `anthropic/claude-sonnet-4.6`) | | `api_keys` | string[] | Sim* | Chave(s) API para autenticação. Múltiplas chaves permitem rotação por requisição. Não necessário para providers locais (Ollama, LM Studio, VLLM) | | `api_base` | string | Não | Substitui a URL base da API padrão | +| `input` | string[] | Não | Tipos de entrada suportados pelo modelo. Valores válidos: `"text"`, `"image"`. Padrão: `["text"]`. Quando um modelo não suporta entrada de imagem, as imagens são baixadas localmente para processamento por ferramentas em vez de enviadas à API do modelo | | `proxy` | string | Não | URL do proxy HTTP para esta entrada de modelo | | `user_agent` | string | Não | Cabeçalho `User-Agent` personalizado enviado com requisições API (suportado por providers OpenAI-compatible, Anthropic e Azure) | | `request_timeout` | int | Não | Timeout de requisição em segundos (o padrão varia por provider) | @@ -125,7 +126,8 @@ Este design também permite **suporte multi-agente** com seleção flexível de { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", - "api_keys": ["sk-..."] + "api_keys": ["sk-..."], + "input": ["text", "image"] } ``` @@ -237,13 +239,15 @@ Configure múltiplos endpoints para o mesmo nome de modelo — o PicoClaw fará "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api1.example.com/v1", - "api_keys": ["sk-key1"] + "api_keys": ["sk-key1"], + "input": ["text", "image"] }, { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api2.example.com/v1", - "api_keys": ["sk-key2"] + "api_keys": ["sk-key2"], + "input": ["text", "image"] } ] } diff --git a/docs/vi/configuration.md b/docs/vi/configuration.md index 56eb8f557..16f2fd899 100644 --- a/docs/vi/configuration.md +++ b/docs/vi/configuration.md @@ -335,8 +335,8 @@ Cấu hình nhiều endpoint cho cùng tên mô hình — PicoClaw sẽ tự đ ```json { "model_list": [ - { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api1.example.com/v1", "api_keys": ["sk-key1"] }, - { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api2.example.com/v1", "api_keys": ["sk-key2"] } + { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api1.example.com/v1", "api_keys": ["sk-key1"], "input": ["text", "image"] }, + { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api2.example.com/v1", "api_keys": ["sk-key2"], "input": ["text", "image"] } ] } ``` diff --git a/docs/vi/providers.md b/docs/vi/providers.md index 46c9de663..ae1358709 100644 --- a/docs/vi/providers.md +++ b/docs/vi/providers.md @@ -107,6 +107,7 @@ Thiết kế này cũng cho phép **hỗ trợ đa agent** với lựa chọn pr | `model` | string | Có | Định danh nhà cung cấp/model (ví dụ: `openai/gpt-5.4`, `azure/gpt-5.4`, `anthropic/claude-sonnet-4.6`) | | `api_keys` | string[] | Có* | Khóa API xác thực. Nhiều khóa cho phép xoay vòng theo yêu cầu. Không cần thiết cho provider nội bộ (Ollama, LM Studio, VLLM) | | `api_base` | string | Không | Ghi đè URL endpoint API mặc định | +| `input` | string[] | Không | Các loại đầu vào được hỗ trợ bởi mô hình. Giá trị hợp lệ: `"text"`, `"image"`. Mặc định: `["text"]`. Khi mô hình không hỗ trợ đầu vào hình ảnh, hình ảnh được tải xuống cục bộ để xử lý bằng công cụ thay vì gửi đến API mô hình | | `proxy` | string | Không | URL proxy HTTP cho entry model này | | `user_agent` | string | Không | Header `User-Agent` tùy chỉnh gửi với yêu cầu API (được hỗ trợ bởi provider OpenAI-compatible, Anthropic và Azure) | | `request_timeout` | int | Không | Timeout yêu cầu tính bằng giây (mặc định khác nhau tùy provider) | @@ -125,7 +126,8 @@ Thiết kế này cũng cho phép **hỗ trợ đa agent** với lựa chọn pr { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", - "api_keys": ["sk-..."] + "api_keys": ["sk-..."], + "input": ["text", "image"] } ``` @@ -237,13 +239,15 @@ Cấu hình nhiều endpoint cho cùng tên mô hình — PicoClaw sẽ tự đ "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api1.example.com/v1", - "api_keys": ["sk-key1"] + "api_keys": ["sk-key1"], + "input": ["text", "image"] }, { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api2.example.com/v1", - "api_keys": ["sk-key2"] + "api_keys": ["sk-key2"], + "input": ["text", "image"] } ] } diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index a405df09c..7b7830cfa 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -392,7 +392,8 @@ Agent 读取 HEARTBEAT.md { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", - "api_keys": ["sk-your-openai-key"] + "api_keys": ["sk-your-openai-key"], + "input": ["text", "image"] }, { "model_name": "claude-sonnet-4.6", @@ -413,6 +414,10 @@ Agent 读取 HEARTBEAT.md } ``` +> **安全提示：** 你可以将 `api_keys` 字段从配置中移除，存储在 `.security.yml` 中。详见上方的 [安全配置](#-安全配置推荐)。 +> +> **注意：** `enabled` 字段可设置为 `false` 以禁用某个模型条目而无需删除它。省略时，迁移过程中对于有 API Key 的模型默认为 `true`。 + #### 各厂商配置示例
@@ -530,7 +535,8 @@ PicoClaw 向 LM Studio 的 OpenAI 兼容终结点发送请求，且将移除首 "model_name": "my-custom-model", "model": "openai/custom-model", "api_base": "https://my-proxy.com/v1", - "api_keys": ["sk-..."] + "api_keys": ["sk-..."], + "input": ["text"] } ``` @@ -549,13 +555,15 @@ PicoClaw 只剥离最外层的 `litellm/` 前缀再发送请求，因此 `litell "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api1.example.com/v1", - "api_keys": ["sk-key1"] + "api_keys": ["sk-key1"], + "input": ["text", "image"] }, { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api2.example.com/v1", - "api_keys": ["sk-key2"] + "api_keys": ["sk-key2"], + "input": ["text", "image"] } ] } diff --git a/docs/zh/providers.md b/docs/zh/providers.md index 6048b929f..3da2fcea7 100644 --- a/docs/zh/providers.md +++ b/docs/zh/providers.md @@ -112,6 +112,7 @@ | `model` | string | 是 | 厂商/模型标识符（如 `openai/gpt-5.4`、`azure/gpt-5.4`、`anthropic/claude-sonnet-4.6`） | | `api_keys` | string[] | 是* | 认证密钥。多个密钥可按请求轮换。本地 provider（Ollama、LM Studio、VLLM）不需要 | | `api_base` | string | 否 | 覆盖默认的 API 端点 URL | +| `input` | string[] | 否 | 模型支持的输入类型。有效值：`"text"`、`"image"`。默认：`["text"]`。当模型不支持图片输入时，图片会下载到本地进行工具处理，而不是发送到模型 API | | `proxy` | string | 否 | 此模型条目的 HTTP 代理 URL | | `user_agent` | string | 否 | 自定义 `User-Agent` 请求头（支持 OpenAI 兼容、Anthropic 和 Azure provider） | | `request_timeout` | int | 否 | 请求超时时间（秒），默认值因 provider 而异 | @@ -157,7 +158,8 @@ { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", - "api_keys": ["sk-..."] + "api_keys": ["sk-..."], + "input": ["text", "image"] } ``` @@ -281,13 +283,15 @@ PicoClaw 在发送请求前仅去除外层 `litellm/` 前缀，因此 `litellm/l "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api1.example.com/v1", - "api_keys": ["sk-key1"] + "api_keys": ["sk-key1"], + "input": ["text", "image"] }, { "model_name": "gpt-5.4", "model": "openai/gpt-5.4", "api_base": "https://api2.example.com/v1", - "api_keys": ["sk-key2"] + "api_keys": ["sk-key2"], + "input": ["text", "image"] } ] } diff --git a/pkg/agent/instance.go b/pkg/agent/instance.go index bacfa49c5..0f62c5a8c 100644 --- a/pkg/agent/instance.go +++ b/pkg/agent/instance.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "regexp" + "slices" "strings" "github.com/sipeed/picoclaw/pkg/config" @@ -33,6 +34,7 @@ type AgentInstance struct { ContextWindow int SummarizeMessageThreshold int SummarizeTokenPercent int + SupportedInput []string Provider providers.LLMProvider Sessions session.SessionStore ContextBuilder *ContextBuilder @@ -53,6 +55,11 @@ type AgentInstance struct { LightProvider providers.LLMProvider } +// SupportsInput checks if the agent supports a specific input type. +func (a *AgentInstance) SupportsInput(inputType string) bool { + return slices.Contains(a.SupportedInput, inputType) +} + // NewAgentInstance creates an agent instance from config. func NewAgentInstance( agentCfg *config.AgentConfig, @@ -157,8 +164,13 @@ func NewAgentInstance( } var thinkingLevelStr string + var supportedInput []string if mc, err := cfg.GetModelConfig(model); err == nil { thinkingLevelStr = mc.ThinkingLevel + supportedInput = mc.Input + } + if len(supportedInput) == 0 { + supportedInput = []string{"text"} } thinkingLevel := parseThinkingLevel(thinkingLevelStr) @@ -220,6 +232,7 @@ func NewAgentInstance( ContextWindow: contextWindow, SummarizeMessageThreshold: summarizeMessageThreshold, SummarizeTokenPercent: summarizeTokenPercent, + SupportedInput: supportedInput, Provider: provider, Sessions: sessions, ContextBuilder: contextBuilder, diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 808d12c07..0b6412734 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -347,8 +347,9 @@ func registerSharedTools( // resolve media:// refs in the same way the main AgentLoop does. // This keeps subagent vision support working even when the optimized // sub-turn spawner path is unavailable. + supportsImage := agent.SupportsInput("image") subagentManager.SetMediaResolver(func(msgs []providers.Message) []providers.Message { - return resolveMediaRefs(msgs, al.mediaStore, cfg.Agents.Defaults.GetMaxMediaSize()) + return resolveMediaRefs(msgs, al.mediaStore, cfg.Agents.Defaults.GetMaxMediaSize(), supportsImage) }) // Set the spawner that links into AgentLoop's turnState @@ -1732,7 +1733,9 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState) (turnResult, er cfg := al.GetConfig() maxMediaSize := cfg.Agents.Defaults.GetMaxMediaSize() - messages = resolveMediaRefs(messages, al.mediaStore, maxMediaSize) + + supportsImage := ts.agent.SupportsInput("image") + messages = resolveMediaRefs(messages, al.mediaStore, maxMediaSize, supportsImage) if !ts.opts.NoHistory { toolDefs := ts.agent.Tools.ToProviderDefs() @@ -1764,7 +1767,7 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState) (turnResult, er ts.opts.SenderID, ts.opts.SenderDisplayName, activeSkillNames(ts.agent, ts.opts)..., ) - messages = resolveMediaRefs(messages, al.mediaStore, maxMediaSize) + messages = resolveMediaRefs(messages, al.mediaStore, maxMediaSize, supportsImage) } } @@ -1849,7 +1852,7 @@ turnLoop: // Inject pending steering messages if len(pendingMessages) > 0 { - resolvedPending := resolveMediaRefs(pendingMessages, al.mediaStore, maxMediaSize) + resolvedPending := resolveMediaRefs(pendingMessages, al.mediaStore, maxMediaSize, supportsImage) totalContentLen := 0 for i, pm := range pendingMessages { messages = append(messages, resolvedPending[i]) @@ -1915,7 +1918,7 @@ turnLoop: // before entering the loop; only subsequent iterations can contain new // tool-generated media refs that need base64 encoding. if iteration > 1 { - messages = resolveMediaRefs(messages, al.mediaStore, maxMediaSize) + messages = resolveMediaRefs(messages, al.mediaStore, maxMediaSize, supportsImage) } callMessages := messages diff --git a/pkg/agent/loop_media.go b/pkg/agent/loop_media.go index e8314c10d..c7aefd523 100644 --- a/pkg/agent/loop_media.go +++ b/pkg/agent/loop_media.go @@ -21,11 +21,18 @@ import ( ) // resolveMediaRefs resolves media:// refs in messages. -// Images are base64-encoded into the Media array for multimodal LLMs. -// Non-image files (documents, audio, video) have their local path injected +// Images are base64-encoded into the Media array for multimodal LLMs that support image input. +// If the model doesn't support image input, images are treated like other media files: +// their local path is injected into Content so the agent can access them via file tools. +// Non-image files (documents, audio, video) always have their local path injected // into Content so the agent can access them via file tools like read_file. // Returns a new slice; original messages are not mutated. -func resolveMediaRefs(messages []providers.Message, store media.MediaStore, maxSize int) []providers.Message { +func resolveMediaRefs( + messages []providers.Message, + store media.MediaStore, + maxSize int, + supportsImage bool, +) []providers.Message { if store == nil { return messages } @@ -67,7 +74,7 @@ func resolveMediaRefs(messages []providers.Message, store media.MediaStore, maxS mime := detectMIME(localPath, meta) - if strings.HasPrefix(mime, "image/") { + if strings.HasPrefix(mime, "image/") && supportsImage { dataURL := encodeImageToDataURL(localPath, mime, info, maxSize) if dataURL != "" { resolved = append(resolved, dataURL) @@ -160,9 +167,11 @@ func encodeImageToDataURL(localPath, mime string, info os.FileInfo, maxSize int) } // buildPathTag creates a structured tag exposing the local file path. -// Tag type is derived from MIME: [audio:/path], [video:/path], or [file:/path]. +// Tag type is derived from MIME: [image:/path], [audio:/path], [video:/path], or [file:/path]. func buildPathTag(mime, localPath string) string { switch { + case strings.HasPrefix(mime, "image/"): + return "[image:" + localPath + "]" case strings.HasPrefix(mime, "audio/"): return "[audio:" + localPath + "]" case strings.HasPrefix(mime, "video/"): @@ -178,6 +187,8 @@ func injectPathTags(content string, tags []string) string { for _, tag := range tags { var generic string switch { + case strings.HasPrefix(tag, "[image:"): + generic = "[image]" case strings.HasPrefix(tag, "[audio:"): generic = "[audio]" case strings.HasPrefix(tag, "[video:"): diff --git a/pkg/agent/loop_test.go b/pkg/agent/loop_test.go index 9513d8aca..bde7b2e64 100644 --- a/pkg/agent/loop_test.go +++ b/pkg/agent/loop_test.go @@ -2600,7 +2600,7 @@ func TestResolveMediaRefs_ResolvesToBase64(t *testing.T) { messages := []providers.Message{ {Role: "user", Content: "describe this", Media: []string{ref}}, } - result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize) + result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, true) if len(result[0].Media) != 1 { t.Fatalf("expected 1 resolved media, got %d", len(result[0].Media)) @@ -2627,7 +2627,7 @@ func TestResolveMediaRefs_SkipsOversizedFile(t *testing.T) { {Role: "user", Content: "hi", Media: []string{ref}}, } // Use a tiny limit (1KB) so the file is oversized - result := resolveMediaRefs(messages, store, 1024) + result := resolveMediaRefs(messages, store, 1024, true) if len(result[0].Media) != 0 { t.Fatalf("expected 0 media (oversized), got %d", len(result[0].Media)) @@ -2647,7 +2647,7 @@ func TestResolveMediaRefs_UnknownTypeInjectsPath(t *testing.T) { messages := []providers.Message{ {Role: "user", Content: "hi", Media: []string{ref}}, } - result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize) + result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, true) if len(result[0].Media) != 0 { t.Fatalf("expected 0 media entries, got %d", len(result[0].Media)) @@ -2662,7 +2662,7 @@ func TestResolveMediaRefs_PassesThroughNonMediaRefs(t *testing.T) { messages := []providers.Message{ {Role: "user", Content: "hi", Media: []string{"https://example.com/img.png"}}, } - result := resolveMediaRefs(messages, nil, config.DefaultMaxMediaSize) + result := resolveMediaRefs(messages, nil, config.DefaultMaxMediaSize, true) if len(result[0].Media) != 1 || result[0].Media[0] != "https://example.com/img.png" { t.Fatalf("expected passthrough of non-media:// URL, got %v", result[0].Media) @@ -2687,7 +2687,7 @@ func TestResolveMediaRefs_DoesNotMutateOriginal(t *testing.T) { } originalRef := original[0].Media[0] - resolveMediaRefs(original, store, config.DefaultMaxMediaSize) + resolveMediaRefs(original, store, config.DefaultMaxMediaSize, true) if original[0].Media[0] != originalRef { t.Fatal("resolveMediaRefs mutated original message slice") @@ -2707,7 +2707,7 @@ func TestResolveMediaRefs_UsesMetaContentType(t *testing.T) { messages := []providers.Message{ {Role: "user", Content: "hi", Media: []string{ref}}, } - result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize) + result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, true) if len(result[0].Media) != 1 { t.Fatalf("expected 1 media, got %d", len(result[0].Media)) @@ -2729,7 +2729,7 @@ func TestResolveMediaRefs_PDFInjectsFilePath(t *testing.T) { messages := []providers.Message{ {Role: "user", Content: "report.pdf [file]", Media: []string{ref}}, } - result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize) + result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, true) if len(result[0].Media) != 0 { t.Fatalf("expected 0 media (non-image), got %d", len(result[0].Media)) @@ -2751,7 +2751,7 @@ func TestResolveMediaRefs_AudioInjectsAudioPath(t *testing.T) { messages := []providers.Message{ {Role: "user", Content: "voice.ogg [audio]", Media: []string{ref}}, } - result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize) + result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, true) if len(result[0].Media) != 0 { t.Fatalf("expected 0 media, got %d", len(result[0].Media)) @@ -2773,7 +2773,7 @@ func TestResolveMediaRefs_VideoInjectsVideoPath(t *testing.T) { messages := []providers.Message{ {Role: "user", Content: "clip.mp4 [video]", Media: []string{ref}}, } - result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize) + result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, true) if len(result[0].Media) != 0 { t.Fatalf("expected 0 media, got %d", len(result[0].Media)) @@ -2795,7 +2795,7 @@ func TestResolveMediaRefs_NoGenericTagAppendsPath(t *testing.T) { messages := []providers.Message{ {Role: "user", Content: "here is my data", Media: []string{ref}}, } - result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize) + result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, true) expected := "here is my data [file:" + csvPath + "]" if result[0].Content != expected { @@ -2815,7 +2815,7 @@ func TestResolveMediaRefs_EmptyContentGetsPathTag(t *testing.T) { messages := []providers.Message{ {Role: "user", Content: "", Media: []string{ref}}, } - result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize) + result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, true) expected := "[file:" + docPath + "]" if result[0].Content != expected { @@ -2844,7 +2844,7 @@ func TestResolveMediaRefs_MixedImageAndFile(t *testing.T) { messages := []providers.Message{ {Role: "user", Content: "check these [file]", Media: []string{imgRef, fileRef}}, } - result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize) + result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, true) if len(result[0].Media) != 1 { t.Fatalf("expected 1 media (image only), got %d", len(result[0].Media)) diff --git a/pkg/config/config.go b/pkg/config/config.go index 4e8733cbf..2c1984242 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -7,6 +7,7 @@ import ( "math/rand" "os" "path/filepath" + "slices" "strings" "sync/atomic" "time" @@ -578,6 +579,10 @@ type ModelConfig struct { ModelName string `json:"model_name"` // User-facing alias for the model Model string `json:"model"` // Protocol/model-identifier (e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4.6") + // Input types supported by this model (e.g., ["text"], ["text", "image"]) + // Default is ["text"] if not specified + Input []string `json:"input,omitempty"` + // HTTP-based providers APIBase string `json:"api_base,omitempty"` // API endpoint URL Proxy string `json:"proxy,omitempty"` // HTTP proxy URL @@ -622,6 +627,15 @@ func (c *ModelConfig) IsVirtual() bool { return c.isVirtual } +// SupportsInput checks if the model supports a specific input type. +// If Input is empty or nil, it defaults to ["text"]. +func (c *ModelConfig) SupportsInput(inputType string) bool { + if len(c.Input) == 0 { + return inputType == "text" + } + return slices.Contains(c.Input, inputType) +} + // Validate checks if the ModelConfig has all required fields. func (c *ModelConfig) Validate() error { if c.ModelName == "" { @@ -1156,7 +1170,8 @@ func (c *Config) GetModelConfig(modelName string) (*ModelConfig, error) { return matches[idx], nil } -// findMatches finds all ModelConfig entries with the given model_name. +// findMatches finds all ModelConfig entries with the given model_name or model field. +// It first tries to match by model_name, then by model field if no match is found. func (c *Config) findMatches(modelName string) []*ModelConfig { var matches []*ModelConfig for i := range c.ModelList { @@ -1164,6 +1179,14 @@ func (c *Config) findMatches(modelName string) []*ModelConfig { matches = append(matches, c.ModelList[i]) } } + if len(matches) > 0 { + return matches + } + for i := range c.ModelList { + if c.ModelList[i].Model == modelName { + matches = append(matches, c.ModelList[i]) + } + } return matches } diff --git a/pkg/config/model_config_test.go b/pkg/config/model_config_test.go index 6e88f4783..a2d73d8e6 100644 --- a/pkg/config/model_config_test.go +++ b/pkg/config/model_config_test.go @@ -144,6 +144,48 @@ func TestGetModelConfig_Concurrent(t *testing.T) { } } +func TestGetModelConfig_ByModelField(t *testing.T) { + cfg := &Config{ + Version: CurrentVersion, + ModelList: []*ModelConfig{ + {ModelName: "test-model", Model: "openai/gpt-4o", APIKeys: SimpleSecureStrings("key1")}, + {ModelName: "other-model", Model: "anthropic/claude", APIKeys: SimpleSecureStrings("key2")}, + }, + } + + result, err := cfg.GetModelConfig("openai/gpt-4o") + if err != nil { + t.Fatalf("GetModelConfig() error = %v", err) + } + if result.Model != "openai/gpt-4o" { + t.Errorf("Model = %q, want %q", result.Model, "openai/gpt-4o") + } + if result.ModelName != "test-model" { + t.Errorf("ModelName = %q, want %q", result.ModelName, "test-model") + } +} + +func TestGetModelConfig_PrefersModelNameOverModel(t *testing.T) { + cfg := &Config{ + Version: CurrentVersion, + ModelList: []*ModelConfig{ + {ModelName: "gpt-4", Model: "openai/gpt-4", APIKeys: SimpleSecureStrings("key1")}, + {ModelName: "openai/gpt-4", Model: "openai/gpt-4-turbo", APIKeys: SimpleSecureStrings("key2")}, + }, + } + + result, err := cfg.GetModelConfig("gpt-4") + if err != nil { + t.Fatalf("GetModelConfig() error = %v", err) + } + if result.Model != "openai/gpt-4" { + t.Errorf("Model = %q, want %q", result.Model, "openai/gpt-4") + } + if result.ModelName != "gpt-4" { + t.Errorf("ModelName = %q, want %q", result.ModelName, "gpt-4") + } +} + func TestAgentDefaultsV0_JSON_BackwardCompat(t *testing.T) { tests := []struct { name string