feat(agent): enhance image content handling and vision support

- Added support for processing image content in the agent, providing placeholder text when image parsing fails.
- Implemented a fallback mechanism to utilize vision tools for image analysis, improving the handling of image content.
- Updated tests to validate the new image handling behavior, ensuring correct responses when vision support is unavailable.
- Enhanced system prompts to guide users on using the new `image_read` tool for image analysis, improving user experience.
This commit is contained in:
Max 2026-05-04 13:46:30 +08:00
parent 222daaf645
commit 54607e26b7
18 changed files with 1010 additions and 26 deletions

View file

@ -77,7 +77,15 @@ func parseContentParts(ctx *agentContext.Context, message agentContext.Message,
for _, part := range content { for _, part := range content {
parsedPart, refs, err := parseContentPart(ctx, part, options) parsedPart, refs, err := parseContentPart(ctx, part, options)
if err != nil { if err != nil {
if part.Type == agentContext.ContentImageURL {
parts = append(parts, agentContext.ContentPart{
Type: agentContext.ContentText,
Text: "[Image content could not be processed]",
})
} else {
parts = append(parts, part) parts = append(parts, part)
}
log.Error("Failed to parse content part type=%s: %v", part.Type, err)
continue continue
} }
parts = append(parts, parsedPart) parts = append(parts, parsedPart)

View file

@ -12,6 +12,7 @@ import (
"github.com/yaoapp/yao/agent/output/message" "github.com/yaoapp/yao/agent/output/message"
searchTypes "github.com/yaoapp/yao/agent/search/types" searchTypes "github.com/yaoapp/yao/agent/search/types"
"github.com/yaoapp/yao/attachment" "github.com/yaoapp/yao/attachment"
"github.com/yaoapp/yao/tools/vision"
) )
// Image handles image content // Image handles image content
@ -65,24 +66,27 @@ func (h *Image) Parse(ctx *agentContext.Context, content agentContext.ContentPar
return h.base64(ctx, content, visionFormat) return h.base64(ctx, content, visionFormat)
} }
// Model doesn't support vision - check cache first, then use vision agent/MCP // Model doesn't support vision - fallback chain:
// Try to get cached text (from attachment's content_preview) // 1. Cache -> 2. Uses.Vision (explicit config) -> 3. tools/vision (auto) -> 4. Placeholder text
cachedText, found, err := h.readFromCache(ctx, content.ImageURL.URL) cachedText, found, err := h.readFromCache(ctx, content.ImageURL.URL)
if err == nil && found { if err == nil && found {
// Cache hit! Return as text content
return agentContext.ContentPart{ return agentContext.ContentPart{
Type: agentContext.ContentText, Type: agentContext.ContentText,
Text: cachedText, Text: cachedText,
}, nil, nil }, nil, nil
} }
// No cache, try to use vision agent/MCP
if h.options.CompletionOptions != nil && h.options.CompletionOptions.Uses != nil && h.options.CompletionOptions.Uses.Vision != "" { if h.options.CompletionOptions != nil && h.options.CompletionOptions.Uses != nil && h.options.CompletionOptions.Uses.Vision != "" {
return h.agent(ctx, content) return h.agent(ctx, content)
} }
// No vision support and no vision tool specified, return error if text, err := h.readImageWithTools(ctx, content); err == nil {
return content, nil, fmt.Errorf("model doesn't support vision and no vision tool specified in uses.Vision") h.saveToCache(ctx, content.ImageURL.URL, text)
return agentContext.ContentPart{Type: agentContext.ContentText, Text: text}, nil, nil
}
return agentContext.ContentPart{Type: agentContext.ContentText, Text: "[Image content - vision model not available]"}, nil, nil
} }
// base64 encodes image content to base64 (for vision support) // base64 encodes image content to base64 (for vision support)
@ -360,6 +364,37 @@ func (h *Image) callMCPVisionTool(ctx *agentContext.Context, serverID string, co
return result, err return result, err
} }
// readImageWithTools calls tools/vision.ReadImage to convert image to text
// using a vision-capable model resolved via llmprovider.
func (h *Image) readImageWithTools(ctx *agentContext.Context, content agentContext.ContentPart) (string, error) {
if ctx.Authorized == nil {
return "", fmt.Errorf("no auth info available for vision model resolution")
}
src := wrapperToAttachURI(content.ImageURL.URL)
loadingID := h.sendLoading(ctx, i18n.T(ctx.Locale, "content.image.analyzing"))
resp, err := vision.ReadImage(ctx.Context, src, "Please describe this image in detail.", 1080, ctx.Authorized)
h.sendLoadingDone(ctx, loadingID)
if err != nil {
return "", err
}
return resp.Content, nil
}
// wrapperToAttachURI converts __uploader://fileID to attach://uploader/fileID
// format expected by tools/vision.readBytes.
func wrapperToAttachURI(url string) string {
uploaderName, fileID, ok := attachment.Parse(url)
if !ok {
return url
}
return "attach://" + uploaderName + "/" + fileID
}
// sendLoading sends a loading message and returns the message ID // sendLoading sends a loading message and returns the message ID
// Returns empty string if SilentLoading is enabled // Returns empty string if SilentLoading is enabled
func (h *Image) sendLoading(ctx *agentContext.Context, msg string) string { func (h *Image) sendLoading(ctx *agentContext.Context, msg string) string {

View file

@ -117,11 +117,12 @@ func TestParseWithoutVisionSupport(t *testing.T) {
} }
handler := image.New(options) handler := image.New(options)
_, _, err := handler.Parse(ctx, content) result, _, err := handler.Parse(ctx, content)
// Should return error because no vision support and no vision tool specified // Should return placeholder text (no error) when no vision support
assert.Error(t, err) assert.NoError(t, err)
assert.Contains(t, err.Error(), "no vision tool specified") assert.Equal(t, agentContext.ContentText, result.Type)
assert.Contains(t, result.Text, "Image content")
} }
// TestParseWithEmptyURL tests parsing image with empty URL // TestParseWithEmptyURL tests parsing image with empty URL

View file

@ -53,19 +53,28 @@ func (a *VisionAdapter) removeImageContent(messages []context.Message) []context
for _, msg := range messages { for _, msg := range messages {
processedMsg := msg processedMsg := msg
// Handle multimodal content (array of map) if contentParts, ok := msg.Content.([]context.ContentPart); ok {
if contentParts, ok := msg.Content.([]map[string]interface{}); ok { filtered := make([]context.ContentPart, 0)
for _, part := range contentParts {
if part.Type != context.ContentImageURL {
filtered = append(filtered, part)
}
}
if len(filtered) == 0 {
processedMsg.Content = "[Image content not supported by this model]"
} else {
processedMsg.Content = filtered
}
} else if contentParts, ok := msg.Content.([]map[string]interface{}); ok {
filteredParts := make([]map[string]interface{}, 0) filteredParts := make([]map[string]interface{}, 0)
for _, part := range contentParts { for _, part := range contentParts {
partType, _ := part["type"].(string) partType, _ := part["type"].(string)
// Skip image content
if partType != "image_url" && partType != "image" { if partType != "image_url" && partType != "image" {
filteredParts = append(filteredParts, part) filteredParts = append(filteredParts, part)
} }
} }
// If all parts were filtered out, add placeholder text
if len(filteredParts) == 0 { if len(filteredParts) == 0 {
processedMsg.Content = "[Image content not supported by this model]" processedMsg.Content = "[Image content not supported by this model]"
} else if len(filteredParts) == 1 { } else if len(filteredParts) == 1 {

View file

@ -45,11 +45,11 @@ func buildAdapters(cap *goullm.Capabilities) []adapters.CapabilityAdapter {
// Tool call adapter // Tool call adapter
result = append(result, adapters.NewToolCallAdapter(cap.ToolCalls)) result = append(result, adapters.NewToolCallAdapter(cap.ToolCalls))
// Vision adapter // Vision adapter (always registered to strip unsupported image content)
visionSupport, visionFormat := context.GetVisionSupport(cap) visionSupport, visionFormat := context.GetVisionSupport(cap)
if visionSupport { if visionSupport {
result = append(result, adapters.NewVisionAdapter(true, visionFormat)) result = append(result, adapters.NewVisionAdapter(true, visionFormat))
} else if cap.Vision != nil { } else {
result = append(result, adapters.NewVisionAdapter(false, context.VisionFormatNone)) result = append(result, adapters.NewVisionAdapter(false, context.VisionFormatNone))
} }

View file

@ -156,12 +156,11 @@ func buildAdapters(cap *goullm.Capabilities) []adapters.CapabilityAdapter {
// Tool call adapter // Tool call adapter
result = append(result, adapters.NewToolCallAdapter(cap.ToolCalls)) result = append(result, adapters.NewToolCallAdapter(cap.ToolCalls))
// Vision adapter // Vision adapter (always registered to strip unsupported image content)
visionSupport, visionFormat := context.GetVisionSupport(cap) visionSupport, visionFormat := context.GetVisionSupport(cap)
if visionSupport { if visionSupport {
result = append(result, adapters.NewVisionAdapter(true, visionFormat)) result = append(result, adapters.NewVisionAdapter(true, visionFormat))
} else if cap.Vision != nil { } else {
// Vision explicitly disabled, add adapter to remove image content
result = append(result, adapters.NewVisionAdapter(false, context.VisionFormatNone)) result = append(result, adapters.NewVisionAdapter(false, context.VisionFormatNone))
} }

View file

@ -77,6 +77,9 @@ func (r *Runner) buildCommand(ctx context.Context, req *types.StreamRequest, p p
var systemPrompt string var systemPrompt string
envPrompt := buildSandboxEnvPrompt(p, workDir) envPrompt := buildSandboxEnvPrompt(p, workDir)
if capPrompt := buildModelCapabilityPrompt(req); capPrompt != "" {
envPrompt += "\n\n" + capPrompt
}
if !isContinuation && req.SystemPrompt != "" { if !isContinuation && req.SystemPrompt != "" {
systemPrompt = req.SystemPrompt + "\n\n" + envPrompt systemPrompt = req.SystemPrompt + "\n\n" + envPrompt
} else if !isContinuation { } else if !isContinuation {
@ -310,6 +313,142 @@ func buildSandboxEnvPrompt(p platform, workDir string) string {
%[4]s`, workDir, osName, shell, shellNote) %[4]s`, workDir, osName, shell, shellNote)
} }
func buildModelCapabilityPrompt(req *types.StreamRequest) string {
if req.Connector == nil {
return ""
}
lc, ok := req.Connector.(goullm.LLMConnector)
if !ok {
return ""
}
primaryModel := lc.GetModel()
if primaryModel == "" {
return ""
}
primaryCaps := lc.GetCapabilities()
type tierInfo struct {
tier string
alias string
model string
caps *goullm.Capabilities
conn connector.Connector
}
tiers := []tierInfo{
{tier: "Default", alias: "sonnet", model: primaryModel, caps: primaryCaps, conn: req.Connector},
}
hasDifferentTier := false
if rc, exists := req.Roles["heavy"]; exists && rc != nil {
m := connectorModel(rc)
if m != "" {
caps := connectorCaps(rc)
tiers = append(tiers, tierInfo{tier: "Heavy", alias: "opus", model: m, caps: caps, conn: rc})
if m != primaryModel {
hasDifferentTier = true
}
}
}
if rc, exists := req.Roles["light"]; exists && rc != nil {
m := connectorModel(rc)
if m != "" {
caps := connectorCaps(rc)
tiers = append(tiers, tierInfo{tier: "Light", alias: "haiku", model: m, caps: caps, conn: rc})
if m != primaryModel {
hasDifferentTier = true
}
}
}
var sb strings.Builder
sb.WriteString("## Model Capabilities\n\n")
sb.WriteString(fmt.Sprintf("Your current model: `%s`\n", primaryModel))
if hasDifferentTier {
sb.WriteString("\n### Available Model Tiers\n\n")
sb.WriteString("| Tier | Alias | Model | Capabilities |\n")
sb.WriteString("| ---- | ----- | ----- | ------------ |\n")
for _, t := range tiers {
capList := formatCapabilities(t.caps, t.conn)
sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s |\n", t.tier, t.alias, t.model, capList))
}
}
var guidance []string
if hasDifferentTier {
guidance = append(guidance,
"For complex reasoning, multi-step analysis, or tasks requiring deep thought, delegate to a sub-agent with `model: \"opus\"`",
"For simple tasks (formatting, translation, summarization), use `model: \"haiku\"` for faster responses",
)
}
primaryHasVision := primaryCaps.HasVision()
if !primaryHasVision {
if _, hasVisionRole := req.Roles["vision"]; hasVisionRole {
guidance = append(guidance,
"**Image/Vision**: Your current model cannot process images directly. Use the `image_read` system tool (`tai tool image_read`) to analyze images — see the yao-vision skill for details",
)
}
}
if len(guidance) > 0 {
sb.WriteString("\n### Usage Guidance\n\n")
for _, g := range guidance {
sb.WriteString("- " + g + "\n")
}
}
if !hasDifferentTier && len(guidance) == 0 {
return ""
}
return sb.String()
}
func connectorModel(c connector.Connector) string {
if lc, ok := c.(goullm.LLMConnector); ok {
if m := lc.GetModel(); m != "" {
return m
}
}
m, _ := c.Setting()["model"].(string)
return m
}
func connectorCaps(c connector.Connector) *goullm.Capabilities {
if lc, ok := c.(goullm.LLMConnector); ok {
return lc.GetCapabilities()
}
return nil
}
func formatCapabilities(caps *goullm.Capabilities, conn connector.Connector) string {
var parts []string
hasThinking := caps.HasReasoning()
if !hasThinking && conn != nil {
if thinking, ok := conn.Setting()["thinking"].(map[string]interface{}); ok {
if t, _ := thinking["type"].(string); t == "enabled" {
hasThinking = true
}
}
}
if hasThinking {
parts = append(parts, "thinking")
}
if caps.HasVision() {
parts = append(parts, "vision")
}
if caps.HasToolCalls() {
parts = append(parts, "tool_calls")
}
if len(parts) == 0 {
return "-"
}
return strings.Join(parts, ", ")
}
func hasExistingSession(ctx context.Context, computer infra.Computer, p platform, assistantID string) bool { func hasExistingSession(ctx context.Context, computer infra.Computer, p platform, assistantID string) bool {
workDir := computer.GetWorkDir() workDir := computer.GetWorkDir()
var sessionDir string var sessionDir string

View file

@ -9,6 +9,7 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/yaoapp/gou/connector" "github.com/yaoapp/gou/connector"
goullm "github.com/yaoapp/gou/llm"
gouTypes "github.com/yaoapp/gou/types" gouTypes "github.com/yaoapp/gou/types"
"github.com/yaoapp/xun/dbal/query" "github.com/yaoapp/xun/dbal/query"
"github.com/yaoapp/xun/dbal/schema" "github.com/yaoapp/xun/dbal/schema"
@ -701,3 +702,155 @@ func TestBuildEnv_Anthropic_MultiConnector_Incompatible(t *testing.T) {
assert.Equal(t, "claude-sonnet-4-20250514", env["ANTHROPIC_DEFAULT_OPUS_MODEL"], assert.Equal(t, "claude-sonnet-4-20250514", env["ANTHROPIC_DEFAULT_OPUS_MODEL"],
"incompatible connector: heavy should keep primary model (different host, no anthropic protocol)") "incompatible connector: heavy should keep primary model (different host, no anthropic protocol)")
} }
// --- fakeLLMConnector implements both connector.Connector and goullm.LLMConnector ---
type fakeLLMConnector struct {
fakeConnector
model string
caps *goullm.Capabilities
}
func (f *fakeLLMConnector) GetAuthMode() goullm.AuthMode { return goullm.AuthBearer }
func (f *fakeLLMConnector) GetURL() string { return "" }
func (f *fakeLLMConnector) GetKey() string { return "" }
func (f *fakeLLMConnector) GetModel() string { return f.model }
func (f *fakeLLMConnector) GetSupportedParams() map[string]*goullm.ParamSpec {
return nil
}
func (f *fakeLLMConnector) GetCapabilities() *goullm.Capabilities { return f.caps }
// --- buildModelCapabilityPrompt tests ---
func TestBuildModelCapabilityPrompt_NilConnector(t *testing.T) {
req := &types.StreamRequest{Config: &types.SandboxConfig{}}
result := buildModelCapabilityPrompt(req)
assert.Empty(t, result)
}
func TestBuildModelCapabilityPrompt_NoRoles_NoSpecialCaps(t *testing.T) {
primary := &fakeLLMConnector{
fakeConnector: fakeConnector{id: "test", typ: connector.OPENAI, settings: map[string]interface{}{"model": "deepseek-v4-flash"}},
model: "deepseek-v4-flash",
caps: &goullm.Capabilities{ToolCalls: true, Streaming: true},
}
req := &types.StreamRequest{
Config: &types.SandboxConfig{},
Connector: primary,
}
result := buildModelCapabilityPrompt(req)
assert.Empty(t, result, "no roles and no special caps → empty")
}
func TestBuildModelCapabilityPrompt_WithHeavyAndLight(t *testing.T) {
primary := &fakeLLMConnector{
fakeConnector: fakeConnector{id: "ds-flash", typ: connector.OPENAI, settings: map[string]interface{}{"model": "deepseek-v4-flash"}},
model: "deepseek-v4-flash",
caps: &goullm.Capabilities{ToolCalls: true},
}
heavy := &fakeLLMConnector{
fakeConnector: fakeConnector{id: "ds-pro", typ: connector.OPENAI, settings: map[string]interface{}{"model": "deepseek-v4-pro"}},
model: "deepseek-v4-pro",
caps: &goullm.Capabilities{Reasoning: true, ToolCalls: true},
}
light := &fakeLLMConnector{
fakeConnector: fakeConnector{id: "ds-lite", typ: connector.OPENAI, settings: map[string]interface{}{"model": "deepseek-v4-flash"}},
model: "deepseek-v4-flash",
caps: &goullm.Capabilities{ToolCalls: true},
}
req := &types.StreamRequest{
Config: &types.SandboxConfig{},
Connector: primary,
Roles: map[string]connector.Connector{
"default": primary,
"heavy": heavy,
"light": light,
},
}
result := buildModelCapabilityPrompt(req)
assert.Contains(t, result, "deepseek-v4-flash")
assert.Contains(t, result, "deepseek-v4-pro")
assert.Contains(t, result, "opus")
assert.Contains(t, result, "haiku")
assert.Contains(t, result, "thinking")
assert.Contains(t, result, "tool_calls")
assert.Contains(t, result, "sub-agent")
}
func TestBuildModelCapabilityPrompt_VisionGuidance(t *testing.T) {
primary := &fakeLLMConnector{
fakeConnector: fakeConnector{id: "ds-flash", typ: connector.OPENAI, settings: map[string]interface{}{"model": "deepseek-v4-flash"}},
model: "deepseek-v4-flash",
caps: &goullm.Capabilities{ToolCalls: true},
}
visionConn := &fakeLLMConnector{
fakeConnector: fakeConnector{id: "vision", typ: connector.OPENAI, settings: map[string]interface{}{"model": "gpt-4o"}},
model: "gpt-4o",
caps: &goullm.Capabilities{Vision: true, ToolCalls: true},
}
req := &types.StreamRequest{
Config: &types.SandboxConfig{},
Connector: primary,
Roles: map[string]connector.Connector{
"default": primary,
"vision": visionConn,
},
}
result := buildModelCapabilityPrompt(req)
assert.Contains(t, result, "image_read")
assert.Contains(t, result, "Image/Vision")
}
func TestBuildModelCapabilityPrompt_PrimaryHasVision_NoGuidance(t *testing.T) {
primary := &fakeLLMConnector{
fakeConnector: fakeConnector{id: "gpt4o", typ: connector.OPENAI, settings: map[string]interface{}{"model": "gpt-4o"}},
model: "gpt-4o",
caps: &goullm.Capabilities{Vision: true, ToolCalls: true},
}
req := &types.StreamRequest{
Config: &types.SandboxConfig{},
Connector: primary,
Roles: map[string]connector.Connector{
"default": primary,
"vision": primary,
},
}
result := buildModelCapabilityPrompt(req)
assert.NotContains(t, result, "image_read", "should not suggest image_read when primary has vision")
}
func TestBuildModelCapabilityPrompt_ThinkingFromSettings(t *testing.T) {
primary := &fakeLLMConnector{
fakeConnector: fakeConnector{
id: "ds-pro",
typ: connector.OPENAI,
settings: map[string]interface{}{
"model": "deepseek-v4-pro",
"thinking": map[string]interface{}{
"type": "enabled",
},
},
},
model: "deepseek-v4-pro",
caps: &goullm.Capabilities{ToolCalls: true},
}
heavy := &fakeLLMConnector{
fakeConnector: fakeConnector{id: "ds-pro2", typ: connector.OPENAI, settings: map[string]interface{}{"model": "deepseek-v4-pro-max"}},
model: "deepseek-v4-pro-max",
caps: &goullm.Capabilities{ToolCalls: true},
}
req := &types.StreamRequest{
Config: &types.SandboxConfig{},
Connector: primary,
Roles: map[string]connector.Connector{
"default": primary,
"heavy": heavy,
},
}
result := buildModelCapabilityPrompt(req)
assert.Contains(t, result, "thinking", "should detect thinking from Setting()[\"thinking\"]")
}

View file

@ -48,8 +48,9 @@ func InjectSystemSkills(ws writerFS, skills fs.FS, targetDir string) error {
}) })
} }
// AppendSystemPrompt appends content to a file in the workspace with an // AppendSystemPrompt injects content into a file in the workspace using an
// idempotent marker. If the marker already exists the call is a no-op. // idempotent marker. If the marker already exists, the injected section is
// replaced with the new content (so updates propagate to existing sandboxes).
// If the file does not exist it is created with just the marker + content. // If the file does not exist it is created with just the marker + content.
func AppendSystemPrompt(ws writerFS, filename string, content []byte) error { func AppendSystemPrompt(ws writerFS, filename string, content []byte) error {
existing, err := ws.ReadFile(filename) existing, err := ws.ReadFile(filename)
@ -61,8 +62,15 @@ func AppendSystemPrompt(ws writerFS, filename string, content []byte) error {
return ws.WriteFile(filename, append(header, content...), 0644) return ws.WriteFile(filename, append(header, content...), 0644)
} }
if bytes.Contains(existing, []byte(systemToolsMarker)) { idx := bytes.Index(existing, []byte(systemToolsMarker))
return nil if idx >= 0 {
injected := append([]byte(systemToolsMarker+"\n\n"), content...)
prefix := existing[:idx]
merged := append(bytes.TrimRight(prefix, "\n\r\t "), []byte("\n\n---\n\n")...)
if idx == 0 {
merged = nil
}
return ws.WriteFile(filename, append(merged, injected...), 0644)
} }
separator := []byte("\n\n---\n\n" + systemToolsMarker + "\n\n") separator := []byte("\n\n---\n\n" + systemToolsMarker + "\n\n")

View file

@ -106,6 +106,67 @@ func TestAppendSystemPrompt_Idempotent(t *testing.T) {
} }
} }
func TestAppendSystemPrompt_UpdatesExistingContent(t *testing.T) {
dir := t.TempDir()
ws := newDirFS(dir)
oldContent := []byte("## Old Tools\nweb_search only\n")
if err := AppendSystemPrompt(ws, "CLAUDE.md", oldContent); err != nil {
t.Fatalf("first call: %v", err)
}
newContent := []byte("## Updated Tools\nweb_search + image_read\n")
if err := AppendSystemPrompt(ws, "CLAUDE.md", newContent); err != nil {
t.Fatalf("second call: %v", err)
}
data, _ := os.ReadFile(filepath.Join(dir, "CLAUDE.md"))
got := string(data)
assertContains(t, got, "image_read")
assertContains(t, got, systemToolsMarker)
if containsStr(got, "Old Tools") {
t.Error("old content should have been replaced")
}
}
func TestAppendSystemPrompt_UpdatesPreservesUserContent(t *testing.T) {
dir := t.TempDir()
ws := newDirFS(dir)
userContent := []byte("# My Project\n\nUser notes.\n")
if err := os.WriteFile(filepath.Join(dir, "CLAUDE.md"), userContent, 0644); err != nil {
t.Fatal(err)
}
oldContent := []byte("## Old Tools\n")
if err := AppendSystemPrompt(ws, "CLAUDE.md", oldContent); err != nil {
t.Fatal(err)
}
newContent := []byte("## Updated Tools\nimage_read added\n")
if err := AppendSystemPrompt(ws, "CLAUDE.md", newContent); err != nil {
t.Fatal(err)
}
data, _ := os.ReadFile(filepath.Join(dir, "CLAUDE.md"))
got := string(data)
assertContains(t, got, "My Project")
assertContains(t, got, "User notes")
assertContains(t, got, "image_read")
if containsStr(got, "Old Tools") {
t.Error("old injected content should have been replaced")
}
}
func containsStr(s, sub string) bool {
for i := 0; i <= len(s)-len(sub); i++ {
if s[i:i+len(sub)] == sub {
return true
}
}
return false
}
func assertContains(t *testing.T, s, sub string) { func assertContains(t *testing.T, s, sub string) {
t.Helper() t.Helper()
if len(s) < len(sub) { if len(s) < len(sub) {

2
go.mod
View file

@ -45,6 +45,7 @@ require (
github.com/yaoapp/xun v0.9.0 github.com/yaoapp/xun v0.9.0
go.mongodb.org/mongo-driver v1.17.3 go.mongodb.org/mongo-driver v1.17.3
golang.org/x/crypto v0.49.0 golang.org/x/crypto v0.49.0
golang.org/x/image v0.38.0
golang.org/x/net v0.52.0 golang.org/x/net v0.52.0
golang.org/x/sys v0.42.0 golang.org/x/sys v0.42.0
golang.org/x/text v0.35.0 golang.org/x/text v0.35.0
@ -211,7 +212,6 @@ require (
go.yaml.in/yaml/v3 v3.0.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/arch v0.17.0 // indirect golang.org/x/arch v0.17.0 // indirect
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
golang.org/x/image v0.38.0 // indirect
golang.org/x/mod v0.33.0 // indirect golang.org/x/mod v0.33.0 // indirect
golang.org/x/oauth2 v0.35.0 // indirect golang.org/x/oauth2 v0.35.0 // indirect
golang.org/x/sync v0.20.0 // indirect golang.org/x/sync v0.20.0 // indirect

8
tools/mcps/vision.json Normal file
View file

@ -0,0 +1,8 @@
{
"name": "yao-vision",
"transport": "process",
"description": "Vision tools for image understanding",
"tools": {
"image_read": "tools.image_read"
}
}

View file

@ -26,6 +26,14 @@ These environment variables are set by the Yao sandbox. **Always use these varia
User-uploaded files are placed in `$WORKDIR/.attachments/{chatID}/`. User-uploaded files are placed in `$WORKDIR/.attachments/{chatID}/`.
When the user references an attached file, read it from this directory. When the user references an attached file, read it from this directory.
### Image Files
When you need to read, analyze, or describe an image (screenshot, photo, chart, diagram, etc.), **always use `image_read`** instead of trying to read binary files directly. The tool sends the image to a vision model and returns a text description.
```bash
tai tool image_read '{"image_path": "<file_path_or_url>", "prompt": "describe this image"}'
```
## Yao System Tools ## Yao System Tools
You have access to Yao system tools via the `tai` command in bash. You have access to Yao system tools via the `tai` command in bash.
@ -41,5 +49,6 @@ You have access to Yao system tools via the `tai` command in bash.
| `doc_list` | yao-doc | Search/list available process documentation | | `doc_list` | yao-doc | Search/list available process documentation |
| `doc_inspect` | yao-doc | Get detailed docs for a specific process | | `doc_inspect` | yao-doc | Get detailed docs for a specific process |
| `doc_validate` | yao-doc | Validate a process name and get suggestions | | `doc_validate` | yao-doc | Validate a process name and get suggestions |
| `image_read` | yao-vision | Read and analyze images using a vision model |
The three system skills (`yao-web`, `yao-process`, `yao-doc`) in `$HOME/.claude/skills/` are **auto-discovered** — they contain detailed parameter docs and workflow guidance. You do not need to manually read them; they are loaded automatically when your task matches their description. The system skills (`yao-web`, `yao-process`, `yao-doc`, `yao-vision`) in `$HOME/.claude/skills/` are **auto-discovered** — they contain detailed parameter docs and workflow guidance. You do not need to manually read them; they are loaded automatically when your task matches their description.

View file

@ -0,0 +1,46 @@
---
name: yao-vision
description: Image understanding expert. ALWAYS invoke this skill when you need to read, analyze, or describe an image that you cannot process directly. Use for screenshots, photos, charts, diagrams, or any visual content.
---
# Vision Tools
Use when you encounter images you cannot read natively (e.g., as a text-only model).
## image_read
Send an image to a vision-capable model and get a text description.
### Local file (most common):
```bash
tai tool image_read '{"image_path": "/path/to/image.png", "prompt": "Describe this image"}'
```
### URL:
```bash
tai tool image_read '{"image_path": "https://example.com/photo.jpg", "prompt": "What is shown?"}'
```
### Cross-workspace file:
```bash
tai tool image_read '{"image_path": "workspace://ws-id/path/to/image.png", "prompt": "Analyze"}'
```
### Attachment file:
```bash
tai tool image_read '{"image_path": "attach://__yao.attachment/file-id-123", "prompt": "Describe"}'
```
### Yao data file:
```bash
tai tool image_read '{"image_path": "yao://uploads/photo.jpg", "prompt": "Describe"}'
```
| Parameter | Type | Required | Description |
| ---------- | ------- | -------- | --------------------------------------------------------------- |
| image_path | string | yes | File path, URL, workspace://, attach://, or yao:// URI |
| prompt | string | no | Analysis instruction (default: describe in detail) |
| max_size | integer | no | Max dimension in pixels for longest edge (default: 1080) |
Images are automatically resized (preserving aspect ratio) before sending to the vision model.
Supported formats: PNG, JPEG, GIF, WebP.

View file

@ -10,6 +10,7 @@ import (
"github.com/yaoapp/kun/log" "github.com/yaoapp/kun/log"
"github.com/yaoapp/yao/tools/docs" "github.com/yaoapp/yao/tools/docs"
"github.com/yaoapp/yao/tools/proc" "github.com/yaoapp/yao/tools/proc"
"github.com/yaoapp/yao/tools/vision"
"github.com/yaoapp/yao/tools/webfetch" "github.com/yaoapp/yao/tools/webfetch"
"github.com/yaoapp/yao/tools/websearch" "github.com/yaoapp/yao/tools/websearch"
) )
@ -23,6 +24,9 @@ var mcpProcessDSL []byte
//go:embed mcps/doc.json //go:embed mcps/doc.json
var mcpDocDSL []byte var mcpDocDSL []byte
//go:embed mcps/vision.json
var mcpVisionDSL []byte
func init() { func init() {
process.RegisterGroup("tools", map[string]process.Handler{ process.RegisterGroup("tools", map[string]process.Handler{
"web_search": websearch.Handler, "web_search": websearch.Handler,
@ -32,6 +36,7 @@ func init() {
"doc_list": docs.ListHandler, "doc_list": docs.ListHandler,
"doc_inspect": docs.InspectHandler, "doc_inspect": docs.InspectHandler,
"doc_validate": docs.ValidateHandler, "doc_validate": docs.ValidateHandler,
"image_read": vision.Handler,
}) })
registerMCPServer(mcpWebDSL, "yao-web", registerMCPServer(mcpWebDSL, "yao-web",
@ -40,6 +45,8 @@ func init() {
proc.SchemaJSON, proc.AllowedSchemaJSON) proc.SchemaJSON, proc.AllowedSchemaJSON)
registerMCPServer(mcpDocDSL, "yao-doc", registerMCPServer(mcpDocDSL, "yao-doc",
docs.ListSchemaJSON, docs.InspectSchemaJSON, docs.ValidateSchemaJSON) docs.ListSchemaJSON, docs.InspectSchemaJSON, docs.ValidateSchemaJSON)
registerMCPServer(mcpVisionDSL, "yao-vision",
vision.SchemaJSON)
} }
func registerMCPServer(dsl []byte, id string, schemas ...[]byte) { func registerMCPServer(dsl []byte, id string, schemas ...[]byte) {

26
tools/vision/schema.json Normal file
View file

@ -0,0 +1,26 @@
{
"name": "image_read",
"description": "Read and analyze an image using a vision-capable model.",
"process": "tools.image_read",
"inputSchema": {
"type": "object",
"properties": {
"image_path": {
"type": "string",
"description": "Path or URI of the image to analyze. Accepts: local file path, URL (http/https), workspace://, attach://, yao://, or data URI."
},
"prompt": {
"type": "string",
"description": "What to analyze about the image (default: describe in detail)",
"default": "Please describe this image in detail."
},
"max_size": {
"type": "integer",
"description": "Max dimension in pixels for the longest edge. Image is resized (preserving aspect ratio) before sending to the vision model. Default 1080.",
"default": 1080
}
},
"required": ["image_path"]
},
"x-process-args": ["$args.image_path", "$args.prompt", "$args.max_size"]
}

237
tools/vision/vision.go Normal file
View file

@ -0,0 +1,237 @@
package vision
import (
"bytes"
"context"
_ "embed"
"encoding/base64"
"fmt"
"image"
_ "image/gif"
"image/jpeg"
_ "image/png"
"io"
"net/http"
"strings"
"time"
"golang.org/x/image/draw"
_ "golang.org/x/image/webp"
goufs "github.com/yaoapp/gou/fs"
"github.com/yaoapp/gou/process"
agentCtx "github.com/yaoapp/yao/agent/context"
agentLLM "github.com/yaoapp/yao/agent/llm"
"github.com/yaoapp/yao/attachment"
"github.com/yaoapp/yao/openapi/oauth/authorized"
oauthTypes "github.com/yaoapp/yao/openapi/oauth/types"
ws "github.com/yaoapp/yao/workspace"
)
//go:embed schema.json
var SchemaJSON []byte
// ImageReadResponse is the return type for image_read.
type ImageReadResponse struct {
Content string `json:"content"`
Model string `json:"model"`
}
// ReadImage reads and analyzes an image using the vision model.
// It resolves the image from src, finds a vision-capable connector via llmprovider,
// and returns the model's text description.
func ReadImage(goCtx context.Context, src string, prompt string, maxSize int, authInfo *oauthTypes.AuthorizedInfo) (*ImageReadResponse, error) {
if prompt == "" {
prompt = "Please describe this image in detail."
}
if maxSize <= 0 {
maxSize = 1080
}
imageURI, err := resolveImage(src, maxSize)
if err != nil {
return nil, fmt.Errorf("resolve image: %w", err)
}
conn, caps, err := agentLLM.ResolveConnector("use::vision", authInfo)
if err != nil {
return nil, fmt.Errorf("resolve vision connector: %w", err)
}
opts := &agentCtx.CompletionOptions{Capabilities: caps}
instance, err := agentLLM.New(conn, opts)
if err != nil {
return nil, fmt.Errorf("create LLM instance: %w", err)
}
messages := []agentCtx.Message{{
Role: "user",
Content: []agentCtx.ContentPart{
{Type: agentCtx.ContentImageURL, ImageURL: &agentCtx.ImageURL{URL: imageURI}},
{Type: agentCtx.ContentText, Text: prompt},
},
}}
chatID := agentCtx.GenChatID()
ctx := agentCtx.New(goCtx, authInfo, chatID)
defer ctx.Release()
resp, err := instance.Post(ctx, messages, opts)
if err != nil {
return nil, fmt.Errorf("vision model call: %w", err)
}
return &ImageReadResponse{
Content: extractTextContent(resp.Content),
Model: resp.Model,
}, nil
}
// Handler is the tools.image_read process handler.
func Handler(proc *process.Process) interface{} {
src := proc.ArgsString(0)
if src == "" {
return map[string]interface{}{"error": "image_path is required: provide a file path, URL, or URI"}
}
authInfo := authorized.ProcessAuthInfo(proc)
if authInfo == nil {
return map[string]interface{}{"error": "unauthorized: no auth info in request"}
}
resp, err := ReadImage(proc.Context, src,
proc.ArgsString(1, "Please describe this image in detail."),
proc.ArgsInt(2, 1080),
authInfo,
)
if err != nil {
return map[string]interface{}{"error": err.Error()}
}
return resp
}
// resolveImage reads image bytes from any supported source, resizes, and returns a data URI.
func resolveImage(src string, maxSize int) (string, error) {
raw, err := readBytes(src)
if err != nil {
return "", err
}
data, mime := resizeImage(raw, maxSize)
if len(data) > 3<<20 {
return "", fmt.Errorf("compressed image still exceeds 3MB, try a smaller max_size (current: %d)", maxSize)
}
return "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data), nil
}
// readBytes dispatches to the appropriate reader based on URI scheme.
func readBytes(src string) ([]byte, error) {
switch {
case strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://"):
return httpGet(src)
case strings.HasPrefix(src, "workspace://"):
rest := strings.TrimPrefix(src, "workspace://")
idx := strings.Index(rest, "/")
if idx < 0 {
return nil, fmt.Errorf("invalid workspace URI: %s", src)
}
return ws.M().ReadFile(context.Background(), rest[:idx], rest[idx+1:])
case strings.HasPrefix(src, "attach://"):
rest := strings.TrimPrefix(src, "attach://")
idx := strings.Index(rest, "/")
if idx < 0 {
return nil, fmt.Errorf("invalid attach URI: %s", src)
}
mgr, ok := attachment.Managers[rest[:idx]]
if !ok {
return nil, fmt.Errorf("uploader not found: %s", rest[:idx])
}
return mgr.Read(context.Background(), rest[idx+1:])
case strings.HasPrefix(src, "yao://"):
dataFS, err := goufs.Get("data")
if err != nil {
return nil, err
}
return dataFS.ReadFile(strings.TrimPrefix(src, "yao://"))
case strings.HasPrefix(src, "data:"):
return decodeDataURI(src)
default:
return nil, fmt.Errorf("unsupported image source: %s", src)
}
}
func httpGet(rawURL string) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil)
if err != nil {
return nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("fetch %s: %d", rawURL, resp.StatusCode)
}
return io.ReadAll(io.LimitReader(resp.Body, 20<<20))
}
// resizeImage decodes, resizes (longest edge <= maxSize), re-encodes as JPEG.
// Returns original bytes unchanged when already small enough or if decode fails.
func resizeImage(data []byte, maxSize int) ([]byte, string) {
img, _, err := image.Decode(bytes.NewReader(data))
if err != nil {
return data, http.DetectContentType(data)
}
bounds := img.Bounds()
w, h := bounds.Dx(), bounds.Dy()
if w <= maxSize && h <= maxSize {
return data, http.DetectContentType(data)
}
ratio := float64(maxSize) / float64(max(w, h))
newW, newH := int(float64(w)*ratio), int(float64(h)*ratio)
if newW < 1 {
newW = 1
}
if newH < 1 {
newH = 1
}
dst := image.NewRGBA(image.Rect(0, 0, newW, newH))
draw.BiLinear.Scale(dst, dst.Bounds(), img, bounds, draw.Over, nil)
var buf bytes.Buffer
if err := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: 85}); err != nil {
return data, http.DetectContentType(data)
}
return buf.Bytes(), "image/jpeg"
}
func decodeDataURI(uri string) ([]byte, error) {
parts := strings.SplitN(uri, ",", 2)
if len(parts) != 2 {
return nil, fmt.Errorf("invalid data URI format")
}
return base64.StdEncoding.DecodeString(parts[1])
}
func extractTextContent(content interface{}) string {
if s, ok := content.(string); ok {
return s
}
if parts, ok := content.([]agentCtx.ContentPart); ok {
for _, p := range parts {
if p.Type == agentCtx.ContentText {
return p.Text
}
}
}
return fmt.Sprintf("%v", content)
}

238
tools/vision/vision_test.go Normal file
View file

@ -0,0 +1,238 @@
package vision
import (
"encoding/base64"
"image"
"image/color"
_ "image/jpeg"
"image/png"
"net/http"
"net/http/httptest"
"strings"
"testing"
agentCtx "github.com/yaoapp/yao/agent/context"
)
func makePNG(w, h int) []byte {
img := image.NewRGBA(image.Rect(0, 0, w, h))
for y := 0; y < h; y++ {
for x := 0; x < w; x++ {
img.Set(x, y, color.RGBA{R: 255, G: 0, B: 0, A: 255})
}
}
var buf strings.Builder
if err := png.Encode(&buf, img); err != nil {
panic(err)
}
return []byte(buf.String())
}
func TestResizeImage_SmallImage(t *testing.T) {
pngData := makePNG(100, 80)
data, mime := resizeImage(pngData, 1080)
if mime != "image/png" {
t.Errorf("mime = %q, want image/png", mime)
}
if len(data) != len(pngData) {
t.Errorf("data length changed for small image")
}
}
func TestResizeImage_LargeImage(t *testing.T) {
pngData := makePNG(2000, 1500)
data, mime := resizeImage(pngData, 1080)
if mime != "image/jpeg" {
t.Errorf("mime = %q, want image/jpeg", mime)
}
img, _, err := image.Decode(strings.NewReader(string(data)))
if err != nil {
t.Fatalf("failed to decode resized image: %v", err)
}
b := img.Bounds()
if b.Dx() > 1080 || b.Dy() > 1080 {
t.Errorf("resized dimensions %dx%d exceed max 1080", b.Dx(), b.Dy())
}
if b.Dx() != 1080 {
t.Errorf("longest edge = %d, want 1080", b.Dx())
}
}
func TestResizeImage_InvalidData(t *testing.T) {
raw := []byte("not an image at all")
data, mime := resizeImage(raw, 1080)
if string(data) != string(raw) {
t.Error("should return original data on decode failure")
}
if mime == "" {
t.Error("mime should not be empty")
}
}
func TestResizeImage_ExactBoundary(t *testing.T) {
pngData := makePNG(1080, 720)
data, _ := resizeImage(pngData, 1080)
if len(data) != len(pngData) {
t.Error("image at exact max_size should not be re-encoded")
}
}
func TestDecodeDataURI(t *testing.T) {
original := []byte("hello world")
b64 := base64.StdEncoding.EncodeToString(original)
uri := "data:application/octet-stream;base64," + b64
data, err := decodeDataURI(uri)
if err != nil {
t.Fatal(err)
}
if string(data) != string(original) {
t.Errorf("decoded = %q, want %q", data, original)
}
}
func TestDecodeDataURI_Invalid(t *testing.T) {
_, err := decodeDataURI("not-a-data-uri")
if err == nil {
t.Error("expected error for invalid data URI")
}
}
func TestExtractTextContent_String(t *testing.T) {
result := extractTextContent("hello world")
if result != "hello world" {
t.Errorf("got %q, want %q", result, "hello world")
}
}
func TestExtractTextContent_ContentParts(t *testing.T) {
parts := []agentCtx.ContentPart{
{Type: agentCtx.ContentImageURL, ImageURL: &agentCtx.ImageURL{URL: "data:image/png;base64,abc"}},
{Type: agentCtx.ContentText, Text: "description of the image"},
}
result := extractTextContent(parts)
if result != "description of the image" {
t.Errorf("got %q, want %q", result, "description of the image")
}
}
func TestExtractTextContent_Fallback(t *testing.T) {
result := extractTextContent(12345)
if result != "12345" {
t.Errorf("got %q, want %q", result, "12345")
}
}
func TestReadBytes_HTTP(t *testing.T) {
pngData := makePNG(50, 50)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "image/png")
w.Write(pngData)
}))
defer srv.Close()
data, err := readBytes(srv.URL + "/test.png")
if err != nil {
t.Fatal(err)
}
if len(data) != len(pngData) {
t.Errorf("got %d bytes, want %d", len(data), len(pngData))
}
}
func TestReadBytes_HTTP404(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
}))
defer srv.Close()
_, err := readBytes(srv.URL + "/missing.png")
if err == nil {
t.Error("expected error for 404")
}
}
func TestReadBytes_DataURI(t *testing.T) {
original := []byte("test data")
b64 := base64.StdEncoding.EncodeToString(original)
uri := "data:application/octet-stream;base64," + b64
data, err := readBytes(uri)
if err != nil {
t.Fatal(err)
}
if string(data) != string(original) {
t.Errorf("got %q, want %q", data, original)
}
}
func TestReadBytes_UnsupportedScheme(t *testing.T) {
_, err := readBytes("ftp://example.com/file.png")
if err == nil {
t.Error("expected error for unsupported scheme")
}
if !strings.Contains(err.Error(), "unsupported") {
t.Errorf("error should mention unsupported: %v", err)
}
}
func TestResolveImage_DataURI(t *testing.T) {
pngData := makePNG(100, 80)
b64 := base64.StdEncoding.EncodeToString(pngData)
uri := "data:image/png;base64," + b64
result, err := resolveImage(uri, 1080)
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(result, "data:image/") {
t.Errorf("expected data URI, got %q", result[:min(50, len(result))])
}
}
func TestResolveImage_HTTP(t *testing.T) {
pngData := makePNG(200, 150)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "image/png")
w.Write(pngData)
}))
defer srv.Close()
result, err := resolveImage(srv.URL+"/img.png", 1080)
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(result, "data:image/") {
t.Errorf("expected data URI, got %q", result[:min(50, len(result))])
}
}
func TestHttpGet_Timeout(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Don't respond — let the client timeout
select {}
}))
defer srv.Close()
// Use a very short timeout for testing — override not possible with global func,
// but the 30s timeout should handle normally. Just verify it returns eventually.
// This test just verifies the function signature works.
_, err := readBytes("http://invalid.host.that.does.not.exist.example.com/img.png")
if err == nil {
t.Error("expected error for unreachable host")
}
}
func TestReadBytes_WorkspaceInvalid(t *testing.T) {
_, err := readBytes("workspace://no-slash-path")
if err == nil {
t.Error("expected error for invalid workspace URI")
}
}
func TestReadBytes_AttachInvalid(t *testing.T) {
_, err := readBytes("attach://no-slash-path")
if err == nil {
t.Error("expected error for invalid attach URI")
}
}