feat(agent): enhance image content handling and vision support
- Added support for processing image content in the agent, providing placeholder text when image parsing fails. - Implemented a fallback mechanism to utilize vision tools for image analysis, improving the handling of image content. - Updated tests to validate the new image handling behavior, ensuring correct responses when vision support is unavailable. - Enhanced system prompts to guide users on using the new `image_read` tool for image analysis, improving user experience.
This commit is contained in:
parent
222daaf645
commit
54607e26b7
18 changed files with 1010 additions and 26 deletions
|
|
@ -77,7 +77,15 @@ func parseContentParts(ctx *agentContext.Context, message agentContext.Message,
|
|||
for _, part := range content {
|
||||
parsedPart, refs, err := parseContentPart(ctx, part, options)
|
||||
if err != nil {
|
||||
if part.Type == agentContext.ContentImageURL {
|
||||
parts = append(parts, agentContext.ContentPart{
|
||||
Type: agentContext.ContentText,
|
||||
Text: "[Image content could not be processed]",
|
||||
})
|
||||
} else {
|
||||
parts = append(parts, part)
|
||||
}
|
||||
log.Error("Failed to parse content part type=%s: %v", part.Type, err)
|
||||
continue
|
||||
}
|
||||
parts = append(parts, parsedPart)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import (
|
|||
"github.com/yaoapp/yao/agent/output/message"
|
||||
searchTypes "github.com/yaoapp/yao/agent/search/types"
|
||||
"github.com/yaoapp/yao/attachment"
|
||||
"github.com/yaoapp/yao/tools/vision"
|
||||
)
|
||||
|
||||
// Image handles image content
|
||||
|
|
@ -65,24 +66,27 @@ func (h *Image) Parse(ctx *agentContext.Context, content agentContext.ContentPar
|
|||
return h.base64(ctx, content, visionFormat)
|
||||
}
|
||||
|
||||
// Model doesn't support vision - check cache first, then use vision agent/MCP
|
||||
// Try to get cached text (from attachment's content_preview)
|
||||
// Model doesn't support vision - fallback chain:
|
||||
// 1. Cache -> 2. Uses.Vision (explicit config) -> 3. tools/vision (auto) -> 4. Placeholder text
|
||||
|
||||
cachedText, found, err := h.readFromCache(ctx, content.ImageURL.URL)
|
||||
if err == nil && found {
|
||||
// Cache hit! Return as text content
|
||||
return agentContext.ContentPart{
|
||||
Type: agentContext.ContentText,
|
||||
Text: cachedText,
|
||||
}, nil, nil
|
||||
}
|
||||
|
||||
// No cache, try to use vision agent/MCP
|
||||
if h.options.CompletionOptions != nil && h.options.CompletionOptions.Uses != nil && h.options.CompletionOptions.Uses.Vision != "" {
|
||||
return h.agent(ctx, content)
|
||||
}
|
||||
|
||||
// No vision support and no vision tool specified, return error
|
||||
return content, nil, fmt.Errorf("model doesn't support vision and no vision tool specified in uses.Vision")
|
||||
if text, err := h.readImageWithTools(ctx, content); err == nil {
|
||||
h.saveToCache(ctx, content.ImageURL.URL, text)
|
||||
return agentContext.ContentPart{Type: agentContext.ContentText, Text: text}, nil, nil
|
||||
}
|
||||
|
||||
return agentContext.ContentPart{Type: agentContext.ContentText, Text: "[Image content - vision model not available]"}, nil, nil
|
||||
}
|
||||
|
||||
// base64 encodes image content to base64 (for vision support)
|
||||
|
|
@ -360,6 +364,37 @@ func (h *Image) callMCPVisionTool(ctx *agentContext.Context, serverID string, co
|
|||
return result, err
|
||||
}
|
||||
|
||||
// readImageWithTools calls tools/vision.ReadImage to convert image to text
|
||||
// using a vision-capable model resolved via llmprovider.
|
||||
func (h *Image) readImageWithTools(ctx *agentContext.Context, content agentContext.ContentPart) (string, error) {
|
||||
if ctx.Authorized == nil {
|
||||
return "", fmt.Errorf("no auth info available for vision model resolution")
|
||||
}
|
||||
|
||||
src := wrapperToAttachURI(content.ImageURL.URL)
|
||||
|
||||
loadingID := h.sendLoading(ctx, i18n.T(ctx.Locale, "content.image.analyzing"))
|
||||
|
||||
resp, err := vision.ReadImage(ctx.Context, src, "Please describe this image in detail.", 1080, ctx.Authorized)
|
||||
|
||||
h.sendLoadingDone(ctx, loadingID)
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return resp.Content, nil
|
||||
}
|
||||
|
||||
// wrapperToAttachURI converts __uploader://fileID to attach://uploader/fileID
|
||||
// format expected by tools/vision.readBytes.
|
||||
func wrapperToAttachURI(url string) string {
|
||||
uploaderName, fileID, ok := attachment.Parse(url)
|
||||
if !ok {
|
||||
return url
|
||||
}
|
||||
return "attach://" + uploaderName + "/" + fileID
|
||||
}
|
||||
|
||||
// sendLoading sends a loading message and returns the message ID
|
||||
// Returns empty string if SilentLoading is enabled
|
||||
func (h *Image) sendLoading(ctx *agentContext.Context, msg string) string {
|
||||
|
|
|
|||
|
|
@ -117,11 +117,12 @@ func TestParseWithoutVisionSupport(t *testing.T) {
|
|||
}
|
||||
|
||||
handler := image.New(options)
|
||||
_, _, err := handler.Parse(ctx, content)
|
||||
result, _, err := handler.Parse(ctx, content)
|
||||
|
||||
// Should return error because no vision support and no vision tool specified
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "no vision tool specified")
|
||||
// Should return placeholder text (no error) when no vision support
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, agentContext.ContentText, result.Type)
|
||||
assert.Contains(t, result.Text, "Image content")
|
||||
}
|
||||
|
||||
// TestParseWithEmptyURL tests parsing image with empty URL
|
||||
|
|
|
|||
|
|
@ -53,19 +53,28 @@ func (a *VisionAdapter) removeImageContent(messages []context.Message) []context
|
|||
for _, msg := range messages {
|
||||
processedMsg := msg
|
||||
|
||||
// Handle multimodal content (array of map)
|
||||
if contentParts, ok := msg.Content.([]map[string]interface{}); ok {
|
||||
if contentParts, ok := msg.Content.([]context.ContentPart); ok {
|
||||
filtered := make([]context.ContentPart, 0)
|
||||
for _, part := range contentParts {
|
||||
if part.Type != context.ContentImageURL {
|
||||
filtered = append(filtered, part)
|
||||
}
|
||||
}
|
||||
if len(filtered) == 0 {
|
||||
processedMsg.Content = "[Image content not supported by this model]"
|
||||
} else {
|
||||
processedMsg.Content = filtered
|
||||
}
|
||||
} else if contentParts, ok := msg.Content.([]map[string]interface{}); ok {
|
||||
filteredParts := make([]map[string]interface{}, 0)
|
||||
|
||||
for _, part := range contentParts {
|
||||
partType, _ := part["type"].(string)
|
||||
// Skip image content
|
||||
if partType != "image_url" && partType != "image" {
|
||||
filteredParts = append(filteredParts, part)
|
||||
}
|
||||
}
|
||||
|
||||
// If all parts were filtered out, add placeholder text
|
||||
if len(filteredParts) == 0 {
|
||||
processedMsg.Content = "[Image content not supported by this model]"
|
||||
} else if len(filteredParts) == 1 {
|
||||
|
|
|
|||
|
|
@ -45,11 +45,11 @@ func buildAdapters(cap *goullm.Capabilities) []adapters.CapabilityAdapter {
|
|||
// Tool call adapter
|
||||
result = append(result, adapters.NewToolCallAdapter(cap.ToolCalls))
|
||||
|
||||
// Vision adapter
|
||||
// Vision adapter (always registered to strip unsupported image content)
|
||||
visionSupport, visionFormat := context.GetVisionSupport(cap)
|
||||
if visionSupport {
|
||||
result = append(result, adapters.NewVisionAdapter(true, visionFormat))
|
||||
} else if cap.Vision != nil {
|
||||
} else {
|
||||
result = append(result, adapters.NewVisionAdapter(false, context.VisionFormatNone))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -156,12 +156,11 @@ func buildAdapters(cap *goullm.Capabilities) []adapters.CapabilityAdapter {
|
|||
// Tool call adapter
|
||||
result = append(result, adapters.NewToolCallAdapter(cap.ToolCalls))
|
||||
|
||||
// Vision adapter
|
||||
// Vision adapter (always registered to strip unsupported image content)
|
||||
visionSupport, visionFormat := context.GetVisionSupport(cap)
|
||||
if visionSupport {
|
||||
result = append(result, adapters.NewVisionAdapter(true, visionFormat))
|
||||
} else if cap.Vision != nil {
|
||||
// Vision explicitly disabled, add adapter to remove image content
|
||||
} else {
|
||||
result = append(result, adapters.NewVisionAdapter(false, context.VisionFormatNone))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -77,6 +77,9 @@ func (r *Runner) buildCommand(ctx context.Context, req *types.StreamRequest, p p
|
|||
|
||||
var systemPrompt string
|
||||
envPrompt := buildSandboxEnvPrompt(p, workDir)
|
||||
if capPrompt := buildModelCapabilityPrompt(req); capPrompt != "" {
|
||||
envPrompt += "\n\n" + capPrompt
|
||||
}
|
||||
if !isContinuation && req.SystemPrompt != "" {
|
||||
systemPrompt = req.SystemPrompt + "\n\n" + envPrompt
|
||||
} else if !isContinuation {
|
||||
|
|
@ -310,6 +313,142 @@ func buildSandboxEnvPrompt(p platform, workDir string) string {
|
|||
%[4]s`, workDir, osName, shell, shellNote)
|
||||
}
|
||||
|
||||
func buildModelCapabilityPrompt(req *types.StreamRequest) string {
|
||||
if req.Connector == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
lc, ok := req.Connector.(goullm.LLMConnector)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
primaryModel := lc.GetModel()
|
||||
if primaryModel == "" {
|
||||
return ""
|
||||
}
|
||||
primaryCaps := lc.GetCapabilities()
|
||||
|
||||
type tierInfo struct {
|
||||
tier string
|
||||
alias string
|
||||
model string
|
||||
caps *goullm.Capabilities
|
||||
conn connector.Connector
|
||||
}
|
||||
|
||||
tiers := []tierInfo{
|
||||
{tier: "Default", alias: "sonnet", model: primaryModel, caps: primaryCaps, conn: req.Connector},
|
||||
}
|
||||
|
||||
hasDifferentTier := false
|
||||
if rc, exists := req.Roles["heavy"]; exists && rc != nil {
|
||||
m := connectorModel(rc)
|
||||
if m != "" {
|
||||
caps := connectorCaps(rc)
|
||||
tiers = append(tiers, tierInfo{tier: "Heavy", alias: "opus", model: m, caps: caps, conn: rc})
|
||||
if m != primaryModel {
|
||||
hasDifferentTier = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if rc, exists := req.Roles["light"]; exists && rc != nil {
|
||||
m := connectorModel(rc)
|
||||
if m != "" {
|
||||
caps := connectorCaps(rc)
|
||||
tiers = append(tiers, tierInfo{tier: "Light", alias: "haiku", model: m, caps: caps, conn: rc})
|
||||
if m != primaryModel {
|
||||
hasDifferentTier = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString("## Model Capabilities\n\n")
|
||||
sb.WriteString(fmt.Sprintf("Your current model: `%s`\n", primaryModel))
|
||||
|
||||
if hasDifferentTier {
|
||||
sb.WriteString("\n### Available Model Tiers\n\n")
|
||||
sb.WriteString("| Tier | Alias | Model | Capabilities |\n")
|
||||
sb.WriteString("| ---- | ----- | ----- | ------------ |\n")
|
||||
for _, t := range tiers {
|
||||
capList := formatCapabilities(t.caps, t.conn)
|
||||
sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s |\n", t.tier, t.alias, t.model, capList))
|
||||
}
|
||||
}
|
||||
|
||||
var guidance []string
|
||||
if hasDifferentTier {
|
||||
guidance = append(guidance,
|
||||
"For complex reasoning, multi-step analysis, or tasks requiring deep thought, delegate to a sub-agent with `model: \"opus\"`",
|
||||
"For simple tasks (formatting, translation, summarization), use `model: \"haiku\"` for faster responses",
|
||||
)
|
||||
}
|
||||
|
||||
primaryHasVision := primaryCaps.HasVision()
|
||||
if !primaryHasVision {
|
||||
if _, hasVisionRole := req.Roles["vision"]; hasVisionRole {
|
||||
guidance = append(guidance,
|
||||
"**Image/Vision**: Your current model cannot process images directly. Use the `image_read` system tool (`tai tool image_read`) to analyze images — see the yao-vision skill for details",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if len(guidance) > 0 {
|
||||
sb.WriteString("\n### Usage Guidance\n\n")
|
||||
for _, g := range guidance {
|
||||
sb.WriteString("- " + g + "\n")
|
||||
}
|
||||
}
|
||||
|
||||
if !hasDifferentTier && len(guidance) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func connectorModel(c connector.Connector) string {
|
||||
if lc, ok := c.(goullm.LLMConnector); ok {
|
||||
if m := lc.GetModel(); m != "" {
|
||||
return m
|
||||
}
|
||||
}
|
||||
m, _ := c.Setting()["model"].(string)
|
||||
return m
|
||||
}
|
||||
|
||||
func connectorCaps(c connector.Connector) *goullm.Capabilities {
|
||||
if lc, ok := c.(goullm.LLMConnector); ok {
|
||||
return lc.GetCapabilities()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func formatCapabilities(caps *goullm.Capabilities, conn connector.Connector) string {
|
||||
var parts []string
|
||||
hasThinking := caps.HasReasoning()
|
||||
if !hasThinking && conn != nil {
|
||||
if thinking, ok := conn.Setting()["thinking"].(map[string]interface{}); ok {
|
||||
if t, _ := thinking["type"].(string); t == "enabled" {
|
||||
hasThinking = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if hasThinking {
|
||||
parts = append(parts, "thinking")
|
||||
}
|
||||
if caps.HasVision() {
|
||||
parts = append(parts, "vision")
|
||||
}
|
||||
if caps.HasToolCalls() {
|
||||
parts = append(parts, "tool_calls")
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "-"
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
func hasExistingSession(ctx context.Context, computer infra.Computer, p platform, assistantID string) bool {
|
||||
workDir := computer.GetWorkDir()
|
||||
var sessionDir string
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import (
|
|||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/yaoapp/gou/connector"
|
||||
goullm "github.com/yaoapp/gou/llm"
|
||||
gouTypes "github.com/yaoapp/gou/types"
|
||||
"github.com/yaoapp/xun/dbal/query"
|
||||
"github.com/yaoapp/xun/dbal/schema"
|
||||
|
|
@ -701,3 +702,155 @@ func TestBuildEnv_Anthropic_MultiConnector_Incompatible(t *testing.T) {
|
|||
assert.Equal(t, "claude-sonnet-4-20250514", env["ANTHROPIC_DEFAULT_OPUS_MODEL"],
|
||||
"incompatible connector: heavy should keep primary model (different host, no anthropic protocol)")
|
||||
}
|
||||
|
||||
// --- fakeLLMConnector implements both connector.Connector and goullm.LLMConnector ---
|
||||
|
||||
type fakeLLMConnector struct {
|
||||
fakeConnector
|
||||
model string
|
||||
caps *goullm.Capabilities
|
||||
}
|
||||
|
||||
func (f *fakeLLMConnector) GetAuthMode() goullm.AuthMode { return goullm.AuthBearer }
|
||||
func (f *fakeLLMConnector) GetURL() string { return "" }
|
||||
func (f *fakeLLMConnector) GetKey() string { return "" }
|
||||
func (f *fakeLLMConnector) GetModel() string { return f.model }
|
||||
func (f *fakeLLMConnector) GetSupportedParams() map[string]*goullm.ParamSpec {
|
||||
return nil
|
||||
}
|
||||
func (f *fakeLLMConnector) GetCapabilities() *goullm.Capabilities { return f.caps }
|
||||
|
||||
// --- buildModelCapabilityPrompt tests ---
|
||||
|
||||
func TestBuildModelCapabilityPrompt_NilConnector(t *testing.T) {
|
||||
req := &types.StreamRequest{Config: &types.SandboxConfig{}}
|
||||
result := buildModelCapabilityPrompt(req)
|
||||
assert.Empty(t, result)
|
||||
}
|
||||
|
||||
func TestBuildModelCapabilityPrompt_NoRoles_NoSpecialCaps(t *testing.T) {
|
||||
primary := &fakeLLMConnector{
|
||||
fakeConnector: fakeConnector{id: "test", typ: connector.OPENAI, settings: map[string]interface{}{"model": "deepseek-v4-flash"}},
|
||||
model: "deepseek-v4-flash",
|
||||
caps: &goullm.Capabilities{ToolCalls: true, Streaming: true},
|
||||
}
|
||||
req := &types.StreamRequest{
|
||||
Config: &types.SandboxConfig{},
|
||||
Connector: primary,
|
||||
}
|
||||
result := buildModelCapabilityPrompt(req)
|
||||
assert.Empty(t, result, "no roles and no special caps → empty")
|
||||
}
|
||||
|
||||
func TestBuildModelCapabilityPrompt_WithHeavyAndLight(t *testing.T) {
|
||||
primary := &fakeLLMConnector{
|
||||
fakeConnector: fakeConnector{id: "ds-flash", typ: connector.OPENAI, settings: map[string]interface{}{"model": "deepseek-v4-flash"}},
|
||||
model: "deepseek-v4-flash",
|
||||
caps: &goullm.Capabilities{ToolCalls: true},
|
||||
}
|
||||
heavy := &fakeLLMConnector{
|
||||
fakeConnector: fakeConnector{id: "ds-pro", typ: connector.OPENAI, settings: map[string]interface{}{"model": "deepseek-v4-pro"}},
|
||||
model: "deepseek-v4-pro",
|
||||
caps: &goullm.Capabilities{Reasoning: true, ToolCalls: true},
|
||||
}
|
||||
light := &fakeLLMConnector{
|
||||
fakeConnector: fakeConnector{id: "ds-lite", typ: connector.OPENAI, settings: map[string]interface{}{"model": "deepseek-v4-flash"}},
|
||||
model: "deepseek-v4-flash",
|
||||
caps: &goullm.Capabilities{ToolCalls: true},
|
||||
}
|
||||
|
||||
req := &types.StreamRequest{
|
||||
Config: &types.SandboxConfig{},
|
||||
Connector: primary,
|
||||
Roles: map[string]connector.Connector{
|
||||
"default": primary,
|
||||
"heavy": heavy,
|
||||
"light": light,
|
||||
},
|
||||
}
|
||||
result := buildModelCapabilityPrompt(req)
|
||||
assert.Contains(t, result, "deepseek-v4-flash")
|
||||
assert.Contains(t, result, "deepseek-v4-pro")
|
||||
assert.Contains(t, result, "opus")
|
||||
assert.Contains(t, result, "haiku")
|
||||
assert.Contains(t, result, "thinking")
|
||||
assert.Contains(t, result, "tool_calls")
|
||||
assert.Contains(t, result, "sub-agent")
|
||||
}
|
||||
|
||||
func TestBuildModelCapabilityPrompt_VisionGuidance(t *testing.T) {
|
||||
primary := &fakeLLMConnector{
|
||||
fakeConnector: fakeConnector{id: "ds-flash", typ: connector.OPENAI, settings: map[string]interface{}{"model": "deepseek-v4-flash"}},
|
||||
model: "deepseek-v4-flash",
|
||||
caps: &goullm.Capabilities{ToolCalls: true},
|
||||
}
|
||||
visionConn := &fakeLLMConnector{
|
||||
fakeConnector: fakeConnector{id: "vision", typ: connector.OPENAI, settings: map[string]interface{}{"model": "gpt-4o"}},
|
||||
model: "gpt-4o",
|
||||
caps: &goullm.Capabilities{Vision: true, ToolCalls: true},
|
||||
}
|
||||
|
||||
req := &types.StreamRequest{
|
||||
Config: &types.SandboxConfig{},
|
||||
Connector: primary,
|
||||
Roles: map[string]connector.Connector{
|
||||
"default": primary,
|
||||
"vision": visionConn,
|
||||
},
|
||||
}
|
||||
result := buildModelCapabilityPrompt(req)
|
||||
assert.Contains(t, result, "image_read")
|
||||
assert.Contains(t, result, "Image/Vision")
|
||||
}
|
||||
|
||||
func TestBuildModelCapabilityPrompt_PrimaryHasVision_NoGuidance(t *testing.T) {
|
||||
primary := &fakeLLMConnector{
|
||||
fakeConnector: fakeConnector{id: "gpt4o", typ: connector.OPENAI, settings: map[string]interface{}{"model": "gpt-4o"}},
|
||||
model: "gpt-4o",
|
||||
caps: &goullm.Capabilities{Vision: true, ToolCalls: true},
|
||||
}
|
||||
|
||||
req := &types.StreamRequest{
|
||||
Config: &types.SandboxConfig{},
|
||||
Connector: primary,
|
||||
Roles: map[string]connector.Connector{
|
||||
"default": primary,
|
||||
"vision": primary,
|
||||
},
|
||||
}
|
||||
result := buildModelCapabilityPrompt(req)
|
||||
assert.NotContains(t, result, "image_read", "should not suggest image_read when primary has vision")
|
||||
}
|
||||
|
||||
func TestBuildModelCapabilityPrompt_ThinkingFromSettings(t *testing.T) {
|
||||
primary := &fakeLLMConnector{
|
||||
fakeConnector: fakeConnector{
|
||||
id: "ds-pro",
|
||||
typ: connector.OPENAI,
|
||||
settings: map[string]interface{}{
|
||||
"model": "deepseek-v4-pro",
|
||||
"thinking": map[string]interface{}{
|
||||
"type": "enabled",
|
||||
},
|
||||
},
|
||||
},
|
||||
model: "deepseek-v4-pro",
|
||||
caps: &goullm.Capabilities{ToolCalls: true},
|
||||
}
|
||||
heavy := &fakeLLMConnector{
|
||||
fakeConnector: fakeConnector{id: "ds-pro2", typ: connector.OPENAI, settings: map[string]interface{}{"model": "deepseek-v4-pro-max"}},
|
||||
model: "deepseek-v4-pro-max",
|
||||
caps: &goullm.Capabilities{ToolCalls: true},
|
||||
}
|
||||
|
||||
req := &types.StreamRequest{
|
||||
Config: &types.SandboxConfig{},
|
||||
Connector: primary,
|
||||
Roles: map[string]connector.Connector{
|
||||
"default": primary,
|
||||
"heavy": heavy,
|
||||
},
|
||||
}
|
||||
result := buildModelCapabilityPrompt(req)
|
||||
assert.Contains(t, result, "thinking", "should detect thinking from Setting()[\"thinking\"]")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,8 +48,9 @@ func InjectSystemSkills(ws writerFS, skills fs.FS, targetDir string) error {
|
|||
})
|
||||
}
|
||||
|
||||
// AppendSystemPrompt appends content to a file in the workspace with an
|
||||
// idempotent marker. If the marker already exists the call is a no-op.
|
||||
// AppendSystemPrompt injects content into a file in the workspace using an
|
||||
// idempotent marker. If the marker already exists, the injected section is
|
||||
// replaced with the new content (so updates propagate to existing sandboxes).
|
||||
// If the file does not exist it is created with just the marker + content.
|
||||
func AppendSystemPrompt(ws writerFS, filename string, content []byte) error {
|
||||
existing, err := ws.ReadFile(filename)
|
||||
|
|
@ -61,8 +62,15 @@ func AppendSystemPrompt(ws writerFS, filename string, content []byte) error {
|
|||
return ws.WriteFile(filename, append(header, content...), 0644)
|
||||
}
|
||||
|
||||
if bytes.Contains(existing, []byte(systemToolsMarker)) {
|
||||
return nil
|
||||
idx := bytes.Index(existing, []byte(systemToolsMarker))
|
||||
if idx >= 0 {
|
||||
injected := append([]byte(systemToolsMarker+"\n\n"), content...)
|
||||
prefix := existing[:idx]
|
||||
merged := append(bytes.TrimRight(prefix, "\n\r\t "), []byte("\n\n---\n\n")...)
|
||||
if idx == 0 {
|
||||
merged = nil
|
||||
}
|
||||
return ws.WriteFile(filename, append(merged, injected...), 0644)
|
||||
}
|
||||
|
||||
separator := []byte("\n\n---\n\n" + systemToolsMarker + "\n\n")
|
||||
|
|
|
|||
|
|
@ -106,6 +106,67 @@ func TestAppendSystemPrompt_Idempotent(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestAppendSystemPrompt_UpdatesExistingContent(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ws := newDirFS(dir)
|
||||
|
||||
oldContent := []byte("## Old Tools\nweb_search only\n")
|
||||
if err := AppendSystemPrompt(ws, "CLAUDE.md", oldContent); err != nil {
|
||||
t.Fatalf("first call: %v", err)
|
||||
}
|
||||
|
||||
newContent := []byte("## Updated Tools\nweb_search + image_read\n")
|
||||
if err := AppendSystemPrompt(ws, "CLAUDE.md", newContent); err != nil {
|
||||
t.Fatalf("second call: %v", err)
|
||||
}
|
||||
|
||||
data, _ := os.ReadFile(filepath.Join(dir, "CLAUDE.md"))
|
||||
got := string(data)
|
||||
assertContains(t, got, "image_read")
|
||||
assertContains(t, got, systemToolsMarker)
|
||||
if containsStr(got, "Old Tools") {
|
||||
t.Error("old content should have been replaced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendSystemPrompt_UpdatesPreservesUserContent(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ws := newDirFS(dir)
|
||||
|
||||
userContent := []byte("# My Project\n\nUser notes.\n")
|
||||
if err := os.WriteFile(filepath.Join(dir, "CLAUDE.md"), userContent, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
oldContent := []byte("## Old Tools\n")
|
||||
if err := AppendSystemPrompt(ws, "CLAUDE.md", oldContent); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
newContent := []byte("## Updated Tools\nimage_read added\n")
|
||||
if err := AppendSystemPrompt(ws, "CLAUDE.md", newContent); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
data, _ := os.ReadFile(filepath.Join(dir, "CLAUDE.md"))
|
||||
got := string(data)
|
||||
assertContains(t, got, "My Project")
|
||||
assertContains(t, got, "User notes")
|
||||
assertContains(t, got, "image_read")
|
||||
if containsStr(got, "Old Tools") {
|
||||
t.Error("old injected content should have been replaced")
|
||||
}
|
||||
}
|
||||
|
||||
func containsStr(s, sub string) bool {
|
||||
for i := 0; i <= len(s)-len(sub); i++ {
|
||||
if s[i:i+len(sub)] == sub {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func assertContains(t *testing.T, s, sub string) {
|
||||
t.Helper()
|
||||
if len(s) < len(sub) {
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -45,6 +45,7 @@ require (
|
|||
github.com/yaoapp/xun v0.9.0
|
||||
go.mongodb.org/mongo-driver v1.17.3
|
||||
golang.org/x/crypto v0.49.0
|
||||
golang.org/x/image v0.38.0
|
||||
golang.org/x/net v0.52.0
|
||||
golang.org/x/sys v0.42.0
|
||||
golang.org/x/text v0.35.0
|
||||
|
|
@ -211,7 +212,6 @@ require (
|
|||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/arch v0.17.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
|
||||
golang.org/x/image v0.38.0 // indirect
|
||||
golang.org/x/mod v0.33.0 // indirect
|
||||
golang.org/x/oauth2 v0.35.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
|
|
|
|||
8
tools/mcps/vision.json
Normal file
8
tools/mcps/vision.json
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
"name": "yao-vision",
|
||||
"transport": "process",
|
||||
"description": "Vision tools for image understanding",
|
||||
"tools": {
|
||||
"image_read": "tools.image_read"
|
||||
}
|
||||
}
|
||||
|
|
@ -26,6 +26,14 @@ These environment variables are set by the Yao sandbox. **Always use these varia
|
|||
User-uploaded files are placed in `$WORKDIR/.attachments/{chatID}/`.
|
||||
When the user references an attached file, read it from this directory.
|
||||
|
||||
### Image Files
|
||||
|
||||
When you need to read, analyze, or describe an image (screenshot, photo, chart, diagram, etc.), **always use `image_read`** instead of trying to read binary files directly. The tool sends the image to a vision model and returns a text description.
|
||||
|
||||
```bash
|
||||
tai tool image_read '{"image_path": "<file_path_or_url>", "prompt": "describe this image"}'
|
||||
```
|
||||
|
||||
## Yao System Tools
|
||||
|
||||
You have access to Yao system tools via the `tai` command in bash.
|
||||
|
|
@ -41,5 +49,6 @@ You have access to Yao system tools via the `tai` command in bash.
|
|||
| `doc_list` | yao-doc | Search/list available process documentation |
|
||||
| `doc_inspect` | yao-doc | Get detailed docs for a specific process |
|
||||
| `doc_validate` | yao-doc | Validate a process name and get suggestions |
|
||||
| `image_read` | yao-vision | Read and analyze images using a vision model |
|
||||
|
||||
The three system skills (`yao-web`, `yao-process`, `yao-doc`) in `$HOME/.claude/skills/` are **auto-discovered** — they contain detailed parameter docs and workflow guidance. You do not need to manually read them; they are loaded automatically when your task matches their description.
|
||||
The system skills (`yao-web`, `yao-process`, `yao-doc`, `yao-vision`) in `$HOME/.claude/skills/` are **auto-discovered** — they contain detailed parameter docs and workflow guidance. You do not need to manually read them; they are loaded automatically when your task matches their description.
|
||||
|
|
|
|||
46
tools/skills/yao-vision/SKILL.md
Normal file
46
tools/skills/yao-vision/SKILL.md
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
---
|
||||
name: yao-vision
|
||||
description: Image understanding expert. ALWAYS invoke this skill when you need to read, analyze, or describe an image that you cannot process directly. Use for screenshots, photos, charts, diagrams, or any visual content.
|
||||
---
|
||||
|
||||
# Vision Tools
|
||||
|
||||
Use when you encounter images you cannot read natively (e.g., as a text-only model).
|
||||
|
||||
## image_read
|
||||
|
||||
Send an image to a vision-capable model and get a text description.
|
||||
|
||||
### Local file (most common):
|
||||
```bash
|
||||
tai tool image_read '{"image_path": "/path/to/image.png", "prompt": "Describe this image"}'
|
||||
```
|
||||
|
||||
### URL:
|
||||
```bash
|
||||
tai tool image_read '{"image_path": "https://example.com/photo.jpg", "prompt": "What is shown?"}'
|
||||
```
|
||||
|
||||
### Cross-workspace file:
|
||||
```bash
|
||||
tai tool image_read '{"image_path": "workspace://ws-id/path/to/image.png", "prompt": "Analyze"}'
|
||||
```
|
||||
|
||||
### Attachment file:
|
||||
```bash
|
||||
tai tool image_read '{"image_path": "attach://__yao.attachment/file-id-123", "prompt": "Describe"}'
|
||||
```
|
||||
|
||||
### Yao data file:
|
||||
```bash
|
||||
tai tool image_read '{"image_path": "yao://uploads/photo.jpg", "prompt": "Describe"}'
|
||||
```
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
| ---------- | ------- | -------- | --------------------------------------------------------------- |
|
||||
| image_path | string | yes | File path, URL, workspace://, attach://, or yao:// URI |
|
||||
| prompt | string | no | Analysis instruction (default: describe in detail) |
|
||||
| max_size | integer | no | Max dimension in pixels for longest edge (default: 1080) |
|
||||
|
||||
Images are automatically resized (preserving aspect ratio) before sending to the vision model.
|
||||
Supported formats: PNG, JPEG, GIF, WebP.
|
||||
|
|
@ -10,6 +10,7 @@ import (
|
|||
"github.com/yaoapp/kun/log"
|
||||
"github.com/yaoapp/yao/tools/docs"
|
||||
"github.com/yaoapp/yao/tools/proc"
|
||||
"github.com/yaoapp/yao/tools/vision"
|
||||
"github.com/yaoapp/yao/tools/webfetch"
|
||||
"github.com/yaoapp/yao/tools/websearch"
|
||||
)
|
||||
|
|
@ -23,6 +24,9 @@ var mcpProcessDSL []byte
|
|||
//go:embed mcps/doc.json
|
||||
var mcpDocDSL []byte
|
||||
|
||||
//go:embed mcps/vision.json
|
||||
var mcpVisionDSL []byte
|
||||
|
||||
func init() {
|
||||
process.RegisterGroup("tools", map[string]process.Handler{
|
||||
"web_search": websearch.Handler,
|
||||
|
|
@ -32,6 +36,7 @@ func init() {
|
|||
"doc_list": docs.ListHandler,
|
||||
"doc_inspect": docs.InspectHandler,
|
||||
"doc_validate": docs.ValidateHandler,
|
||||
"image_read": vision.Handler,
|
||||
})
|
||||
|
||||
registerMCPServer(mcpWebDSL, "yao-web",
|
||||
|
|
@ -40,6 +45,8 @@ func init() {
|
|||
proc.SchemaJSON, proc.AllowedSchemaJSON)
|
||||
registerMCPServer(mcpDocDSL, "yao-doc",
|
||||
docs.ListSchemaJSON, docs.InspectSchemaJSON, docs.ValidateSchemaJSON)
|
||||
registerMCPServer(mcpVisionDSL, "yao-vision",
|
||||
vision.SchemaJSON)
|
||||
}
|
||||
|
||||
func registerMCPServer(dsl []byte, id string, schemas ...[]byte) {
|
||||
|
|
|
|||
26
tools/vision/schema.json
Normal file
26
tools/vision/schema.json
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
{
|
||||
"name": "image_read",
|
||||
"description": "Read and analyze an image using a vision-capable model.",
|
||||
"process": "tools.image_read",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"image_path": {
|
||||
"type": "string",
|
||||
"description": "Path or URI of the image to analyze. Accepts: local file path, URL (http/https), workspace://, attach://, yao://, or data URI."
|
||||
},
|
||||
"prompt": {
|
||||
"type": "string",
|
||||
"description": "What to analyze about the image (default: describe in detail)",
|
||||
"default": "Please describe this image in detail."
|
||||
},
|
||||
"max_size": {
|
||||
"type": "integer",
|
||||
"description": "Max dimension in pixels for the longest edge. Image is resized (preserving aspect ratio) before sending to the vision model. Default 1080.",
|
||||
"default": 1080
|
||||
}
|
||||
},
|
||||
"required": ["image_path"]
|
||||
},
|
||||
"x-process-args": ["$args.image_path", "$args.prompt", "$args.max_size"]
|
||||
}
|
||||
237
tools/vision/vision.go
Normal file
237
tools/vision/vision.go
Normal file
|
|
@ -0,0 +1,237 @@
|
|||
package vision
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
_ "embed"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/gif"
|
||||
"image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/image/draw"
|
||||
_ "golang.org/x/image/webp"
|
||||
|
||||
goufs "github.com/yaoapp/gou/fs"
|
||||
"github.com/yaoapp/gou/process"
|
||||
agentCtx "github.com/yaoapp/yao/agent/context"
|
||||
agentLLM "github.com/yaoapp/yao/agent/llm"
|
||||
"github.com/yaoapp/yao/attachment"
|
||||
"github.com/yaoapp/yao/openapi/oauth/authorized"
|
||||
oauthTypes "github.com/yaoapp/yao/openapi/oauth/types"
|
||||
ws "github.com/yaoapp/yao/workspace"
|
||||
)
|
||||
|
||||
//go:embed schema.json
|
||||
var SchemaJSON []byte
|
||||
|
||||
// ImageReadResponse is the return type for image_read.
|
||||
type ImageReadResponse struct {
|
||||
Content string `json:"content"`
|
||||
Model string `json:"model"`
|
||||
}
|
||||
|
||||
// ReadImage reads and analyzes an image using the vision model.
|
||||
// It resolves the image from src, finds a vision-capable connector via llmprovider,
|
||||
// and returns the model's text description.
|
||||
func ReadImage(goCtx context.Context, src string, prompt string, maxSize int, authInfo *oauthTypes.AuthorizedInfo) (*ImageReadResponse, error) {
|
||||
if prompt == "" {
|
||||
prompt = "Please describe this image in detail."
|
||||
}
|
||||
if maxSize <= 0 {
|
||||
maxSize = 1080
|
||||
}
|
||||
|
||||
imageURI, err := resolveImage(src, maxSize)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve image: %w", err)
|
||||
}
|
||||
|
||||
conn, caps, err := agentLLM.ResolveConnector("use::vision", authInfo)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve vision connector: %w", err)
|
||||
}
|
||||
|
||||
opts := &agentCtx.CompletionOptions{Capabilities: caps}
|
||||
instance, err := agentLLM.New(conn, opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create LLM instance: %w", err)
|
||||
}
|
||||
|
||||
messages := []agentCtx.Message{{
|
||||
Role: "user",
|
||||
Content: []agentCtx.ContentPart{
|
||||
{Type: agentCtx.ContentImageURL, ImageURL: &agentCtx.ImageURL{URL: imageURI}},
|
||||
{Type: agentCtx.ContentText, Text: prompt},
|
||||
},
|
||||
}}
|
||||
|
||||
chatID := agentCtx.GenChatID()
|
||||
ctx := agentCtx.New(goCtx, authInfo, chatID)
|
||||
defer ctx.Release()
|
||||
|
||||
resp, err := instance.Post(ctx, messages, opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("vision model call: %w", err)
|
||||
}
|
||||
|
||||
return &ImageReadResponse{
|
||||
Content: extractTextContent(resp.Content),
|
||||
Model: resp.Model,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Handler is the tools.image_read process handler.
|
||||
func Handler(proc *process.Process) interface{} {
|
||||
src := proc.ArgsString(0)
|
||||
if src == "" {
|
||||
return map[string]interface{}{"error": "image_path is required: provide a file path, URL, or URI"}
|
||||
}
|
||||
|
||||
authInfo := authorized.ProcessAuthInfo(proc)
|
||||
if authInfo == nil {
|
||||
return map[string]interface{}{"error": "unauthorized: no auth info in request"}
|
||||
}
|
||||
|
||||
resp, err := ReadImage(proc.Context, src,
|
||||
proc.ArgsString(1, "Please describe this image in detail."),
|
||||
proc.ArgsInt(2, 1080),
|
||||
authInfo,
|
||||
)
|
||||
if err != nil {
|
||||
return map[string]interface{}{"error": err.Error()}
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
// resolveImage reads image bytes from any supported source, resizes, and returns a data URI.
|
||||
func resolveImage(src string, maxSize int) (string, error) {
|
||||
raw, err := readBytes(src)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
data, mime := resizeImage(raw, maxSize)
|
||||
|
||||
if len(data) > 3<<20 {
|
||||
return "", fmt.Errorf("compressed image still exceeds 3MB, try a smaller max_size (current: %d)", maxSize)
|
||||
}
|
||||
|
||||
return "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data), nil
|
||||
}
|
||||
|
||||
// readBytes dispatches to the appropriate reader based on URI scheme.
|
||||
func readBytes(src string) ([]byte, error) {
|
||||
switch {
|
||||
case strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://"):
|
||||
return httpGet(src)
|
||||
|
||||
case strings.HasPrefix(src, "workspace://"):
|
||||
rest := strings.TrimPrefix(src, "workspace://")
|
||||
idx := strings.Index(rest, "/")
|
||||
if idx < 0 {
|
||||
return nil, fmt.Errorf("invalid workspace URI: %s", src)
|
||||
}
|
||||
return ws.M().ReadFile(context.Background(), rest[:idx], rest[idx+1:])
|
||||
|
||||
case strings.HasPrefix(src, "attach://"):
|
||||
rest := strings.TrimPrefix(src, "attach://")
|
||||
idx := strings.Index(rest, "/")
|
||||
if idx < 0 {
|
||||
return nil, fmt.Errorf("invalid attach URI: %s", src)
|
||||
}
|
||||
mgr, ok := attachment.Managers[rest[:idx]]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("uploader not found: %s", rest[:idx])
|
||||
}
|
||||
return mgr.Read(context.Background(), rest[idx+1:])
|
||||
|
||||
case strings.HasPrefix(src, "yao://"):
|
||||
dataFS, err := goufs.Get("data")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return dataFS.ReadFile(strings.TrimPrefix(src, "yao://"))
|
||||
|
||||
case strings.HasPrefix(src, "data:"):
|
||||
return decodeDataURI(src)
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported image source: %s", src)
|
||||
}
|
||||
}
|
||||
|
||||
func httpGet(rawURL string) ([]byte, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("fetch %s: %d", rawURL, resp.StatusCode)
|
||||
}
|
||||
return io.ReadAll(io.LimitReader(resp.Body, 20<<20))
|
||||
}
|
||||
|
||||
// resizeImage decodes, resizes (longest edge <= maxSize), re-encodes as JPEG.
|
||||
// Returns original bytes unchanged when already small enough or if decode fails.
|
||||
func resizeImage(data []byte, maxSize int) ([]byte, string) {
|
||||
img, _, err := image.Decode(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return data, http.DetectContentType(data)
|
||||
}
|
||||
bounds := img.Bounds()
|
||||
w, h := bounds.Dx(), bounds.Dy()
|
||||
if w <= maxSize && h <= maxSize {
|
||||
return data, http.DetectContentType(data)
|
||||
}
|
||||
ratio := float64(maxSize) / float64(max(w, h))
|
||||
newW, newH := int(float64(w)*ratio), int(float64(h)*ratio)
|
||||
if newW < 1 {
|
||||
newW = 1
|
||||
}
|
||||
if newH < 1 {
|
||||
newH = 1
|
||||
}
|
||||
dst := image.NewRGBA(image.Rect(0, 0, newW, newH))
|
||||
draw.BiLinear.Scale(dst, dst.Bounds(), img, bounds, draw.Over, nil)
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: 85}); err != nil {
|
||||
return data, http.DetectContentType(data)
|
||||
}
|
||||
return buf.Bytes(), "image/jpeg"
|
||||
}
|
||||
|
||||
func decodeDataURI(uri string) ([]byte, error) {
|
||||
parts := strings.SplitN(uri, ",", 2)
|
||||
if len(parts) != 2 {
|
||||
return nil, fmt.Errorf("invalid data URI format")
|
||||
}
|
||||
return base64.StdEncoding.DecodeString(parts[1])
|
||||
}
|
||||
|
||||
func extractTextContent(content interface{}) string {
|
||||
if s, ok := content.(string); ok {
|
||||
return s
|
||||
}
|
||||
if parts, ok := content.([]agentCtx.ContentPart); ok {
|
||||
for _, p := range parts {
|
||||
if p.Type == agentCtx.ContentText {
|
||||
return p.Text
|
||||
}
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("%v", content)
|
||||
}
|
||||
238
tools/vision/vision_test.go
Normal file
238
tools/vision/vision_test.go
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
package vision
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"image"
|
||||
"image/color"
|
||||
_ "image/jpeg"
|
||||
"image/png"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
agentCtx "github.com/yaoapp/yao/agent/context"
|
||||
)
|
||||
|
||||
func makePNG(w, h int) []byte {
|
||||
img := image.NewRGBA(image.Rect(0, 0, w, h))
|
||||
for y := 0; y < h; y++ {
|
||||
for x := 0; x < w; x++ {
|
||||
img.Set(x, y, color.RGBA{R: 255, G: 0, B: 0, A: 255})
|
||||
}
|
||||
}
|
||||
var buf strings.Builder
|
||||
if err := png.Encode(&buf, img); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return []byte(buf.String())
|
||||
}
|
||||
|
||||
func TestResizeImage_SmallImage(t *testing.T) {
|
||||
pngData := makePNG(100, 80)
|
||||
data, mime := resizeImage(pngData, 1080)
|
||||
if mime != "image/png" {
|
||||
t.Errorf("mime = %q, want image/png", mime)
|
||||
}
|
||||
if len(data) != len(pngData) {
|
||||
t.Errorf("data length changed for small image")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResizeImage_LargeImage(t *testing.T) {
|
||||
pngData := makePNG(2000, 1500)
|
||||
data, mime := resizeImage(pngData, 1080)
|
||||
if mime != "image/jpeg" {
|
||||
t.Errorf("mime = %q, want image/jpeg", mime)
|
||||
}
|
||||
img, _, err := image.Decode(strings.NewReader(string(data)))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to decode resized image: %v", err)
|
||||
}
|
||||
b := img.Bounds()
|
||||
if b.Dx() > 1080 || b.Dy() > 1080 {
|
||||
t.Errorf("resized dimensions %dx%d exceed max 1080", b.Dx(), b.Dy())
|
||||
}
|
||||
if b.Dx() != 1080 {
|
||||
t.Errorf("longest edge = %d, want 1080", b.Dx())
|
||||
}
|
||||
}
|
||||
|
||||
func TestResizeImage_InvalidData(t *testing.T) {
|
||||
raw := []byte("not an image at all")
|
||||
data, mime := resizeImage(raw, 1080)
|
||||
if string(data) != string(raw) {
|
||||
t.Error("should return original data on decode failure")
|
||||
}
|
||||
if mime == "" {
|
||||
t.Error("mime should not be empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResizeImage_ExactBoundary(t *testing.T) {
|
||||
pngData := makePNG(1080, 720)
|
||||
data, _ := resizeImage(pngData, 1080)
|
||||
if len(data) != len(pngData) {
|
||||
t.Error("image at exact max_size should not be re-encoded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeDataURI(t *testing.T) {
|
||||
original := []byte("hello world")
|
||||
b64 := base64.StdEncoding.EncodeToString(original)
|
||||
uri := "data:application/octet-stream;base64," + b64
|
||||
|
||||
data, err := decodeDataURI(uri)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != string(original) {
|
||||
t.Errorf("decoded = %q, want %q", data, original)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeDataURI_Invalid(t *testing.T) {
|
||||
_, err := decodeDataURI("not-a-data-uri")
|
||||
if err == nil {
|
||||
t.Error("expected error for invalid data URI")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTextContent_String(t *testing.T) {
|
||||
result := extractTextContent("hello world")
|
||||
if result != "hello world" {
|
||||
t.Errorf("got %q, want %q", result, "hello world")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTextContent_ContentParts(t *testing.T) {
|
||||
parts := []agentCtx.ContentPart{
|
||||
{Type: agentCtx.ContentImageURL, ImageURL: &agentCtx.ImageURL{URL: "data:image/png;base64,abc"}},
|
||||
{Type: agentCtx.ContentText, Text: "description of the image"},
|
||||
}
|
||||
result := extractTextContent(parts)
|
||||
if result != "description of the image" {
|
||||
t.Errorf("got %q, want %q", result, "description of the image")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTextContent_Fallback(t *testing.T) {
|
||||
result := extractTextContent(12345)
|
||||
if result != "12345" {
|
||||
t.Errorf("got %q, want %q", result, "12345")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadBytes_HTTP(t *testing.T) {
|
||||
pngData := makePNG(50, 50)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
w.Write(pngData)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
data, err := readBytes(srv.URL + "/test.png")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(data) != len(pngData) {
|
||||
t.Errorf("got %d bytes, want %d", len(data), len(pngData))
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadBytes_HTTP404(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
_, err := readBytes(srv.URL + "/missing.png")
|
||||
if err == nil {
|
||||
t.Error("expected error for 404")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadBytes_DataURI(t *testing.T) {
|
||||
original := []byte("test data")
|
||||
b64 := base64.StdEncoding.EncodeToString(original)
|
||||
uri := "data:application/octet-stream;base64," + b64
|
||||
|
||||
data, err := readBytes(uri)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != string(original) {
|
||||
t.Errorf("got %q, want %q", data, original)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadBytes_UnsupportedScheme(t *testing.T) {
|
||||
_, err := readBytes("ftp://example.com/file.png")
|
||||
if err == nil {
|
||||
t.Error("expected error for unsupported scheme")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "unsupported") {
|
||||
t.Errorf("error should mention unsupported: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveImage_DataURI(t *testing.T) {
|
||||
pngData := makePNG(100, 80)
|
||||
b64 := base64.StdEncoding.EncodeToString(pngData)
|
||||
uri := "data:image/png;base64," + b64
|
||||
|
||||
result, err := resolveImage(uri, 1080)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasPrefix(result, "data:image/") {
|
||||
t.Errorf("expected data URI, got %q", result[:min(50, len(result))])
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveImage_HTTP(t *testing.T) {
|
||||
pngData := makePNG(200, 150)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
w.Write(pngData)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
result, err := resolveImage(srv.URL+"/img.png", 1080)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasPrefix(result, "data:image/") {
|
||||
t.Errorf("expected data URI, got %q", result[:min(50, len(result))])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHttpGet_Timeout(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Don't respond — let the client timeout
|
||||
select {}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
// Use a very short timeout for testing — override not possible with global func,
|
||||
// but the 30s timeout should handle normally. Just verify it returns eventually.
|
||||
// This test just verifies the function signature works.
|
||||
_, err := readBytes("http://invalid.host.that.does.not.exist.example.com/img.png")
|
||||
if err == nil {
|
||||
t.Error("expected error for unreachable host")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadBytes_WorkspaceInvalid(t *testing.T) {
|
||||
_, err := readBytes("workspace://no-slash-path")
|
||||
if err == nil {
|
||||
t.Error("expected error for invalid workspace URI")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadBytes_AttachInvalid(t *testing.T) {
|
||||
_, err := readBytes("attach://no-slash-path")
|
||||
if err == nil {
|
||||
t.Error("expected error for invalid attach URI")
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue