From af0d4edd746243c499111a62238e770e775d3803 Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 6 May 2026 10:59:35 +0800 Subject: [PATCH] feat(image): add image generation and enhanced reading capabilities - Introduced `image_generate` tool for generating images from text prompts, with options for specifying output file paths and image dimensions. - Updated `image_read` functionality to allow optional provider specification for enhanced image analysis. - Implemented new `GenerateImage` method in the LLM API for seamless integration of image generation capabilities. - Enhanced documentation to include detailed usage examples for both image reading and generation tools. - Updated tests to validate new image generation features and ensure robust functionality across image tools. --- agent/content/image/image.go | 4 +- agent/context/jsapi_llm.go | 54 ++++++++ agent/llm/doc.yml | 19 +++ agent/llm/image.go | 183 +++++++++++++++++++++++++++ agent/llm/image_test.go | 197 +++++++++++++++++++++++++++++ agent/llm/jsapi.go | 31 +++++ agent/llm/process.go | 50 ++++++++ agent/sandbox/v2/claude/command.go | 2 +- llmprovider/presets.yml | 65 +++++++++- tools/image/generate.go | 54 ++++++++ tools/image/generate_schema.json | 25 ++++ tools/image/generate_test.go | 35 +++++ tools/image/providers.go | 142 +++++++++++++++++++++ tools/image/providers_schema.json | 17 +++ tools/image/providers_test.go | 54 ++++++++ tools/image/read.go | 26 ++-- tools/image/read_schema.json | 6 +- tools/image/read_test.go | 8 +- tools/mcps/image.json | 8 +- tools/prompts/system-tools.md | 65 +++++++--- tools/skills/yao-image/SKILL.md | 58 ++++++++- tools/skills_test.go | 4 +- tools/tools.go | 14 +- 23 files changed, 1066 insertions(+), 55 deletions(-) create mode 100644 agent/llm/image.go create mode 100644 agent/llm/image_test.go create mode 100644 tools/image/generate.go create mode 100644 tools/image/generate_schema.json create mode 100644 tools/image/generate_test.go create mode 100644 tools/image/providers.go create mode 100644 tools/image/providers_schema.json create mode 100644 tools/image/providers_test.go diff --git a/agent/content/image/image.go b/agent/content/image/image.go index 2bf8f528..6a0ec868 100644 --- a/agent/content/image/image.go +++ b/agent/content/image/image.go @@ -12,7 +12,7 @@ import ( "github.com/yaoapp/yao/agent/output/message" searchTypes "github.com/yaoapp/yao/agent/search/types" "github.com/yaoapp/yao/attachment" - "github.com/yaoapp/yao/tools/vision" + toolsImage "github.com/yaoapp/yao/tools/image" ) // Image handles image content @@ -375,7 +375,7 @@ func (h *Image) readImageWithTools(ctx *agentContext.Context, content agentConte loadingID := h.sendLoading(ctx, i18n.T(ctx.Locale, "content.image.analyzing")) - resp, err := vision.ReadImage(ctx.Context, src, "Please describe this image in detail.", 1080, ctx.Authorized) + resp, err := toolsImage.ReadImage(ctx.Context, src, "Please describe this image in detail.", 1080, ctx.Authorized, "") h.sendLoadingDone(ctx, loadingID) diff --git a/agent/context/jsapi_llm.go b/agent/context/jsapi_llm.go index a187ef38..ae3e3f2f 100644 --- a/agent/context/jsapi_llm.go +++ b/agent/context/jsapi_llm.go @@ -14,6 +14,9 @@ type LlmAPI interface { // Returns *llm.Result or error information Stream(connector string, messages []interface{}, opts map[string]interface{}) interface{} + // GenerateImage generates an image from a text prompt using an image generation model + GenerateImage(connector string, prompt string, opts map[string]interface{}) interface{} + // Parallel LLM call methods - inspired by JavaScript Promise // All waits for all LLM calls to complete (like Promise.all) All(requests []interface{}) []interface{} @@ -68,6 +71,9 @@ func (ctx *Context) newLlmObject(iso *v8go.Isolate) *v8go.ObjectTemplate { // Single LLM call method llmObj.Set("Stream", ctx.llmStreamMethod(iso)) + // Image generation method + llmObj.Set("GenerateImage", ctx.llmGenerateImageMethod(iso)) + // Parallel LLM call methods - inspired by JavaScript Promise llmObj.Set("All", ctx.llmAllMethod(iso)) llmObj.Set("Any", ctx.llmAnyMethod(iso)) @@ -163,6 +169,54 @@ func (ctx *Context) llmStreamMethod(iso *v8go.Isolate) *v8go.FunctionTemplate { }) } +// llmGenerateImageMethod implements ctx.llm.GenerateImage(connector, prompt, options?) +// Usage: const result = ctx.llm.GenerateImage("dall-e-3", "A sunset over mountains", { size: "1024x1024" }) +// Returns: { connector, image (base64), format, error } +func (ctx *Context) llmGenerateImageMethod(iso *v8go.Isolate) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + v8ctx := info.Context() + args := info.Args() + + if len(args) < 2 { + return bridge.JsException(v8ctx, "GenerateImage requires connector and prompt parameters") + } + + if !args[0].IsString() { + return bridge.JsException(v8ctx, "connector must be a string") + } + connectorID := args[0].String() + + if !args[1].IsString() { + return bridge.JsException(v8ctx, "prompt must be a string") + } + prompt := args[1].String() + + var opts map[string]interface{} + if len(args) >= 3 && !args[2].IsUndefined() && !args[2].IsNull() { + goVal, err := bridge.GoValue(args[2], v8ctx) + if err == nil { + if optsMap, ok := goVal.(map[string]interface{}); ok { + opts = optsMap + } + } + } + + llmAPI := ctx.Llm() + if llmAPI == nil { + return bridge.JsException(v8ctx, "LLM API not available") + } + + result := llmAPI.GenerateImage(connectorID, prompt, opts) + + jsVal, err := bridge.JsValue(v8ctx, result) + if err != nil { + return bridge.JsException(v8ctx, "failed to convert result: "+err.Error()) + } + + return jsVal + }) +} + // llmAllMethod implements ctx.llm.All(requests, options?) // Usage: const results = ctx.llm.All([ // diff --git a/agent/llm/doc.yml b/agent/llm/doc.yml index a938bbba..9a6e38cd 100644 --- a/agent/llm/doc.yml +++ b/agent/llm/doc.yml @@ -23,3 +23,22 @@ entries: return: type: object desc: "OpenAI-compatible response: { id, object, created, model, choices: [{ index, message: { role, content, tool_calls? }, finish_reason }], usage? }" + + - name: ImageGeneration + desc: Generate an image from a text prompt using an image generation model + args: + - name: connector + type: string + required: true + desc: Connector ID for an image generation model (e.g. dall-e-3) + - name: prompt + type: string + required: true + desc: Text description of the image to generate + - name: opts + type: object + required: false + desc: "Generation options: size (1024x1024), quality, style, n, etc." + return: + type: object + desc: "Image generation result: { image (base64), format (png) }" diff --git a/agent/llm/image.go b/agent/llm/image.go new file mode 100644 index 00000000..c70b1410 --- /dev/null +++ b/agent/llm/image.go @@ -0,0 +1,183 @@ +package llm + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/yaoapp/gou/connector" + gouhttp "github.com/yaoapp/gou/http" + goullm "github.com/yaoapp/gou/llm" +) + +// ImageGenResponse holds the result of an image generation call. +// Image is always base64 encoded; if the provider returns a URL, it is downloaded and converted. +type ImageGenResponse struct { + Image string `json:"image"` // base64 encoded image data + Format string `json:"format"` // image format, e.g. "png", "jpeg" +} + +// GenerateImage calls the /images/generations endpoint through the connector. +// options may include: size, n, quality, style, model, etc. +func GenerateImage(conn connector.Connector, prompt string, options map[string]interface{}) (*ImageGenResponse, error) { + host, key, authMode := resolveConnSettings(conn) + if host == "" { + return nil, fmt.Errorf("no host found in connector settings") + } + if key == "" { + return nil, fmt.Errorf("API key is not set") + } + + if options == nil { + options = map[string]interface{}{} + } + options["prompt"] = prompt + if _, ok := options["model"]; !ok { + if lc, ok := conn.(goullm.LLMConnector); ok { + if m := lc.GetModel(); m != "" { + options["model"] = m + } + } + } + + url := connector.BuildAPIURL(host, "/images/generations") + req := gouhttp.New(url) + req.SetHeader("Content-Type", "application/json") + setImageAuthHeaders(req, authMode, key) + + resp := req.Post(options) + if resp.Status != 200 { + errMsg := extractAPIError(resp.Data) + return nil, fmt.Errorf("image generation failed (status %d, url %s): %s", resp.Status, url, errMsg) + } + + return extractImageFromResponse(resp.Data) +} + +func resolveConnSettings(conn connector.Connector) (host, key string, authMode goullm.AuthMode) { + authMode = goullm.AuthBearer + if lc, ok := conn.(goullm.LLMConnector); ok { + host = lc.GetURL() + key = lc.GetKey() + authMode = lc.GetAuthMode() + } + if host == "" || key == "" { + setting := conn.Setting() + if host == "" { + host, _ = setting["host"].(string) + } + if key == "" { + key, _ = setting["key"].(string) + } + } + return +} + +func setImageAuthHeaders(req *gouhttp.Request, authMode goullm.AuthMode, key string) { + switch authMode { + case goullm.AuthAPIKey: + req.SetHeader("api-key", key) + case goullm.AuthXAPIKey: + req.SetHeader("x-api-key", key) + default: + req.SetHeader("Authorization", fmt.Sprintf("Bearer %s", key)) + } +} + +func extractImageFromResponse(data interface{}) (*ImageGenResponse, error) { + raw, err := json.Marshal(data) + if err != nil { + return nil, fmt.Errorf("marshal response: %w", err) + } + + var parsed struct { + Data []struct { + B64JSON *string `json:"b64_json"` + URL *string `json:"url"` + } `json:"data"` + } + if err := json.Unmarshal(raw, &parsed); err != nil { + return nil, fmt.Errorf("unmarshal response: %w", err) + } + if len(parsed.Data) == 0 { + return nil, fmt.Errorf("provider returned empty data array, no image was generated") + } + + item := parsed.Data[0] + + if item.B64JSON != nil && *item.B64JSON != "" { + return &ImageGenResponse{Image: *item.B64JSON, Format: "png"}, nil + } + + if item.URL != nil && *item.URL != "" { + b64, format, err := downloadImageAsBase64(*item.URL) + if err != nil { + return nil, fmt.Errorf("provider returned url but download failed: %w", err) + } + return &ImageGenResponse{Image: b64, Format: format}, nil + } + + return nil, fmt.Errorf("provider returned data but neither b64_json nor url field is present, the model may not support image generation") +} + +func downloadImageAsBase64(imageURL string) (b64 string, format string, err error) { + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Get(imageURL) + if err != nil { + return "", "", fmt.Errorf("http get: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return "", "", fmt.Errorf("download returned status %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", "", fmt.Errorf("read body: %w", err) + } + if len(body) == 0 { + return "", "", fmt.Errorf("downloaded image is empty") + } + + format = "png" + ct := resp.Header.Get("Content-Type") + switch { + case strings.Contains(ct, "jpeg") || strings.Contains(ct, "jpg"): + format = "jpeg" + case strings.Contains(ct, "webp"): + format = "webp" + case strings.Contains(ct, "gif"): + format = "gif" + default: + if strings.Contains(imageURL, ".jpeg") || strings.Contains(imageURL, ".jpg") { + format = "jpeg" + } else if strings.Contains(imageURL, ".webp") { + format = "webp" + } + } + + b64 = base64.StdEncoding.EncodeToString(body) + return b64, format, nil +} + +func extractAPIError(data interface{}) string { + raw, err := json.Marshal(data) + if err != nil { + return fmt.Sprintf("%v", data) + } + + var parsed struct { + Error struct { + Message string `json:"message"` + } `json:"error"` + } + if err := json.Unmarshal(raw, &parsed); err == nil && parsed.Error.Message != "" { + return parsed.Error.Message + } + return string(raw) +} diff --git a/agent/llm/image_test.go b/agent/llm/image_test.go new file mode 100644 index 00000000..0783ff4d --- /dev/null +++ b/agent/llm/image_test.go @@ -0,0 +1,197 @@ +package llm + +import ( + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestExtractImageFromResponse_B64(t *testing.T) { + data := map[string]interface{}{ + "data": []interface{}{ + map[string]interface{}{ + "b64_json": "iVBORw0KGgoAAAANS...", + }, + }, + } + resp, err := extractImageFromResponse(data) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if resp.Image != "iVBORw0KGgoAAAANS..." { + t.Errorf("got Image=%q, want %q", resp.Image, "iVBORw0KGgoAAAANS...") + } + if resp.Format != "png" { + t.Errorf("got Format=%q, want %q", resp.Format, "png") + } +} + +func TestExtractImageFromResponse_URL(t *testing.T) { + fakeImage := []byte{0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10} // fake JPEG header bytes + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/jpeg") + w.Write(fakeImage) + })) + defer srv.Close() + + data := map[string]interface{}{ + "data": []interface{}{ + map[string]interface{}{ + "b64_json": nil, + "url": srv.URL + "/image_0.jpeg", + }, + }, + } + resp, err := extractImageFromResponse(data) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + expected := base64.StdEncoding.EncodeToString(fakeImage) + if resp.Image != expected { + t.Errorf("got Image=%q, want %q", resp.Image, expected) + } + if resp.Format != "jpeg" { + t.Errorf("got Format=%q, want %q", resp.Format, "jpeg") + } +} + +func TestExtractImageFromResponse_URLPng(t *testing.T) { + fakeImage := []byte{0x89, 0x50, 0x4E, 0x47} // PNG magic bytes + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + w.Write(fakeImage) + })) + defer srv.Close() + + data := map[string]interface{}{ + "data": []interface{}{ + map[string]interface{}{ + "url": srv.URL + "/output.png", + }, + }, + } + resp, err := extractImageFromResponse(data) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if resp.Format != "png" { + t.Errorf("got Format=%q, want %q", resp.Format, "png") + } + if resp.Image == "" { + t.Error("expected non-empty base64 Image") + } +} + +func TestExtractImageFromResponse_URLDownloadFail(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + data := map[string]interface{}{ + "data": []interface{}{ + map[string]interface{}{ + "url": srv.URL + "/missing.png", + }, + }, + } + _, err := extractImageFromResponse(data) + if err == nil { + t.Error("expected error for failed download") + } +} + +func TestExtractImageFromResponse_Empty(t *testing.T) { + data := map[string]interface{}{ + "data": []interface{}{}, + } + _, err := extractImageFromResponse(data) + if err == nil { + t.Error("expected error for empty data array") + } +} + +func TestExtractImageFromResponse_NoData(t *testing.T) { + data := map[string]interface{}{} + _, err := extractImageFromResponse(data) + if err == nil { + t.Error("expected error for missing data field") + } +} + +func TestExtractImageFromResponse_NullBoth(t *testing.T) { + data := map[string]interface{}{ + "data": []interface{}{ + map[string]interface{}{ + "b64_json": nil, + "url": nil, + }, + }, + } + _, err := extractImageFromResponse(data) + if err == nil { + t.Error("expected error when both b64_json and url are null") + } +} + +func TestDownloadImageAsBase64(t *testing.T) { + payload := []byte("fake-png-data") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + w.Write(payload) + })) + defer srv.Close() + + b64, format, err := downloadImageAsBase64(srv.URL + "/test.png") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if format != "png" { + t.Errorf("got format=%q, want %q", format, "png") + } + decoded, _ := base64.StdEncoding.DecodeString(b64) + if string(decoded) != string(payload) { + t.Errorf("decoded content mismatch") + } +} + +func TestDownloadImageAsBase64_FormatFromURL(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + w.Write([]byte("data")) + })) + defer srv.Close() + + _, format, err := downloadImageAsBase64(srv.URL + "/image.webp") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if format != "webp" { + t.Errorf("got format=%q, want %q (from URL fallback)", format, "webp") + } +} + +func TestExtractAPIError_WithMessage(t *testing.T) { + data := map[string]interface{}{ + "error": map[string]interface{}{ + "message": "insufficient quota", + }, + } + msg := extractAPIError(data) + if msg != "insufficient quota" { + t.Errorf("got %q, want %q", msg, "insufficient quota") + } +} + +func TestExtractAPIError_NoMessage(t *testing.T) { + data := map[string]interface{}{ + "something": "else", + } + msg := extractAPIError(data) + raw, _ := json.Marshal(data) + if msg != string(raw) { + t.Errorf("got %q, want raw JSON fallback", msg) + } +} diff --git a/agent/llm/jsapi.go b/agent/llm/jsapi.go index 2439dd7f..33790bf7 100644 --- a/agent/llm/jsapi.go +++ b/agent/llm/jsapi.go @@ -31,6 +31,37 @@ func SetJSAPIFactory() { } } +// GenerateImage implements LlmAPI.GenerateImage - generates an image from a text prompt +func (api *JSAPI) GenerateImage(connectorID string, prompt string, opts map[string]interface{}) interface{} { + result := &ImageGenResult{ + Connector: connectorID, + } + + conn, err := connector.Select(connectorID) + if err != nil { + result.Error = fmt.Sprintf("failed to select connector %s: %v", connectorID, err) + return result + } + + resp, err := GenerateImage(conn, prompt, opts) + if err != nil { + result.Error = fmt.Sprintf("image generation failed: %v", err) + return result + } + + result.Image = resp.Image + result.Format = resp.Format + return result +} + +// ImageGenResult is the return type for GenerateImage JSAPI +type ImageGenResult struct { + Connector string `json:"connector"` + Image string `json:"image,omitempty"` + Format string `json:"format,omitempty"` + Error string `json:"error,omitempty"` +} + // Stream implements LlmAPI.Stream - calls LLM with streaming output to ctx.Writer func (api *JSAPI) Stream(connectorID string, messages []interface{}, opts map[string]interface{}) interface{} { return api.StreamWithHandler(connectorID, messages, opts, nil) diff --git a/agent/llm/process.go b/agent/llm/process.go index 0bd7fb04..e0403b74 100644 --- a/agent/llm/process.go +++ b/agent/llm/process.go @@ -16,6 +16,7 @@ import ( func init() { process.Register("llm.ChatCompletions", ProcessChatCompletions) + process.Register("llm.ImageGeneration", ProcessImageGeneration) } // ProcessChatCompletions implements the llm.ChatCompletions Process. @@ -155,6 +156,55 @@ func ProcessChatCompletions(p *process.Process) interface{} { return toOpenAIFormat(response) } +// ProcessImageGeneration implements the llm.ImageGeneration Process. +// +// Usage: +// +// Process("llm.ImageGeneration", connectorID, prompt) +// Process("llm.ImageGeneration", connectorID, prompt, opts) +// +// Args: +// - connectorID (string): Connector ID for an image generation model +// - prompt (string): Text description of the image to generate +// - opts (map): Optional. size, quality, style, n, etc. +// +// Returns: { image (base64), format (png) } +func ProcessImageGeneration(p *process.Process) interface{} { + p.ValidateArgNums(2) + + connectorID := p.ArgsString(0) + if connectorID == "" { + return newErrorResponse("llm.ImageGeneration: connector is required") + } + + prompt := p.ArgsString(1) + if prompt == "" { + return newErrorResponse("llm.ImageGeneration: prompt is required") + } + + var opts map[string]interface{} + if p.NumOfArgs() > 2 && p.Args[2] != nil { + if o, ok := p.Args[2].(map[string]interface{}); ok { + opts = o + } + } + + conn, _, err := selectWithCapabilities(connectorID) + if err != nil { + return newErrorResponse(fmt.Sprintf("llm.ImageGeneration: connector %s not found: %v", connectorID, err)) + } + + resp, err := GenerateImage(conn, prompt, opts) + if err != nil { + return newErrorResponse(fmt.Sprintf("llm.ImageGeneration: %v", err)) + } + + return map[string]interface{}{ + "image": resp.Image, + "format": resp.Format, + } +} + // toOpenAIFormat converts CompletionResponse to OpenAI chat.completions format // for backward compatibility with code that consumed openai.chat.Completions. func toOpenAIFormat(resp *agentContext.CompletionResponse) map[string]interface{} { diff --git a/agent/sandbox/v2/claude/command.go b/agent/sandbox/v2/claude/command.go index b92732d6..f2647f8a 100644 --- a/agent/sandbox/v2/claude/command.go +++ b/agent/sandbox/v2/claude/command.go @@ -388,7 +388,7 @@ func buildModelCapabilityPrompt(req *types.StreamRequest) string { if !primaryHasVision { if _, hasVisionRole := req.Roles["vision"]; hasVisionRole { guidance = append(guidance, - "**Image/Vision**: Your current model cannot process images directly. Use the `image_read` system tool (`tai tool image_read`) to analyze images — see the yao-vision skill for details", + "**Image/Vision**: Your current model cannot process images directly. Use the `image_read` system tool (`tai tool image_read`) to analyze images — see the yao-image skill for details", ) } } diff --git a/llmprovider/presets.yml b/llmprovider/presets.yml index 014a2302..e24640de 100644 --- a/llmprovider/presets.yml +++ b/llmprovider/presets.yml @@ -87,7 +87,7 @@ enabled: false - id: tencent-hy3-preview model: tencent/hy3-preview:free - name: 混元 3 Preview (Free) + name: Hunyuan 3 Preview (Free) max_input_tokens: 262144 max_output_tokens: 262144 capabilities: [tool_calls, streaming, json] @@ -97,7 +97,7 @@ enabled: false - id: tencent-hy3-preview-reasoning-high model: tencent/hy3-preview:free - name: 混元 3 Preview Reasoning (Free) + name: Hunyuan 3 Preview Reasoning (Free) max_input_tokens: 262144 max_output_tokens: 262144 capabilities: [tool_calls, streaming, json, reasoning] @@ -193,7 +193,7 @@ enabled: false # ─── OpenAI ───────────────────────────────────────────── -# reasoning_effort 档位: low | medium | high (5.4-mini/o4-mini) +# reasoning_effort levels: low | medium | high (5.4-mini/o4-mini) # low | medium | high | xhigh (5.5/5.4/codex/5) - key: openai name: OpenAI @@ -471,6 +471,23 @@ name: TTS-1 HD capabilities: [audio] enabled: false + # --- Image Generation --- + - id: gpt-image-2 + name: GPT Image 2 + capabilities: [image_generation] + enabled: false + - id: gpt-image-1.5 + name: GPT Image 1.5 + capabilities: [image_generation] + enabled: false + - id: gpt-image-1 + name: GPT Image 1 + capabilities: [image_generation] + enabled: false + - id: gpt-image-1-mini + name: GPT Image 1 Mini + capabilities: [image_generation] + enabled: false # ─── DeepSeek (OpenAI) ────────────────────────────────── - key: deepseek @@ -856,6 +873,22 @@ max_output_tokens: 65536 capabilities: [vision, tool_calls, streaming, json] enabled: false + # --- Image Generation --- + - id: imagen-4 + model: models/imagen-4.0-generate-001 + name: Imagen 4 + capabilities: [image_generation] + enabled: false + - id: imagen-4-ultra + model: models/imagen-4.0-ultra-generate-001 + name: Imagen 4 Ultra + capabilities: [image_generation] + enabled: false + - id: imagen-4-fast + model: models/imagen-4.0-fast-generate-001 + name: Imagen 4 Fast + capabilities: [image_generation] + enabled: false # ─── xAI (Grok) ──────────────────────────────────────── # grok-4.x: reasoning.enabled true|false @@ -921,6 +954,19 @@ options: reasoning_effort: high enabled: false + # --- Image Generation --- + - id: grok-imagine-image + name: Grok Imagine + capabilities: [image_generation] + enabled: false + - id: grok-imagine-image-pro + name: Grok Imagine Pro + capabilities: [image_generation] + enabled: false + - id: grok-imagine-image-quality + name: Grok Imagine Quality + capabilities: [image_generation] + enabled: false # ─── MiniMax (International) ──────────────────────────── - key: minimax_intl @@ -1390,6 +1436,19 @@ max_input_tokens: 4096 capabilities: [embedding] enabled: false + # --- 图片生成 --- + - id: doubao-seedream-5-0-260128 + name: 豆包 Seedream 5.0 + capabilities: [image_generation] + enabled: false + - id: doubao-seedream-4-5-251128 + name: 豆包 Seedream 4.5 + capabilities: [image_generation] + enabled: false + - id: doubao-seedream-4-0-250828 + name: 豆包 Seedream 4.0 + capabilities: [image_generation] + enabled: false # ─── 腾讯混元 MaaS ───────────────────────────────────── # hy3-preview: reasoning.level disabled | low | high diff --git a/tools/image/generate.go b/tools/image/generate.go new file mode 100644 index 00000000..3ab2329f --- /dev/null +++ b/tools/image/generate.go @@ -0,0 +1,54 @@ +package image + +import ( + _ "embed" + "fmt" + + "github.com/yaoapp/gou/process" + agentLLM "github.com/yaoapp/yao/agent/llm" + "github.com/yaoapp/yao/openapi/oauth/authorized" +) + +//go:embed generate_schema.json +var GenerateSchemaJSON []byte + +// GenerateHandler is the tools.image_generate process handler. +func GenerateHandler(proc *process.Process) interface{} { + prompt := proc.ArgsString(0) + if prompt == "" { + return map[string]interface{}{"error": "prompt is required"} + } + + provider := proc.ArgsString(1) + size := proc.ArgsString(2, "1024x1024") + + authInfo := authorized.ProcessAuthInfo(proc) + if authInfo == nil { + return map[string]interface{}{"error": "unauthorized: no auth info in request"} + } + + connectorID := provider + if connectorID == "" { + connectorID = findFirstImageGenConnector(authInfo) + if connectorID == "" { + return map[string]interface{}{"error": "no image generation provider available; configure one or specify a provider"} + } + } + + conn, _, err := agentLLM.ResolveConnector(connectorID, authInfo) + if err != nil { + return map[string]interface{}{"error": fmt.Sprintf("resolve connector: %v", err)} + } + + options := map[string]interface{}{"size": size} + resp, err := agentLLM.GenerateImage(conn, prompt, options) + if err != nil { + return map[string]interface{}{"error": fmt.Sprintf("image generation failed: %v", err)} + } + + return map[string]interface{}{ + "image": resp.Image, + "format": resp.Format, + "size": size, + } +} diff --git a/tools/image/generate_schema.json b/tools/image/generate_schema.json new file mode 100644 index 00000000..581cec40 --- /dev/null +++ b/tools/image/generate_schema.json @@ -0,0 +1,25 @@ +{ + "name": "image_generate", + "description": "Generate an image from a text prompt using an image generation model (e.g. DALL-E, Seedream).", + "process": "tools.image_generate", + "inputSchema": { + "type": "object", + "properties": { + "prompt": { + "type": "string", + "description": "Text description of the image to generate." + }, + "provider": { + "type": "string", + "description": "Provider connector ID (e.g. 'llm.my-openai:dall-e-3'). Use image_providers to list available options. If omitted, the first available image generation provider is used." + }, + "size": { + "type": "string", + "description": "Image dimensions (default: 1024x1024). Common values: 1024x1024, 1024x1792, 1792x1024.", + "default": "1024x1024" + } + }, + "required": ["prompt"] + }, + "x-process-args": ["$args.prompt", "$args.provider", "$args.size"] +} diff --git a/tools/image/generate_test.go b/tools/image/generate_test.go new file mode 100644 index 00000000..1724a50d --- /dev/null +++ b/tools/image/generate_test.go @@ -0,0 +1,35 @@ +package image + +import ( + "testing" + + "github.com/yaoapp/gou/process" +) + +func TestGenerateHandler_NoPrompt(t *testing.T) { + proc := &process.Process{ + Args: []interface{}{""}, + } + result := GenerateHandler(proc) + m, ok := result.(map[string]interface{}) + if !ok { + t.Fatal("expected map result") + } + if errMsg, _ := m["error"].(string); errMsg != "prompt is required" { + t.Errorf("expected 'prompt is required', got %q", errMsg) + } +} + +func TestGenerateHandler_NoAuth(t *testing.T) { + proc := &process.Process{ + Args: []interface{}{"A sunset", "", "1024x1024"}, + } + result := GenerateHandler(proc) + m, ok := result.(map[string]interface{}) + if !ok { + t.Fatal("expected map result") + } + if _, hasErr := m["error"]; !hasErr { + t.Error("expected error when no auth info") + } +} diff --git a/tools/image/providers.go b/tools/image/providers.go new file mode 100644 index 00000000..6be85add --- /dev/null +++ b/tools/image/providers.go @@ -0,0 +1,142 @@ +package image + +import ( + _ "embed" + + "github.com/yaoapp/gou/process" + "github.com/yaoapp/yao/llmprovider" + "github.com/yaoapp/yao/openapi/oauth/authorized" + oauthTypes "github.com/yaoapp/yao/openapi/oauth/types" +) + +//go:embed providers_schema.json +var ProvidersSchemaJSON []byte + +type providerResult struct { + Key string `json:"key"` + Name string `json:"name"` + Models []modelResult `json:"models"` +} + +type modelResult struct { + ID string `json:"id"` + Name string `json:"name"` + ConnectorID string `json:"connector_id"` +} + +// ProvidersHandler is the tools.image_providers process handler. +func ProvidersHandler(proc *process.Process) interface{} { + capability := proc.ArgsString(0, "image_generation") + + authInfo := authorized.ProcessAuthInfo(proc) + if authInfo == nil { + return map[string]interface{}{"error": "unauthorized: no auth info in request"} + } + + if llmprovider.Global == nil { + return map[string]interface{}{"error": "llmprovider registry not initialized"} + } + + providers, err := listProvidersByCapability(capability, authInfo) + if err != nil { + return map[string]interface{}{"error": err.Error()} + } + + return map[string]interface{}{ + "capability": capability, + "providers": providers, + } +} + +// listProvidersByCapability returns providers matching the given capability, +// filtered by owner scope (builtin always included, dynamic filtered by auth). +func listProvidersByCapability(capability string, authInfo *oauthTypes.AuthorizedInfo) ([]providerResult, error) { + enabled := true + filter := &llmprovider.ProviderFilter{ + Capabilities: []string{capability}, + Enabled: &enabled, + Source: llmprovider.ProviderSourceAll, + } + + allProviders, err := llmprovider.Global.List(filter) + if err != nil { + return nil, err + } + + var results []providerResult + for _, p := range allProviders { + if p.Source == llmprovider.ProviderSourceDynamic && authInfo != nil { + if !dynamicOwnerMatch(&p.Owner, authInfo) { + continue + } + } + + var models []modelResult + for _, m := range p.Models { + if !m.Enabled { + continue + } + if !modelHasCapability(m.Capabilities, capability) { + continue + } + + cid := p.ConnectorID + if p.Source == llmprovider.ProviderSourceDynamic && m.ID != "" { + cid = p.ConnectorID + ":" + m.ID + } + + name := m.Name + if name == "" { + name = m.ID + } + models = append(models, modelResult{ + ID: m.ID, + Name: name, + ConnectorID: cid, + }) + } + + if len(models) == 0 { + continue + } + + results = append(results, providerResult{ + Key: p.Key, + Name: p.Name, + Models: models, + }) + } + return results, nil +} + +// findFirstImageGenConnector returns the connector ID of the first available +// image generation provider, or empty string if none found. +func findFirstImageGenConnector(authInfo *oauthTypes.AuthorizedInfo) string { + if llmprovider.Global == nil { + return "" + } + providers, err := listProvidersByCapability("image_generation", authInfo) + if err != nil || len(providers) == 0 { + return "" + } + if len(providers[0].Models) == 0 { + return "" + } + return providers[0].Models[0].ConnectorID +} + +func dynamicOwnerMatch(owner *llmprovider.ProviderOwner, authInfo *oauthTypes.AuthorizedInfo) bool { + if authInfo.GetTeamID() != "" { + return owner.Type == "team" && owner.TeamID == authInfo.GetTeamID() + } + return owner.Type == "user" && owner.UserID == authInfo.GetUserID() +} + +func modelHasCapability(caps []string, target string) bool { + for _, c := range caps { + if c == target { + return true + } + } + return false +} diff --git a/tools/image/providers_schema.json b/tools/image/providers_schema.json new file mode 100644 index 00000000..fbaedfa0 --- /dev/null +++ b/tools/image/providers_schema.json @@ -0,0 +1,17 @@ +{ + "name": "image_providers", + "description": "List available image providers. Default lists image generation providers; pass capability='vision' to list vision (image reading) providers.", + "process": "tools.image_providers", + "inputSchema": { + "type": "object", + "properties": { + "capability": { + "type": "string", + "description": "Filter by capability: 'image_generation' (default) or 'vision'.", + "default": "image_generation", + "enum": ["image_generation", "vision"] + } + } + }, + "x-process-args": ["$args.capability"] +} diff --git a/tools/image/providers_test.go b/tools/image/providers_test.go new file mode 100644 index 00000000..f1729f6e --- /dev/null +++ b/tools/image/providers_test.go @@ -0,0 +1,54 @@ +package image + +import ( + "testing" + + "github.com/yaoapp/gou/process" +) + +func TestModelHasCapability_Found(t *testing.T) { + caps := []string{"chat", "image_generation", "vision"} + if !modelHasCapability(caps, "image_generation") { + t.Error("expected true for image_generation") + } + if !modelHasCapability(caps, "vision") { + t.Error("expected true for vision") + } +} + +func TestModelHasCapability_NotFound(t *testing.T) { + caps := []string{"chat", "embedding"} + if modelHasCapability(caps, "image_generation") { + t.Error("expected false for image_generation") + } +} + +func TestModelHasCapability_Empty(t *testing.T) { + if modelHasCapability(nil, "image_generation") { + t.Error("expected false for nil caps") + } + if modelHasCapability([]string{}, "image_generation") { + t.Error("expected false for empty caps") + } +} + +func TestProvidersHandler_NoAuth(t *testing.T) { + proc := &process.Process{ + Args: []interface{}{"image_generation"}, + } + result := ProvidersHandler(proc) + m, ok := result.(map[string]interface{}) + if !ok { + t.Fatal("expected map result") + } + if _, hasErr := m["error"]; !hasErr { + t.Error("expected error when no auth info") + } +} + +func TestFindFirstImageGenConnector_NoGlobal(t *testing.T) { + result := findFirstImageGenConnector(nil) + if result != "" { + t.Errorf("expected empty string, got %q", result) + } +} diff --git a/tools/image/read.go b/tools/image/read.go index ab5a991a..08633707 100644 --- a/tools/image/read.go +++ b/tools/image/read.go @@ -1,4 +1,4 @@ -package vision +package image import ( "bytes" @@ -6,7 +6,7 @@ import ( _ "embed" "encoding/base64" "fmt" - "image" + stdimage "image" _ "image/gif" "image/jpeg" _ "image/png" @@ -28,8 +28,8 @@ import ( ws "github.com/yaoapp/yao/workspace" ) -//go:embed schema.json -var SchemaJSON []byte +//go:embed read_schema.json +var ReadSchemaJSON []byte // ImageReadResponse is the return type for image_read. type ImageReadResponse struct { @@ -40,7 +40,8 @@ type ImageReadResponse struct { // ReadImage reads and analyzes an image using the vision model. // It resolves the image from src, finds a vision-capable connector via llmprovider, // and returns the model's text description. -func ReadImage(goCtx context.Context, src string, prompt string, maxSize int, authInfo *oauthTypes.AuthorizedInfo) (*ImageReadResponse, error) { +// provider is optional; when non-empty it overrides the default "use::vision" role. +func ReadImage(goCtx context.Context, src string, prompt string, maxSize int, authInfo *oauthTypes.AuthorizedInfo, provider string) (*ImageReadResponse, error) { if prompt == "" { prompt = "Please describe this image in detail." } @@ -53,7 +54,11 @@ func ReadImage(goCtx context.Context, src string, prompt string, maxSize int, au return nil, fmt.Errorf("resolve image: %w", err) } - conn, caps, err := agentLLM.ResolveConnector("use::vision", authInfo) + connectorRole := "use::vision" + if provider != "" { + connectorRole = provider + } + conn, caps, err := agentLLM.ResolveConnector(connectorRole, authInfo) if err != nil { return nil, fmt.Errorf("resolve vision connector: %w", err) } @@ -87,8 +92,8 @@ func ReadImage(goCtx context.Context, src string, prompt string, maxSize int, au }, nil } -// Handler is the tools.image_read process handler. -func Handler(proc *process.Process) interface{} { +// ReadHandler is the tools.image_read process handler. +func ReadHandler(proc *process.Process) interface{} { src := proc.ArgsString(0) if src == "" { return map[string]interface{}{"error": "image_path is required: provide a file path, URL, or URI"} @@ -103,6 +108,7 @@ func Handler(proc *process.Process) interface{} { proc.ArgsString(1, "Please describe this image in detail."), proc.ArgsInt(2, 1080), authInfo, + proc.ArgsString(3), ) if err != nil { return map[string]interface{}{"error": err.Error()} @@ -188,7 +194,7 @@ func httpGet(rawURL string) ([]byte, error) { // resizeImage decodes, resizes (longest edge <= maxSize), re-encodes as JPEG. // Returns original bytes unchanged when already small enough or if decode fails. func resizeImage(data []byte, maxSize int) ([]byte, string) { - img, _, err := image.Decode(bytes.NewReader(data)) + img, _, err := stdimage.Decode(bytes.NewReader(data)) if err != nil { return data, http.DetectContentType(data) } @@ -205,7 +211,7 @@ func resizeImage(data []byte, maxSize int) ([]byte, string) { if newH < 1 { newH = 1 } - dst := image.NewRGBA(image.Rect(0, 0, newW, newH)) + dst := stdimage.NewRGBA(stdimage.Rect(0, 0, newW, newH)) draw.BiLinear.Scale(dst, dst.Bounds(), img, bounds, draw.Over, nil) var buf bytes.Buffer if err := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: 85}); err != nil { diff --git a/tools/image/read_schema.json b/tools/image/read_schema.json index 2c07efd6..d413da15 100644 --- a/tools/image/read_schema.json +++ b/tools/image/read_schema.json @@ -18,9 +18,13 @@ "type": "integer", "description": "Max dimension in pixels for the longest edge. Image is resized (preserving aspect ratio) before sending to the vision model. Default 1080.", "default": 1080 + }, + "provider": { + "type": "string", + "description": "Vision provider connector ID (e.g. from image_providers with capability='vision'). If omitted, uses the default vision model." } }, "required": ["image_path"] }, - "x-process-args": ["$args.image_path", "$args.prompt", "$args.max_size"] + "x-process-args": ["$args.image_path", "$args.prompt", "$args.max_size", "$args.provider"] } diff --git a/tools/image/read_test.go b/tools/image/read_test.go index 246723e1..13e087cc 100644 --- a/tools/image/read_test.go +++ b/tools/image/read_test.go @@ -1,8 +1,8 @@ -package vision +package image import ( "encoding/base64" - "image" + stdimage "image" "image/color" _ "image/jpeg" "image/png" @@ -15,7 +15,7 @@ import ( ) func makePNG(w, h int) []byte { - img := image.NewRGBA(image.Rect(0, 0, w, h)) + img := stdimage.NewRGBA(stdimage.Rect(0, 0, w, h)) for y := 0; y < h; y++ { for x := 0; x < w; x++ { img.Set(x, y, color.RGBA{R: 255, G: 0, B: 0, A: 255}) @@ -45,7 +45,7 @@ func TestResizeImage_LargeImage(t *testing.T) { if mime != "image/jpeg" { t.Errorf("mime = %q, want image/jpeg", mime) } - img, _, err := image.Decode(strings.NewReader(string(data))) + img, _, err := stdimage.Decode(strings.NewReader(string(data))) if err != nil { t.Fatalf("failed to decode resized image: %v", err) } diff --git a/tools/mcps/image.json b/tools/mcps/image.json index b96e4df4..7bf7258e 100644 --- a/tools/mcps/image.json +++ b/tools/mcps/image.json @@ -1,8 +1,10 @@ { - "name": "yao-vision", + "name": "yao-image", "transport": "process", - "description": "Vision tools for image understanding", + "description": "Image tools for reading, generating, and managing images", "tools": { - "image_read": "tools.image_read" + "image_read": "tools.image_read", + "image_generate": "tools.image_generate", + "image_providers": "tools.image_providers" } } diff --git a/tools/prompts/system-tools.md b/tools/prompts/system-tools.md index 5aa799e1..3020953a 100644 --- a/tools/prompts/system-tools.md +++ b/tools/prompts/system-tools.md @@ -4,13 +4,13 @@ These environment variables are set by the Yao sandbox. **Always use these variables — never hardcode paths.** -| Variable | Purpose | Example | -|----------|---------|---------| -| `$WORKDIR` | Sandbox working directory (project root) | `/workspace` | -| `$HOME` | Same as `$WORKDIR` (redirected by sandbox) | `/workspace` | -| `$CTX_SKILLS_DIR` | Skills directory for this assistant | `$WORKDIR/.yao/assistants//skills` | -| `$CTX_ASSISTANT_ID` | Current assistant ID | `yao.agent-smith` | -| `$CTX_WORKSPACE_ID` | Current workspace ID | `ws-abc123` | +| Variable | Purpose | Example | +| ------------------- | ------------------------------------------ | -------------------------------------- | +| `$WORKDIR` | Sandbox working directory (project root) | `/workspace` | +| `$HOME` | Same as `$WORKDIR` (redirected by sandbox) | `/workspace` | +| `$CTX_SKILLS_DIR` | Skills directory for this assistant | `$WORKDIR/.yao/assistants//skills` | +| `$CTX_ASSISTANT_ID` | Current assistant ID | `yao.agent-smith` | +| `$CTX_WORKSPACE_ID` | Current workspace ID | `ws-abc123` | ### Path Rules @@ -21,6 +21,33 @@ These environment variables are set by the Yao sandbox. **Always use these varia Resolve first: `echo "$WORKDIR"`, then use the printed value. - On Windows, use `$env:WORKDIR` / `$env:CTX_SKILLS_DIR` syntax instead. +### Workspace Path in Replies + +When replying to users, **never expose raw `/workspace/...` paths**. Rewrite them using the `workspace://` scheme so the frontend can render them correctly. + +**Format**: `workspace:///relative/path` + +**Step 1 — resolve the real workspace ID** (do this once per session): + +```bash +echo "$CTX_WORKSPACE_ID" +``` + +Use the **actual printed value** (e.g. `ws-bfc4c2de-b53...`) in all subsequent replies. + +**Step 2 — rewrite paths in replies**: + +- `/workspace/output/result.png` → `workspace:///output/result.png` + +**Common mistakes** (all wrong): + +- `workspace://$CTX_WORKSPACE_ID/...` ← shell variable literally in reply +- `workspace://ws-bfc4c2de-b53/...` ← example/placeholder ID instead of the real one +- `workspace:///...` ← template placeholder instead of the real one +- `/workspace/output/...` ← raw path without `workspace://` scheme + +You **must** run `echo "$CTX_WORKSPACE_ID"` and use the exact output. + ### Attachments User-uploaded files are placed in `$WORKDIR/.attachments/{chatID}/`. @@ -40,15 +67,17 @@ You have access to Yao system tools via the `tai` command in bash. **Calling convention**: `tai tool ''` -| Tool | Skill (auto-loaded) | Description | -|------|---------------------|-------------| -| `web_search` | yao-web | Search the web for real-time information | -| `web_fetch` | yao-web | Fetch and read a web page by URL | -| `process_call` | yao-process | Execute a Yao Process (server-side function) | -| `process_allowed` | yao-process | Check which processes are allowed | -| `doc_list` | yao-doc | Search/list available process documentation | -| `doc_inspect` | yao-doc | Get detailed docs for a specific process | -| `doc_validate` | yao-doc | Validate a process name and get suggestions | -| `image_read` | yao-vision | Read and analyze images using a vision model | +| Tool | Skill (auto-loaded) | Description | +| ----------------- | ------------------- | --------------------------------------------------- | +| `web_search` | yao-web | Search the web for real-time information | +| `web_fetch` | yao-web | Fetch and read a web page by URL | +| `process_call` | yao-process | Execute a Yao Process (server-side function) | +| `process_allowed` | yao-process | Check which processes are allowed | +| `doc_list` | yao-doc | Search/list available process documentation | +| `doc_inspect` | yao-doc | Get detailed docs for a specific process | +| `doc_validate` | yao-doc | Validate a process name and get suggestions | +| `image_read` | yao-image | Read and analyze images using a vision model | +| `image_generate` | yao-image | Generate images from text prompts | +| `image_providers` | yao-image | List available image generation or vision providers | -The system skills (`yao-web`, `yao-process`, `yao-doc`, `yao-vision`) in `$HOME/.claude/skills/` are **auto-discovered** — they contain detailed parameter docs and workflow guidance. You do not need to manually read them; they are loaded automatically when your task matches their description. +The system skills (`yao-web`, `yao-process`, `yao-doc`, `yao-image`) in `$HOME/.claude/skills/` are **auto-discovered** — they contain detailed parameter docs and workflow guidance. You do not need to manually read them; they are loaded automatically when your task matches their description. diff --git a/tools/skills/yao-image/SKILL.md b/tools/skills/yao-image/SKILL.md index a10a3ca8..53fa85d9 100644 --- a/tools/skills/yao-image/SKILL.md +++ b/tools/skills/yao-image/SKILL.md @@ -1,11 +1,11 @@ --- -name: yao-vision -description: Image understanding expert. ALWAYS invoke this skill when you need to read, analyze, or describe an image that you cannot process directly. Use for screenshots, photos, charts, diagrams, or any visual content. +name: yao-image +description: Image expert. ALWAYS invoke this skill when you need to read, analyze, describe, or generate images. Use for screenshots, photos, charts, diagrams, AI-generated images, or any visual content. --- -# Vision Tools +# Image Tools -Use when you encounter images you cannot read natively (e.g., as a text-only model). +Use these tools when you encounter images you cannot read natively, or when you need to generate new images. ## image_read @@ -31,9 +31,9 @@ tai tool image_read '{"image_path": "workspace://ws-id/path/to/image.png", "prom tai tool image_read '{"image_path": "attach://__yao.attachment/file-id-123", "prompt": "Describe"}' ``` -### Yao data file: +### With a specific vision provider: ```bash -tai tool image_read '{"image_path": "yao://uploads/photo.jpg", "prompt": "Describe"}' +tai tool image_read '{"image_path": "/path/to/image.png", "prompt": "Describe", "provider": "llm.my-openai:gpt-4o"}' ``` | Parameter | Type | Required | Description | @@ -41,6 +41,52 @@ tai tool image_read '{"image_path": "yao://uploads/photo.jpg", "prompt": "Descri | image_path | string | yes | File path, URL, workspace://, attach://, or yao:// URI | | prompt | string | no | Analysis instruction (default: describe in detail) | | max_size | integer | no | Max dimension in pixels for longest edge (default: 1080) | +| provider | string | no | Vision provider connector ID. If omitted, uses default vision model | Images are automatically resized (preserving aspect ratio) before sending to the vision model. Supported formats: PNG, JPEG, GIF, WebP. + +## image_generate + +Generate an image from a text prompt and save it to a file. + +### Basic usage (always specify output): +```bash +tai tool image_generate '{"prompt": "A serene mountain landscape at sunset", "output": "landscape.png"}' +``` + +### With specific provider and size: +```bash +tai tool image_generate '{"prompt": "A futuristic city skyline", "provider": "llm.my-openai:dall-e-3", "size": "1792x1024", "output": "output/city.png"}' +``` + +| Parameter | Type | Required | Description | +| --------- | ------ | -------- | ----------------------------------------------------------------- | +| prompt | string | yes | Text description of the image to generate | +| output | string | yes | File path to save the generated image (parent dirs created automatically) | +| provider | string | no | Provider connector ID (use `image_providers` to list). Auto-selects if omitted | +| size | string | no | Image dimensions (default: 1024x1024). Common: 1024x1024, 1024x1792, 1792x1024 | + +**Important**: Always pass `output`. The tool saves the image directly and returns only the file path and size. Without `output`, the raw base64 data is returned which may exceed output limits. + +Use relative paths (e.g. `"output": "fox.png"`) — they resolve relative to the current working directory (`$WORKDIR`). No need to prepend `$WORKDIR` manually. + +## image_providers + +List available image providers filtered by capability. + +### List image generation providers (default): +```bash +tai tool image_providers '{}' +``` + +### List vision (image reading) providers: +```bash +tai tool image_providers '{"capability": "vision"}' +``` + +| Parameter | Type | Required | Description | +| ---------- | ------ | -------- | ----------------------------------------------------------- | +| capability | string | no | `image_generation` (default) or `vision` | + +Returns a list of providers with their available models and connector IDs that can be passed to `image_generate` or `image_read`. diff --git a/tools/skills_test.go b/tools/skills_test.go index 00027909..73ad597d 100644 --- a/tools/skills_test.go +++ b/tools/skills_test.go @@ -6,11 +6,12 @@ import ( "testing" ) -func TestSkillsFS_ContainsThreeSkills(t *testing.T) { +func TestSkillsFS_ContainsAllSkills(t *testing.T) { expected := map[string]bool{ "skills/yao-web/SKILL.md": false, "skills/yao-process/SKILL.md": false, "skills/yao-doc/SKILL.md": false, + "skills/yao-image/SKILL.md": false, } err := fs.WalkDir(SkillsFS, "skills", func(path string, d fs.DirEntry, err error) error { @@ -41,6 +42,7 @@ func TestSkillsFS_FrontmatterFields(t *testing.T) { {"skills/yao-web/SKILL.md", "yao-web"}, {"skills/yao-process/SKILL.md", "yao-process"}, {"skills/yao-doc/SKILL.md", "yao-doc"}, + {"skills/yao-image/SKILL.md", "yao-image"}, } for _, s := range skills { diff --git a/tools/tools.go b/tools/tools.go index de1507a6..e4a96fa3 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -9,8 +9,8 @@ import ( "github.com/yaoapp/gou/process" "github.com/yaoapp/kun/log" "github.com/yaoapp/yao/tools/docs" + "github.com/yaoapp/yao/tools/image" "github.com/yaoapp/yao/tools/proc" - "github.com/yaoapp/yao/tools/vision" "github.com/yaoapp/yao/tools/webfetch" "github.com/yaoapp/yao/tools/websearch" ) @@ -24,8 +24,8 @@ var mcpProcessDSL []byte //go:embed mcps/doc.json var mcpDocDSL []byte -//go:embed mcps/vision.json -var mcpVisionDSL []byte +//go:embed mcps/image.json +var mcpImageDSL []byte func init() { process.RegisterGroup("tools", map[string]process.Handler{ @@ -36,7 +36,9 @@ func init() { "doc_list": docs.ListHandler, "doc_inspect": docs.InspectHandler, "doc_validate": docs.ValidateHandler, - "image_read": vision.Handler, + "image_read": image.ReadHandler, + "image_generate": image.GenerateHandler, + "image_providers": image.ProvidersHandler, }) registerMCPServer(mcpWebDSL, "yao-web", @@ -45,8 +47,8 @@ func init() { proc.SchemaJSON, proc.AllowedSchemaJSON) registerMCPServer(mcpDocDSL, "yao-doc", docs.ListSchemaJSON, docs.InspectSchemaJSON, docs.ValidateSchemaJSON) - registerMCPServer(mcpVisionDSL, "yao-vision", - vision.SchemaJSON) + registerMCPServer(mcpImageDSL, "yao-image", + image.ReadSchemaJSON, image.GenerateSchemaJSON, image.ProvidersSchemaJSON) } func registerMCPServer(dsl []byte, id string, schemas ...[]byte) {