From 983dbd3ceea577866661b57cad496e2fbc234caa Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 29 Dec 2025 14:26:48 +0800 Subject: [PATCH] Refactor Content Processing and Enhance Search Functionality - Updated the BuildContent method to utilize content parsing with improved error handling and context injection, enhancing the processing of user input. - Refactored the loadMap function to support multiple search configuration types, improving flexibility in handling search settings. - Enhanced the shouldAutoSearch method to include a check for search disabling via context metadata, allowing for more granular control over search behavior. - Updated the getMergedSearchUses method to prioritize options.Uses, ensuring that search configurations can be dynamically adjusted based on provided options. - Removed deprecated audio and excel handling code, streamlining the content processing package and improving maintainability. --- agent/assistant/build_content.go | 30 +- agent/assistant/load.go | 14 +- agent/assistant/search.go | 43 +- agent/assistant/utils.go | 19 + agent/content/README.md | 326 ------------- agent/content/audio.go | 61 --- agent/content/content.go | 601 +++++------------------- agent/content/content_vision_test.go | 675 --------------------------- agent/content/excel.go | 48 -- agent/content/fetch.go | 95 ---- agent/content/image.go | 188 -------- agent/content/image/image.go | 406 ++++++++++++++++ agent/content/image/image_test.go | 374 +++++++++++++++ agent/content/image_test.go | 261 ----------- agent/content/interfaces.go | 26 -- agent/content/link/link.go | 1 + agent/content/pdf.go | 50 -- agent/content/pdf/pdf.go | 49 ++ agent/content/registry.go | 47 -- agent/content/text.go | 123 ----- agent/content/text_test.go | 152 ------ agent/content/tools.go | 240 ---------- agent/content/tools/tools.go | 125 +++++ agent/content/types.go | 262 ----------- agent/content/types/types.go | 23 + agent/content/word.go | 39 -- agent/context/types.go | 3 + agent/i18n/builtin.go | 9 + 28 files changed, 1187 insertions(+), 3103 deletions(-) delete mode 100644 agent/content/README.md delete mode 100644 agent/content/audio.go delete mode 100644 agent/content/content_vision_test.go delete mode 100644 agent/content/excel.go delete mode 100644 agent/content/fetch.go delete mode 100644 agent/content/image.go create mode 100644 agent/content/image/image.go create mode 100644 agent/content/image/image_test.go delete mode 100644 agent/content/image_test.go delete mode 100644 agent/content/interfaces.go create mode 100644 agent/content/link/link.go delete mode 100644 agent/content/pdf.go create mode 100644 agent/content/pdf/pdf.go delete mode 100644 agent/content/registry.go delete mode 100644 agent/content/text.go delete mode 100644 agent/content/text_test.go delete mode 100644 agent/content/tools.go create mode 100644 agent/content/tools/tools.go delete mode 100644 agent/content/types.go create mode 100644 agent/content/types/types.go delete mode 100644 agent/content/word.go diff --git a/agent/assistant/build_content.go b/agent/assistant/build_content.go index ce2f939c..00e71a5a 100644 --- a/agent/assistant/build_content.go +++ b/agent/assistant/build_content.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/yaoapp/yao/agent/content" + contentTypes "github.com/yaoapp/yao/agent/content/types" "github.com/yaoapp/yao/agent/context" ) @@ -23,19 +24,24 @@ func (ast *Assistant) BuildContent(ctx *context.Context, messages []context.Mess if err != nil { return nil, fmt.Errorf("failed to get connector: %w", err) } - _ = connector // unused but needed for GetConnector call - // Get Uses configuration from options (already merged in BuildRequest) - uses := options.Uses - - // Get ForceUses configuration from options - forceUses := options.ForceUses - - // Process content through Vision function - processedMessages, err := content.Vision(ctx, capabilities, messages, uses, forceUses) - if err != nil { - return nil, fmt.Errorf("failed to process content: %w", err) + // Build parse options + parseOptions := &contentTypes.Options{ + Capabilities: capabilities, + CompletionOptions: options, + Connector: connector, + StreamOptions: options.StreamOptions, } - return processedMessages, nil + contentMessages, referenceContext, err := content.ParseUserInput(ctx, messages, parseOptions) + if err != nil { + return nil, fmt.Errorf("failed to parse content: %w", err) + } + + // Inject reference context into messages + if referenceContext != nil { + contentMessages = ast.injectSearchContext(contentMessages, referenceContext) + } + + return contentMessages, nil } diff --git a/agent/assistant/load.go b/agent/assistant/load.go index 2814fcef..9d763837 100644 --- a/agent/assistant/load.go +++ b/agent/assistant/load.go @@ -609,7 +609,15 @@ func loadMap(data map[string]interface{}) (*Assistant, error) { // Search configuration (from package.yao search block) // This contains search options like web.max_results, kb.threshold, citation.format, etc. // Merge hierarchy: global config < assistant config - if v, ok := data["search"].(map[string]interface{}); ok { + switch v := data["search"].(type) { + + case *searchTypes.Config: + assistant.Search = v + + case searchTypes.Config: + assistant.Search = &v + + case map[string]interface{}: var assistantSearch searchTypes.Config raw, err := jsoniter.Marshal(v) if err != nil { @@ -621,8 +629,8 @@ func loadMap(data map[string]interface{}) (*Assistant, error) { } // Merge with global search config assistant.Search = mergeSearchConfig(globalSearchConfig, &assistantSearch) - } else if globalSearchConfig != nil { - // No assistant-specific config, use global + + default: assistant.Search = globalSearchConfig } diff --git a/agent/assistant/search.go b/agent/assistant/search.go index af6e2982..018d7b5b 100644 --- a/agent/assistant/search.go +++ b/agent/assistant/search.go @@ -31,6 +31,15 @@ func (ast *Assistant) shouldAutoSearch(ctx *context.Context, messages []context. return nil } + // Check if search is skipped via ctx.Metadata["__disable_search"] + if ctx != nil && ctx.Metadata != nil { + disableSearch := getBool(ctx.Metadata, "__disable_search") + if disableSearch { + ctx.Logger.Debug("Auto search skipped by ctx.Metadata['__disable_search']") + return nil + } + } + // Check createResponse.Search field (highest priority from Create hook) // Supports: bool | SearchIntent | nil if createResponse != nil && createResponse.Search != nil { @@ -46,7 +55,7 @@ func (ast *Assistant) shouldAutoSearch(ctx *context.Context, messages []context. } // Get merged uses configuration - uses := ast.getMergedSearchUses(createResponse) + uses := ast.getMergedSearchUses(createResponse, opts) // Check if search is explicitly disabled if uses != nil && uses.Search == "disabled" { @@ -357,8 +366,9 @@ func (ast *Assistant) sendIntentDone(ctx *context.Context, loadingID string, nee } // getMergedSearchUses returns the merged uses configuration for search -// Priority: createResponse > assistant -func (ast *Assistant) getMergedSearchUses(createResponse *context.HookCreateResponse) *context.Uses { +// Priority: createResponse > options.Uses > assistant +func (ast *Assistant) getMergedSearchUses(createResponse *context.HookCreateResponse, opts ...*context.Options) *context.Uses { + // Start with assistant uses var uses *context.Uses if ast.Uses != nil { @@ -371,6 +381,31 @@ func (ast *Assistant) getMergedSearchUses(createResponse *context.HookCreateResp } } + // Override with options.Uses if provided (highest priority) + if len(opts) > 0 && opts[0] != nil && opts[0].Uses != nil { + + if uses == nil { + uses = &context.Uses{} + } + + if opts[0].Uses.Search != "" { + uses.Search = opts[0].Uses.Search + } + if opts[0].Uses.Web != "" { + uses.Web = opts[0].Uses.Web + } + + if opts[0].Uses.Keyword != "" { + uses.Keyword = opts[0].Uses.Keyword + } + if opts[0].Uses.QueryDSL != "" { + uses.QueryDSL = opts[0].Uses.QueryDSL + } + if opts[0].Uses.Rerank != "" { + uses.Rerank = opts[0].Uses.Rerank + } + } + // Override with createResponse.Uses if provided (highest priority) if createResponse != nil && createResponse.Uses != nil { if uses == nil { @@ -405,7 +440,7 @@ func (ast *Assistant) executeAutoSearch(ctx *context.Context, messages []context defer ctx.Logger.PhaseComplete("Search") // Get merged uses configuration - uses := ast.getMergedSearchUses(createResponse) + uses := ast.getMergedSearchUses(createResponse, opts...) // Convert to search.Uses searchUses := &search.Uses{} diff --git a/agent/assistant/utils.go b/agent/assistant/utils.go index e005ad80..49eb673f 100644 --- a/agent/assistant/utils.go +++ b/agent/assistant/utils.go @@ -43,6 +43,25 @@ func getTimestamp(v interface{}) (int64, error) { return 0, fmt.Errorf("invalid timestamp type %T", v) } +// getBool gets bool from data map[string]interface{}, key string +func getBool(data map[string]interface{}, key string) bool { + switch v := data[key].(type) { + case bool: + return v + case int64: + return v != 0 + case int: + return v != 0 + case float64: + return v != 0 + case string: + return v == "true" || v == "1" || v == "enabled" || v == "yes" || v == "on" + case nil: + return false + } + return false +} + // stringHash returns the sha256 hash of the string func stringHash(v string) string { h := sha256.New() diff --git a/agent/content/README.md b/agent/content/README.md deleted file mode 100644 index 9f33b5bd..00000000 --- a/agent/content/README.md +++ /dev/null @@ -1,326 +0,0 @@ -# Content Processing Package - -This package handles content transformation for multimodal messages in agent conversations. It is called **BEFORE** sending messages to the LLM and converts extended content types into standard LLM-compatible formats. - -## ⚠️ Critical Design Principle - -**Input**: Messages with extended content types (`file`, `data`, etc.) -**Output**: Messages with ONLY standard LLM-compatible types (`text`, `image_url`, `input_audio`) - -The LLM should NEVER receive `type="file"` or `type="data"` content parts. These MUST be converted to `text` (or `image_url` for images if model supports vision). - -## Architecture - -``` -Vision (main entry) - ↓ -Initialize processedFiles cache (map[fileID]text) - ↓ -processMessage (for each message) - ↓ -processContentPart (for each content part) - ↓ -Is uploader wrapper? - ├── Yes → Check cache - │ ├── In cache? → Use cached text ✅ - │ └── Not in cache → Try GetText(fileID) preview - │ ├── Has preview? → Use preview + cache ✅ - │ └── No preview → Proceed to full processing ↓ - └── No (HTTP/other) → Proceed to full processing ↓ - ↓ -├── Fetch content (if needed) -│ ├── HTTP URL -│ └── Uploader Wrapper (__uploader://fileid) - ↓ -├── Determine Processing Strategy -│ ├── Model supports? → Format for model -│ └── Model doesn't support? → Use agent/MCP - ↓ -ProcessorRegistry - ↓ -├── ImageProcessor -├── AudioProcessor -├── PDFProcessor -├── WordProcessor -├── ExcelProcessor -└── TextProcessor - ↓ -Cache result (if uploader wrapper) -``` - -## Content Type Transformation - -### Input → Output Mapping - -| Input Type | Model Supports? | Output Type | Processing | -|------------|-----------------|-------------|------------| -| `text` | - | `text` | Pass through | -| `image_url` | ✅ Yes | `image_url` | Convert format if needed (base64/URL) | -| `image_url` | ❌ No | `text` | Use vision agent/MCP to describe | -| `input_audio` | ✅ Yes | `input_audio` | Keep as audio | -| `input_audio` | ❌ No | `text` | Transcribe using audio agent/MCP | -| `file` (image) | ✅ Yes | `image_url` | Same as image_url processing | -| `file` (image) | ❌ No | `text` | Use vision tool to describe | -| `file` (document) | - | `text` | Extract text from PDF/Word/Excel/etc | -| `data` | - | `text` | Fetch and format data sources | - -### 1. Images and Audio - -**If model supports (vision/audio capability):** - -- Keep as multimodal content: - - `image_url`: Convert to appropriate format (OpenAI URL vs Claude base64) - - `input_audio`: Convert to base64 format - -**If model doesn't support:** - -- Convert to text: - - Use agent/MCP specified in `uses.Vision` or `uses.Audio` - - Extract text description or transcription - - Return as `type="text"` content - -**HTTP URLs:** - -- Fetch content first -- Then process the same way as above - -### 2. Files (type="file") - -**Critical**: All `type="file"` content MUST be converted to `text` or `image_url` (if image and model supports). - -**Processing Steps:** - -1. **Fetch file content**: - - Uploader wrapper: `__uploader://fileid` → Parse and fetch from attachment manager - - HTTP URL: Download from URL - -2. **Detect file type** from content-type and magic bytes - -3. **Process based on file type**: - -| File Type | Output Type | Processing Method | -| ------------ | ----------- | -------------------------------------------------------------------------------------------------------- | -| **Image** | `image_url` or `text` | If model supports vision → `image_url`
If not → use vision tool → `text` | -| **PDF** | `text` | If `uses.Vision` supports PDF → use vision tool
Otherwise → extract text directly | -| **Word** | `text` | Extract text using Word document parser | -| **Excel** | `text` | Extract and format as readable table/CSV | -| **PPT** | `text` | Extract text and slide content | -| **CSV** | `text` | Format as readable table | -| **Text** | `text` | Read directly (with encoding detection) | -| **JSON/XML** | `text` | Pretty print for readability | - -### 3. Data Sources (type="data") - -**Critical**: All `type="data"` content MUST be converted to `text`. - -**Processing Steps:** - -1. **Parse DataContent.Sources** array -2. **Fetch data** from each source: - - `model`: Query data model - - `kb_collection`: Search knowledge base collection - - `kb_document`: Get document content - - `table`: Query database table - - `api`: Call API endpoint - - `mcp_resource`: Fetch MCP resource -3. **Format as readable text**: - - Tables: Format as markdown tables or CSV - - Documents: Include title and content - - JSON: Pretty print -4. **Return as** `type="text"` content - -## Components - -### Core Files - -- **content.go** - Main entry point (`Vision` function) -- **types.go** - Type definitions and constants -- **interfaces.go** - Interface definitions - -### Fetching - -- **fetch.go** - Fetch content from HTTP or uploader - -### Processors - -- **processor.go** - Processor registry and routing -- **image.go** - Image processing -- **audio.go** - Audio processing -- **pdf.go** - PDF document processing -- **word.go** - Word document processing -- **excel.go** - Excel spreadsheet processing -- **text.go** - Plain text and CSV processing - -## Frontend Message Format - -The frontend (InputArea) sends messages in the following format: - -### Image Attachments -```json -{ - "type": "image_url", - "image_url": { - "url": "__yao.attachment://file_id", - "detail": "auto" - } -} -``` - -### File Attachments -```json -{ - "type": "file", - "file": { - "url": "__yao.attachment://file_id", - "filename": "document.pdf" - } -} -``` - -The `url` field contains an uploader wrapper in the format `__uploader://fileid`. - -## Data Structures - -### ContentInfo - -Holds information about content to be processed: - -```go -type ContentInfo struct { - Source ContentSource // http, uploader, base64, local - FileType FileType // image, audio, pdf, word, excel, etc. - ContentType string // MIME type - URL string // Original URL or file ID - Data []byte // File data - - // For uploader wrapper - UploaderName string - FileID string -} -``` - -### ProcessedContent - -Result of content processing: - -```go -type ProcessedContent struct { - Text string // Extracted text - ContentPart *context.ContentPart // For model input - Metadata map[string]interface{} - Error error -} -``` - -## Usage Example - -```go -import ( - "github.com/yaoapp/yao/agent/content" - "github.com/yaoapp/yao/agent/context" -) - -// Process messages before sending to LLM -processedMessages, err := content.Vision( - ctx, - capabilities, // Model capabilities - messages, // Original messages - uses, // Tool specifications (vision, audio, etc.) -) -``` - -## Performance Optimization - -### File Processing Cache - -**Problem**: Same file (uploader wrapper) might appear in multiple messages or be referenced multiple times. - -**Solution**: Three-level caching strategy: - -1. **In-memory cache** (`processedFiles` map): - - Caches processed text for the duration of the Vision() call - - Key: file ID from uploader wrapper - - Value: extracted text content - -2. **Attachment preview** (attachment.GetText with preview): - - Tries to get preview (first 2000 chars) from attachment manager - - If file was previously processed and saved, preview is available immediately - - Much faster than full file processing - -3. **Full processing** (only if needed): - - Falls back to complete file processing if no cache/preview available - - Result is cached in memory and optionally saved to attachment manager - -### Cache Flow - -```go -// For uploader://file_id -1. Check processedFiles[file_id] - └── Found? → Return cached text ⚡ (fastest) - -2. Not in cache → Call attachment.GetText(file_id, false) // preview only - └── Has preview? → Cache and return ⚡ (fast) - -3. No preview → Process file fully 🔄 (slower) - └── Cache result in processedFiles - └── Optional: Save to attachment using SaveText for future use -``` - -### Benefits - -- **Avoid duplicate processing**: Same file processed only once per Vision() call -- **Fast preview access**: Leverage pre-processed content from attachment manager -- **Reduced latency**: Especially important for large documents (PDFs, Word, Excel) -- **Resource efficient**: Less CPU/memory usage for repeated file references - -## Implementation Status - -### ✅ Completed - -- [x] Package structure -- [x] Type definitions -- [x] Interface definitions -- [x] Skeleton functions with TODO comments -- [x] File processing cache infrastructure -- [x] Cache helper functions (tryGetCachedText, cacheProcessedText) - -### 🚧 To Implement - -- [ ] tryGetCachedText implementation (attachment.GetText integration) -- [ ] cacheProcessedText implementation (attachment.SaveText integration) -- [ ] HTTP fetching logic -- [ ] Uploader wrapper parsing and fetching -- [ ] Image processing (base64, vision API) -- [ ] Audio processing (transcription) -- [ ] PDF text extraction -- [ ] Word document parsing -- [ ] Excel spreadsheet parsing -- [ ] Text/CSV formatting -- [ ] Content part processing logic -- [ ] Model capability detection -- [ ] Agent/MCP tool invocation - -## Configuration - -Content processing behavior is controlled by: - -1. **Model Capabilities** (`openai.Capabilities`) - - - Determines if model can handle images/audio directly - - Specifies vision format (OpenAI vs Claude) - -2. **Uses** (`context.Uses`) - ```go - type Uses struct { - Vision string // "agent" or "mcp:server_id" - Audio string // "agent" or "mcp:server_id" - Search string - Fetch string - } - ``` - -## Error Handling - -- Errors during processing are logged but don't stop the entire pipeline -- Original content is kept if processing fails -- Graceful degradation: if advanced processing fails, fall back to simpler methods diff --git a/agent/content/audio.go b/agent/content/audio.go deleted file mode 100644 index 1b4c755c..00000000 --- a/agent/content/audio.go +++ /dev/null @@ -1,61 +0,0 @@ -package content - -import ( - "fmt" - "strings" - - "github.com/yaoapp/gou/connector/openai" - agentContext "github.com/yaoapp/yao/agent/context" -) - -// AudioHandler handles audio content -type AudioHandler struct{} - -// CanHandle checks if this handler can handle the content type -func (h *AudioHandler) CanHandle(contentType string, fileType FileType) bool { - return fileType == FileTypeAudio || strings.HasPrefix(contentType, "audio/") -} - -// Handle processes audio content -// Logic similar to image: -// 1. If model supports audio input -> convert to base64 format -// 2. If model doesn't support audio -> use agent/MCP specified in uses.Audio -func (h *AudioHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) { - // TODO: Implement audio handling - // 1. Check model audio capabilities - // 2. If supported: - // - Encode audio as base64 with proper format - // 3. If not supported: - // - Call audio agent/MCP to transcribe audio to text - // 4. Return Result with text or ContentPart - return nil, fmt.Errorf("not implemented") -} - -// handleWithAudioModel processes audio using model's audio capability -func (h *AudioHandler) handleWithAudioModel(ctx *agentContext.Context, info *Info) (*Result, error) { - // TODO: Implement audio model processing - // Format audio according to model's audio input format - return nil, fmt.Errorf("not implemented") -} - -// handleWithAudioAgent processes audio using audio agent or MCP -func (h *AudioHandler) handleWithAudioAgent(ctx *agentContext.Context, info *Info, audioTool string) (string, error) { - // TODO: Implement audio agent/MCP processing - // 1. Parse audioTool (format: "agent" or "mcp:server_id") - // 2. Call appropriate tool to transcribe audio - // 3. Return transcribed text - return "", fmt.Errorf("not implemented") -} - -// encodeAudioBase64 encodes audio data to base64 with proper format -func encodeAudioBase64(data []byte, contentType string) string { - // TODO: Implement audio base64 encoding - return "" -} - -// detectAudioFormat detects audio format from content type or data -func detectAudioFormat(contentType string, data []byte) string { - // TODO: Implement audio format detection - // Return format like "wav", "mp3", "flac", etc. - return "" -} diff --git a/agent/content/content.go b/agent/content/content.go index 4686b134..f7565142 100644 --- a/agent/content/content.go +++ b/agent/content/content.go @@ -2,528 +2,118 @@ package content import ( "fmt" - "strings" - "github.com/yaoapp/gou/connector/openai" + "github.com/yaoapp/kun/log" + "github.com/yaoapp/yao/agent/content/image" + "github.com/yaoapp/yao/agent/content/types" agentContext "github.com/yaoapp/yao/agent/context" - "github.com/yaoapp/yao/attachment" + searchTypes "github.com/yaoapp/yao/agent/search/types" ) -// Vision transforms extended content types to LLM-compatible formats -// This is the main entry point for content preprocessing before sending to LLM -// -// IMPORTANT: This function is called BEFORE sending messages to LLM (in agent.executeLLMStream) -// It must convert all extended content types to standard LLM-compatible types. -// -// Input Content Types (Extended): -// - type="text" -> Pass through (already standard) -// - type="image_url" -> Process based on model capability (may need base64 conversion or vision tool) -// - type="input_audio" -> Process based on model capability (may need transcription) -// - type="file" -> Convert to text or image_url (MUST be converted) -// - type="data" -> Convert to text (MUST be converted) -// -// Output Content Types (LLM-compatible only): -// - type="text" -> Text content -// - type="image_url" -> Image (only if model supports vision) -// - type="input_audio" -> Audio (only if model supports audio) -// -// Processing Logic: -// 1. For images (image_url): -// - If model supports vision -> keep as image_url (may convert URL to base64) -// - If model doesn't support -> use vision agent/MCP to extract text -> convert to type="text" -// -// 2. For audio (input_audio): -// - If model supports audio -> keep as input_audio -// - If model doesn't support -> use audio agent/MCP to transcribe -> convert to type="text" -// -// 3. For files (type="file"): -// - Parse uploader wrapper (__uploader://fileid) or fetch HTTP URL -// - Detect file type (PDF, Word, Excel, Image, etc.) -// - Process based on file type: -// - Images: same as image processing above -// - PDF: use vision tool if available, otherwise extract text -> type="text" -// - Word/Excel/PPT/CSV: extract text -> type="text" -// - MUST convert to type="text" or type="image_url" (if image and model supports) -// -// 4. For data (type="data"): -// - Fetch data from sources (models, KB, MCP resources, etc.) -// - Format as readable text -// - MUST convert to type="text" -// -// Return: Messages with only standard LLM-compatible content types (text, image_url, input_audio) -func Vision(ctx *agentContext.Context, capabilities *openai.Capabilities, messages []agentContext.Message, uses *agentContext.Uses, forceUses ...bool) ([]agentContext.Message, error) { - // Determine if we should force using Uses tools even when model has native capabilities - shouldForceUses := false - if len(forceUses) > 0 { - shouldForceUses = forceUses[0] - } - // Initialize handlers and fetcher - registry := NewRegistry() - fetcher := NewFetcher() - - // Cache for processed files (uploader wrapper -> extracted text) - // Ensures each file is only processed once - processedFiles := make(map[string]string) - - // Process each message - processedMessages := make([]agentContext.Message, 0, len(messages)) - - for _, msg := range messages { - processedMsg, err := processMessage(ctx, &msg, capabilities, uses, shouldForceUses, registry, fetcher, processedFiles) - if err != nil { - // Log error but continue processing other messages - // TODO: Add proper logging - fmt.Printf("Warning: failed to process message: %v\n", err) - processedMessages = append(processedMessages, msg) // Keep original on error +// ParseUserInput ParseUserInput +func ParseUserInput(ctx *agentContext.Context, messages []agentContext.Message, options *types.Options) ([]agentContext.Message, *searchTypes.ReferenceContext, error) { + var referenceContext *searchTypes.ReferenceContext = nil + var parsedMessages []agentContext.Message = make([]agentContext.Message, 0) + for _, message := range messages { + // Only process user messages (current or from history) + if message.Role != agentContext.RoleUser { + parsedMessages = append(parsedMessages, message) continue } - processedMessages = append(processedMessages, processedMsg) - } - return processedMessages, nil -} - -// processMessage processes a single message and its content parts -func processMessage( - ctx *agentContext.Context, - msg *agentContext.Message, - capabilities *openai.Capabilities, - uses *agentContext.Uses, - forceUses bool, - registry *Registry, - fetcher Fetcher, - processedFiles map[string]string, -) (agentContext.Message, error) { - // If content is simple string, no processing needed - if _, ok := msg.GetContentAsString(); ok { - return *msg, nil - } - - // Get content parts - try typed first, then convert from interface{} - parts, ok := msg.GetContentAsParts() - if !ok { - // Try to convert from []interface{} (common when loaded from history/JSON) - parts, ok = convertToContentParts(msg.Content) - if !ok { - return *msg, nil - } - } - - // Note: File information will be collected and stored in Space by CallAgentWithFileInfo - // when calling vision agents, using the agent ID as namespace prefix - - // Process each content part - processedParts := make([]agentContext.ContentPart, 0, len(parts)) - for _, part := range parts { - processedPart, err := processContentPart(ctx, &part, capabilities, uses, forceUses, registry, fetcher, processedFiles) + // Parse user input message (Ignore errors) + parsedMessage, refs, err := parseUserInputMessage(ctx, message, options) if err != nil { - // Log error and handle gracefully - fmt.Printf("Warning: failed to process content part: %v\n", err) + parsedMessages = append(parsedMessages, message) + log.Error("Failed to parse user input message: %v, %v", message.Content, err) + continue + } + parsedMessages = append(parsedMessages, parsedMessage) - // For image_url that failed to process, convert to text description - // This prevents sending unsupported multimodal content to non-vision models - if part.Type == agentContext.ContentImageURL { - processedParts = append(processedParts, agentContext.ContentPart{ - Type: agentContext.ContentText, - Text: fmt.Sprintf("[Image processing failed: %s]", part.ImageURL.URL), - }) - } else { - // For other types, keep original - processedParts = append(processedParts, part) + // Add reference to reference context + if refs != nil { + if referenceContext == nil { + referenceContext = &searchTypes.ReferenceContext{} } - continue - } - - // If handling returned text, convert to text part - if processedPart.Text != "" { - processedParts = append(processedParts, agentContext.ContentPart{ - Type: agentContext.ContentText, - Text: processedPart.Text, - }) - } else if processedPart.ContentPart != nil { - // Use the processed content part (e.g., base64 image) - processedParts = append(processedParts, *processedPart.ContentPart) - } else { - // Keep original if no handling result - processedParts = append(processedParts, part) + referenceContext.References = append(referenceContext.References, refs...) } } - // Return new message with processed content - return agentContext.Message{ - Role: msg.Role, - Content: processedParts, - Name: msg.Name, - ToolCallID: msg.ToolCallID, - ToolCalls: msg.ToolCalls, - Refusal: msg.Refusal, - }, nil + return parsedMessages, referenceContext, nil } -// processContentPart processes a single content part -// IMPORTANT: Must convert extended types (file, data) to standard types (text, image_url, input_audio) -func processContentPart( - ctx *agentContext.Context, - part *agentContext.ContentPart, - capabilities *openai.Capabilities, - uses *agentContext.Uses, - forceUses bool, - registry *Registry, - fetcher Fetcher, - processedFiles map[string]string, -) (*Result, error) { - // 1. Handle standard types - pass through - switch part.Type { +// parseUserInputMessage parse a user input message +func parseUserInputMessage(ctx *agentContext.Context, message agentContext.Message, options *types.Options) (agentContext.Message, []*searchTypes.Reference, error) { + + // Context content type + switch content := message.Content.(type) { + case string: + return message, nil, nil + + case []agentContext.ContentPart: + return parseContentParts(ctx, message, content, options) + + case []interface{}: + // Handle content loaded from history/JSON ([]interface{} instead of []ContentPart) + parts, ok := convertToContentParts(content) + if !ok { + return message, nil, nil + } + return parseContentParts(ctx, message, parts, options) + } + + return message, nil, fmt.Errorf("unsupported content type: %T", message.Content) +} + +// parseContentParts parses content parts and returns the parsed message +func parseContentParts(ctx *agentContext.Context, message agentContext.Message, content []agentContext.ContentPart, options *types.Options) (agentContext.Message, []*searchTypes.Reference, error) { + allRefs := []*searchTypes.Reference{} + parts := make([]agentContext.ContentPart, 0, len(content)) + for _, part := range content { + parsedPart, refs, err := parseContentPart(ctx, part, options) + if err != nil { + parts = append(parts, part) + continue + } + parts = append(parts, parsedPart) + if refs != nil { + allRefs = append(allRefs, refs...) + } + } + + parsedMessage := message + parsedMessage.Content = parts + return parsedMessage, allRefs, nil +} + +// parseContentPart parse a content part +func parseContentPart(ctx *agentContext.Context, content agentContext.ContentPart, options *types.Options) (agentContext.ContentPart, []*searchTypes.Reference, error) { + switch content.Type { case agentContext.ContentText: - // Text is already standard, pass through - return &Result{ - ContentPart: part, - }, nil + return content, nil, nil case agentContext.ContentImageURL: - // Image URL - check if it needs processing - return processImageURLContent(ctx, part, capabilities, uses, forceUses, registry, fetcher, processedFiles) + return image.New(options).Parse(ctx, content) case agentContext.ContentInputAudio: - // Audio - check if it needs processing - return processAudioContent(ctx, part, capabilities, uses, registry, fetcher, processedFiles) - } + return content, nil, nil - // 2. Handle extended types - MUST convert to standard types - switch part.Type { case agentContext.ContentFile: - return processFileContent(ctx, part, capabilities, uses, forceUses, registry, fetcher, processedFiles) + return content, nil, nil case agentContext.ContentData: - return processDataContent(ctx, part) + return content, nil, nil default: - // Unknown type, return error - return nil, fmt.Errorf("unsupported content type: %s", part.Type) + return content, nil, fmt.Errorf("unsupported content part type: %s", content.Type) } } -// processFileContent processes file content with caching -func processFileContent( - ctx *agentContext.Context, - part *agentContext.ContentPart, - capabilities *openai.Capabilities, - uses *agentContext.Uses, - forceUses bool, - registry *Registry, - fetcher Fetcher, - processedFiles map[string]string, -) (*Result, error) { - if part.File == nil || part.File.URL == "" { - return nil, fmt.Errorf("file content part missing URL") - } - - url := part.File.URL - - // Step 1: Try to get cached text (three-tier cache) - cachedText, found, err := tryGetCachedText(ctx, url, processedFiles) - if err != nil { - return nil, fmt.Errorf("failed to check cache: %w", err) - } - if found { - // Cache hit! Return as text - return &Result{ - Text: cachedText, - }, nil - } - - // Step 2: No cache, need to process the file - // Determine content source - source, sourceURL, err := determineContentSource(part) - if err != nil { - return nil, fmt.Errorf("failed to determine content source: %w", err) - } - - // Fetch content - info, err := fetcher.Fetch(ctx, source, sourceURL) - if err != nil { - return nil, fmt.Errorf("failed to fetch content: %w", err) - } - - // Set filename from part if not already set - if info.Filename == "" && part.File.Filename != "" { - info.Filename = part.File.Filename - } - - // Detect file type if not already set - if info.FileType == FileTypeUnknown { - info.FileType = DetectFileType(info.ContentType, part.File.Filename) - } - - // Process with appropriate handler - result, err := registry.Handle(ctx, info, capabilities, uses, forceUses) - if err != nil { - return nil, fmt.Errorf("failed to handle content: %w", err) - } - - // Step 3: Cache the result if it's text - if result.Text != "" { - if cacheErr := cacheProcessedText(ctx, url, result.Text, processedFiles); cacheErr != nil { - // Log error but don't fail the request - fmt.Printf("Warning: failed to cache processed text: %v\n", cacheErr) - } - } - - return result, nil -} - -// processImageURLContent processes image_url content -// If URL is uploader wrapper or HTTP, fetch and process it -func processImageURLContent( - ctx *agentContext.Context, - part *agentContext.ContentPart, - capabilities *openai.Capabilities, - uses *agentContext.Uses, - forceUses bool, - registry *Registry, - fetcher Fetcher, - processedFiles map[string]string, -) (*Result, error) { - if part.ImageURL == nil || part.ImageURL.URL == "" { - return nil, fmt.Errorf("image_url content missing URL") - } - - url := part.ImageURL.URL - - // If it's a data URI (base64), pass through - if strings.HasPrefix(url, "data:") { - return &Result{ - ContentPart: part, - }, nil - } - - // Check model capabilities - supportsVision := false - if capabilities != nil && capabilities.Vision != nil { - // Vision can be bool or string (format) - switch v := capabilities.Vision.(type) { - case bool: - supportsVision = v - case string: - supportsVision = v != "" && v != "false" && v != "none" - } - } - - // If model supports vision AND we're not forcing uses, pass through - if supportsVision && !forceUses { - return &Result{ - ContentPart: part, - }, nil - } - - // Model doesn't support vision OR forceUses is true - // Need to convert image to text - - // If it's uploader wrapper or HTTP URL, need to process - // Check cache first - cachedText, found, err := tryGetCachedText(ctx, url, processedFiles) - if err != nil { - return nil, fmt.Errorf("failed to check cache: %w", err) - } - if found { - // Cache hit! Return as text - return &Result{ - Text: cachedText, - }, nil - } - - // Determine source - source, sourceURL, err := determineContentSource(part) - if err != nil { - return nil, fmt.Errorf("failed to determine content source: %w", err) - } - - // Fetch content - info, err := fetcher.Fetch(ctx, source, sourceURL) - if err != nil { - return nil, fmt.Errorf("failed to fetch image: %w", err) - } - - // Set file type as image - info.FileType = FileTypeImage - - // Process with image handler - result, err := registry.Handle(ctx, info, capabilities, uses, forceUses) - if err != nil { - return nil, fmt.Errorf("failed to handle image: %w", err) - } - - // Cache if result is text - if result.Text != "" { - if cacheErr := cacheProcessedText(ctx, url, result.Text, processedFiles); cacheErr != nil { - fmt.Printf("Warning: failed to cache processed text: %v\n", cacheErr) - } - } - - return result, nil -} - -// processAudioContent processes input_audio content -func processAudioContent( - ctx *agentContext.Context, - part *agentContext.ContentPart, - capabilities *openai.Capabilities, - uses *agentContext.Uses, - registry *Registry, - fetcher Fetcher, - processedFiles map[string]string, -) (*Result, error) { - if part.InputAudio == nil || part.InputAudio.Data == "" { - return nil, fmt.Errorf("input_audio content missing data") - } - - // For now, pass through audio as-is - // TODO: Implement audio processing (transcription, etc.) - return &Result{ - ContentPart: part, - }, nil -} - -// processDataContent processes data content (converts to text) -func processDataContent(ctx *agentContext.Context, part *agentContext.ContentPart) (*Result, error) { - if part.Data == nil { - return nil, fmt.Errorf("data content part missing data") - } - - // TODO: Implement data processing - // For now, just return error - return nil, fmt.Errorf("data content processing not implemented yet") -} - -// determineContentSource determines where the content comes from -func determineContentSource(part *agentContext.ContentPart) (Source, string, error) { - var url string - - // Extract URL based on content type - switch part.Type { - case agentContext.ContentFile: - if part.File == nil || part.File.URL == "" { - return "", "", fmt.Errorf("file content missing URL") - } - url = part.File.URL - - case agentContext.ContentImageURL: - if part.ImageURL == nil || part.ImageURL.URL == "" { - return "", "", fmt.Errorf("image_url content missing URL") - } - url = part.ImageURL.URL - - case agentContext.ContentInputAudio: - if part.InputAudio == nil || part.InputAudio.Data == "" { - return "", "", fmt.Errorf("input_audio content missing data") - } - // Audio data is base64, treat as base64 source - return SourceBase64, part.InputAudio.Data, nil - - default: - return "", "", fmt.Errorf("unsupported content type for source detection: %s", part.Type) - } - - // Determine source type based on URL format - if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") { - return SourceHTTP, url, nil - } - - if strings.HasPrefix(url, "__") { - // Uploader wrapper format: __uploader://fileid - return SourceUploader, url, nil - } - - if strings.HasPrefix(url, "data:") { - // Data URI (base64) - return SourceBase64, url, nil - } - - // Default to treating as uploader if no prefix matches - return SourceUploader, url, nil -} - -// shouldProcessWithModel checks if content should be processed by the model directly -func shouldProcessWithModel(capabilities *openai.Capabilities, fileType FileType) (bool, agentContext.VisionFormat) { - // TODO: Implement model capability check - // For images: check if model supports vision - // For audio: check if model supports audio input - // Return whether to use model and the format to use - return false, agentContext.VisionFormatNone -} - -// getToolForProcessing gets the agent/MCP tool to use for processing -func getToolForProcessing(uses *agentContext.Uses, fileType FileType) string { - // TODO: Implement tool selection - // Based on file type, return the appropriate tool from uses - // - Images -> uses.Vision - // - Audio -> uses.Audio - // - PDF (if vision available) -> uses.Vision - return "" -} - -// tryGetCachedText checks if the URL is an uploader wrapper and tries to get cached text -// Returns (text, found, error) -func tryGetCachedText(ctx *agentContext.Context, url string, processedFiles map[string]string) (string, bool, error) { - // Parse URL to check if it's an uploader wrapper - uploaderName, fileID, isWrapper := attachment.Parse(url) - - if !isWrapper { - return "", false, nil // Not an uploader wrapper, no cache - } - - // 1. Check in-memory cache for this Vision call - if text, ok := processedFiles[fileID]; ok { - return text, true, nil - } - - // 2. Try attachment manager's content_preview (cross-call cache) - manager, exists := attachment.Managers[uploaderName] - if exists { - // GetText with fullContent=false to get preview (default) - text, err := manager.GetText(ctx.Context, fileID, false) - if err == nil && text != "" { - // Cache in-memory for this Vision call - processedFiles[fileID] = text - return text, true, nil - } - } - - // No cache found - return "", false, nil -} - -// cacheProcessedText caches the processed text for an uploader wrapper -func cacheProcessedText(ctx *agentContext.Context, url string, text string, processedFiles map[string]string) error { - // Parse URL to get uploader name and file ID - uploaderName, fileID, isWrapper := attachment.Parse(url) - if !isWrapper { - return nil // Not an uploader wrapper, nothing to cache - } - - // 1. Cache in-memory for this Vision call - processedFiles[fileID] = text - - // 2. Save to attachment manager for future Vision calls - manager, exists := attachment.Managers[uploaderName] - if exists { - return manager.SaveText(ctx.Context, fileID, text) - } - - return nil -} - // convertToContentParts converts []interface{} to []ContentPart // This is needed when content is loaded from JSON/history and is []interface{} instead of []ContentPart -func convertToContentParts(content interface{}) ([]agentContext.ContentPart, bool) { - // Check if it's []interface{} - arr, ok := content.([]interface{}) - if !ok { - return nil, false - } - - parts := make([]agentContext.ContentPart, 0, len(arr)) - for _, item := range arr { +func convertToContentParts(content []interface{}) ([]agentContext.ContentPart, bool) { + parts := make([]agentContext.ContentPart, 0, len(content)) + for _, item := range content { // Each item should be a map m, ok := item.(map[string]interface{}) if !ok { @@ -578,6 +168,35 @@ func convertToContentParts(content interface{}) ([]agentContext.ContentPart, boo part.InputAudio.Format = format } } + + case "data": + if dataContent, ok := m["data"].(map[string]interface{}); ok { + part.Data = &agentContext.DataContent{} + if sources, ok := dataContent["sources"].([]interface{}); ok { + part.Data.Sources = make([]agentContext.DataSource, 0, len(sources)) + for _, src := range sources { + if srcMap, ok := src.(map[string]interface{}); ok { + source := agentContext.DataSource{} + if t, ok := srcMap["type"].(string); ok { + source.Type = agentContext.DataSourceType(t) + } + if name, ok := srcMap["name"].(string); ok { + source.Name = name + } + if id, ok := srcMap["id"].(string); ok { + source.ID = id + } + if filters, ok := srcMap["filters"].(map[string]interface{}); ok { + source.Filters = filters + } + if metadata, ok := srcMap["metadata"].(map[string]interface{}); ok { + source.Metadata = metadata + } + part.Data.Sources = append(part.Data.Sources, source) + } + } + } + } } parts = append(parts, part) diff --git a/agent/content/content_vision_test.go b/agent/content/content_vision_test.go deleted file mode 100644 index 02f4fb05..00000000 --- a/agent/content/content_vision_test.go +++ /dev/null @@ -1,675 +0,0 @@ -package content_test - -import ( - "bytes" - "context" - "image" - "image/color" - "image/png" - "mime/multipart" - "strings" - "testing" - - "github.com/yaoapp/gou/connector/openai" - "github.com/yaoapp/yao/agent/content" - agentContext "github.com/yaoapp/yao/agent/context" - "github.com/yaoapp/yao/agent/testutils" - "github.com/yaoapp/yao/attachment" -) - -// setupTestUploader creates and registers a test uploader manager -// The manager will be registered with "__" prefix as required by attachment.Parse -func setupTestUploader(t *testing.T, name string) attachment.FileManager { - // Register with __ prefix to match Parse behavior - managerName := "__" + name - manager, err := attachment.Register(managerName, "local", attachment.ManagerOption{ - Driver: "local", - MaxSize: "10M", - AllowedTypes: []string{"text/*", "image/*", "application/*"}, - Options: map[string]interface{}{ - "path": "/tmp/test_vision_attachments_" + name, - }, - }) - if err != nil { - t.Fatalf("Failed to register attachment manager '%s': %v", managerName, err) - } - return manager -} - -// cleanupTestUploader removes the test uploader from registry -func cleanupTestUploader(name string) { - delete(attachment.Managers, "__"+name) -} - -// generateTestImage creates a valid PNG image (100x100 red square) -func generateTestImage(t *testing.T) []byte { - img := image.NewRGBA(image.Rect(0, 0, 100, 100)) - red := color.RGBA{255, 0, 0, 255} - for y := 0; y < 100; y++ { - for x := 0; x < 100; x++ { - img.Set(x, y, red) - } - } - - var buf bytes.Buffer - if err := png.Encode(&buf, img); err != nil { - t.Fatalf("Failed to encode test image: %v", err) - } - return buf.Bytes() -} - -// TestVision_TextFile tests Vision function with text/code file parsing -func TestVision_TextFile(t *testing.T) { - testutils.Prepare(t) - defer testutils.Clean(t) - - // Setup test uploader - uploaderName := "test-vision-text" - manager := setupTestUploader(t, uploaderName) - defer cleanupTestUploader(uploaderName) - - // 1. Create and upload a Go source file - testContent := `package main - -import "fmt" - -func main() { - fmt.Println("Hello, Vision Test!") -} -` - - // Upload file - reader := strings.NewReader(testContent) - fileHeader := &attachment.FileHeader{ - FileHeader: &multipart.FileHeader{ - Filename: "main.go", - Size: int64(len(testContent)), - Header: make(map[string][]string), - }, - } - fileHeader.Header.Set("Content-Type", "text/x-go") - - uploadedFile, err := manager.Upload(context.Background(), fileHeader, reader, attachment.UploadOption{ - Groups: []string{"vision", "test"}, - }) - if err != nil { - t.Fatalf("Failed to upload file: %v", err) - } - - t.Logf("Uploaded file ID: %s", uploadedFile.ID) - - // 2. Prepare Vision context (text files don't need special capabilities) - ctx := agentContext.New(context.Background(), nil, "test") - - capabilities := &openai.Capabilities{} - - messages := []agentContext.Message{ - { - Role: "user", - Content: []agentContext.ContentPart{ - { - Type: agentContext.ContentFile, - File: &agentContext.FileAttachment{ - URL: "__" + uploaderName + "://" + uploadedFile.ID, - Filename: "main.go", - }, - }, - }, - }, - } - - // 3. Call Vision function - result, err := content.Vision(ctx, capabilities, messages, nil, false) - if err != nil { - t.Fatalf("Vision function failed: %v", err) - } - - if len(result) != 1 { - t.Fatalf("Expected 1 message, got %d", len(result)) - } - - // 4. Verify result - contentParts, ok := result[0].Content.([]agentContext.ContentPart) - if !ok { - t.Fatalf("Expected content to be []ContentPart, got %T", result[0].Content) - } - - if len(contentParts) != 1 { - t.Fatalf("Expected 1 content part, got %d", len(contentParts)) - } - - // Should be converted to text - if contentParts[0].Type != agentContext.ContentText { - t.Errorf("Expected ContentText type, got %s", contentParts[0].Type) - } - - if !strings.Contains(contentParts[0].Text, "package main") { - t.Errorf("Expected text to contain 'package main', got: %s", contentParts[0].Text) - } - - if !strings.Contains(contentParts[0].Text, "Hello, Vision Test!") { - t.Errorf("Expected text to contain 'Hello, Vision Test!', got: %s", contentParts[0].Text) - } - - t.Logf("✓ Text file successfully parsed: %d characters", len(contentParts[0].Text)) -} - -// TestVision_ImageWithVisionSupport tests image processing with vision-capable model -func TestVision_ImageWithVisionSupport(t *testing.T) { - testutils.Prepare(t) - defer testutils.Clean(t) - - // Setup test uploader - uploaderName := "test-vision-image" - manager := setupTestUploader(t, uploaderName) - defer cleanupTestUploader(uploaderName) - - // 1. Create and upload a test image (1x1 red PNG) - imageData := []byte{ - 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, - 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52, - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, - 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53, - 0xDE, 0x00, 0x00, 0x00, 0x0C, 0x49, 0x44, 0x41, - 0x54, 0x08, 0xD7, 0x63, 0xF8, 0xCF, 0xC0, 0x00, - 0x00, 0x03, 0x01, 0x01, 0x00, 0x18, 0xDD, 0x8D, - 0xB4, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, - 0x44, 0xAE, 0x42, 0x60, 0x82, - } - - // Upload image - reader := strings.NewReader(string(imageData)) - fileHeader := &attachment.FileHeader{ - FileHeader: &multipart.FileHeader{ - Filename: "test.png", - Size: int64(len(imageData)), - Header: make(map[string][]string), - }, - } - fileHeader.Header.Set("Content-Type", "image/png") - - uploadedFile, err := manager.Upload(context.Background(), fileHeader, reader, attachment.UploadOption{ - Groups: []string{"vision", "test"}, - }) - if err != nil { - t.Fatalf("Failed to upload image: %v", err) - } - - // 2. Prepare Vision context with vision-capable model - ctx := agentContext.New(context.Background(), nil, "test") - - // Construct capabilities with vision support (OpenAI format) - capabilities := &openai.Capabilities{ - Vision: agentContext.VisionFormatOpenAI, // OpenAI vision format - } - - messages := []agentContext.Message{ - { - Role: "user", - Content: []agentContext.ContentPart{ - { - Type: agentContext.ContentImageURL, - ImageURL: &agentContext.ImageURL{ - URL: "__" + uploaderName + "://" + uploadedFile.ID, - }, - }, - }, - }, - } - - // 3. Call Vision function (no uses needed for direct vision support) - result, err := content.Vision(ctx, capabilities, messages, nil, false) - if err != nil { - t.Fatalf("Vision function failed: %v", err) - } - - if len(result) != 1 { - t.Fatalf("Expected 1 message, got %d", len(result)) - } - - // 4. Verify result - contentParts, ok := result[0].Content.([]agentContext.ContentPart) - if !ok { - t.Fatalf("Expected content to be []ContentPart, got %T", result[0].Content) - } - - if len(contentParts) != 1 { - t.Fatalf("Expected 1 content part, got %d", len(contentParts)) - } - - // If model supports vision, should be image_url with base64 - if capabilities.Vision != nil { - if contentParts[0].Type != agentContext.ContentImageURL { - t.Errorf("Expected ContentImageURL type, got %s", contentParts[0].Type) - } - - if contentParts[0].ImageURL == nil { - t.Fatal("Expected ImageURL to be set") - } - - if !strings.Contains(contentParts[0].ImageURL.URL, "data:image/png;base64,") { - t.Errorf("Expected base64 data URI, got: %s", contentParts[0].ImageURL.URL) - } - - t.Logf("✓ Image processed with vision support: %d bytes (base64)", len(contentParts[0].ImageURL.URL)) - } else { - // If no vision support, should fall back to text (via agent/MCP) - t.Logf("ℹ Model doesn't support vision, result type: %s", contentParts[0].Type) - } -} - -// TestVision_ImageWithAgent tests image processing with vision agent when model doesn't support vision -// Note: This test demonstrates the agent fallback mechanism when the model doesn't support vision -func TestVision_ImageWithAgent(t *testing.T) { - testutils.Prepare(t) - defer testutils.Clean(t) - - // Setup test uploader - uploaderName := "test-vision-agent" - manager := setupTestUploader(t, uploaderName) - defer cleanupTestUploader(uploaderName) - - // 1. Generate and upload a valid test image (100x100 red PNG) - imageData := generateTestImage(t) - - reader := strings.NewReader(string(imageData)) - fileHeader := &attachment.FileHeader{ - FileHeader: &multipart.FileHeader{ - Filename: "test.png", - Size: int64(len(imageData)), - Header: make(map[string][]string), - }, - } - fileHeader.Header.Set("Content-Type", "image/png") - - uploadedFile, err := manager.Upload(context.Background(), fileHeader, reader, attachment.UploadOption{ - Groups: []string{"vision", "test"}, - }) - if err != nil { - t.Fatalf("Failed to upload image: %v", err) - } - - // 2. Prepare Vision context with proper setup - // Model does NOT support vision, but uses.Vision specifies a vision agent - ctx := agentContext.New(context.Background(), nil, "test") - - // Capabilities without vision support - capabilities := &openai.Capabilities{ - Vision: nil, // No vision support - } - - // Uses configuration with vision agent - uses := &agentContext.Uses{ - Vision: "tests.vision-helper", // Use vision-helper agent - } - - messages := []agentContext.Message{ - { - Role: "user", - Content: []agentContext.ContentPart{ - { - Type: agentContext.ContentImageURL, - ImageURL: &agentContext.ImageURL{ - URL: "__" + uploaderName + "://" + uploadedFile.ID, - }, - }, - }, - }, - } - - // 3. Call Vision - should use agent since model doesn't support vision - result, err := content.Vision(ctx, capabilities, messages, uses, false) - if err != nil { - t.Fatalf("Vision function failed: %v", err) - } - - if len(result) != 1 { - t.Fatalf("Expected 1 message, got %d", len(result)) - } - - // 4. Verify result is text (processed by vision agent) - contentParts, ok := result[0].Content.([]agentContext.ContentPart) - if !ok { - t.Fatalf("Expected content to be []ContentPart, got %T", result[0].Content) - } - - if len(contentParts) != 1 { - t.Fatalf("Expected 1 content part, got %d", len(contentParts)) - } - - if contentParts[0].Type != agentContext.ContentText { - t.Errorf("Expected ContentText (from agent), got: %s", contentParts[0].Type) - } - - if contentParts[0].Text == "" { - t.Error("Expected non-empty text from vision agent processing") - } - - t.Logf("✓ Vision agent processed image to text: %d characters", len(contentParts[0].Text)) - t.Logf("Agent response text:\n%s", contentParts[0].Text) -} - -// TestVision_CachedContent tests that file content is cached and reused -func TestVision_CachedContent(t *testing.T) { - testutils.Prepare(t) - defer testutils.Clean(t) - - // Setup test uploader - uploaderName := "test-vision-cache" - manager := setupTestUploader(t, uploaderName) - defer cleanupTestUploader(uploaderName) - - // 1. Upload a text file - testContent := "Test content for caching verification" - - reader := strings.NewReader(testContent) - fileHeader := &attachment.FileHeader{ - FileHeader: &multipart.FileHeader{ - Filename: "cache-test.txt", - Size: int64(len(testContent)), - Header: make(map[string][]string), - }, - } - fileHeader.Header.Set("Content-Type", "text/plain") - - uploadedFile, err := manager.Upload(context.Background(), fileHeader, reader, attachment.UploadOption{ - Groups: []string{"vision", "test"}, - }) - if err != nil { - t.Fatalf("Failed to upload file: %v", err) - } - - // 2. Prepare Vision context with same file referenced twice - ctx := agentContext.New(context.Background(), nil, "test") - - // Construct simple capabilities (text files don't need vision) - capabilities := &openai.Capabilities{} - - messages := []agentContext.Message{ - { - Role: "user", - Content: []agentContext.ContentPart{ - { - Type: agentContext.ContentFile, - File: &agentContext.FileAttachment{ - URL: "__" + uploaderName + "://" + uploadedFile.ID, - Filename: "cache-test.txt", - }, - }, - { - Type: agentContext.ContentFile, - File: &agentContext.FileAttachment{ - URL: "__" + uploaderName + "://" + uploadedFile.ID, // Same file - Filename: "cache-test.txt", - }, - }, - }, - }, - } - - // 3. Call Vision - result, err := content.Vision(ctx, capabilities, messages, nil, false) - if err != nil { - t.Fatalf("Vision function failed: %v", err) - } - - if len(result) != 1 { - t.Fatalf("Expected 1 message, got %d", len(result)) - } - - // 4. Verify both file references were processed - contentParts, ok := result[0].Content.([]agentContext.ContentPart) - if !ok { - t.Fatalf("Expected content to be []ContentPart") - } - - if len(contentParts) != 2 { - t.Fatalf("Expected 2 content parts (both files), got %d", len(contentParts)) - } - - // Both should be text with same content - if contentParts[0].Type != agentContext.ContentText { - t.Errorf("First part: expected ContentText, got %s", contentParts[0].Type) - } - - if contentParts[1].Type != agentContext.ContentText { - t.Errorf("Second part: expected ContentText, got %s", contentParts[1].Type) - } - - if !strings.Contains(contentParts[0].Text, testContent) { - t.Errorf("First part text doesn't contain expected content") - } - - if !strings.Contains(contentParts[1].Text, testContent) { - t.Errorf("Second part text doesn't contain expected content") - } - - // Verify content was cached (check attachment manager) - cachedText, err := manager.GetText(context.Background(), uploadedFile.ID) - if err != nil { - t.Fatalf("Failed to get cached text: %v", err) - } - - if cachedText == "" { - t.Error("Expected content to be cached in attachment manager") - } - - t.Logf("✓ Content successfully cached and reused: %d characters", len(cachedText)) -} - -// TestVision_FileMetadataInMemory tests that file metadata is correctly passed to vision agent via ctx.Memory.Context -func TestVision_FileMetadataInMemory(t *testing.T) { - testutils.Prepare(t) - defer testutils.Clean(t) - - // Setup test uploader - uploaderName := "test-vision-metadata" - manager := setupTestUploader(t, uploaderName) - defer cleanupTestUploader(uploaderName) - - // 1. Generate and upload a test image - imageData := generateTestImage(t) - - reader := strings.NewReader(string(imageData)) - fileHeader := &attachment.FileHeader{ - FileHeader: &multipart.FileHeader{ - Filename: "test-metadata.png", - Size: int64(len(imageData)), - Header: make(map[string][]string), - }, - } - fileHeader.Header.Set("Content-Type", "image/png") - - uploadedFile, err := manager.Upload(context.Background(), fileHeader, reader, attachment.UploadOption{ - Groups: []string{"vision", "test"}, - }) - if err != nil { - t.Fatalf("Failed to upload image: %v", err) - } - - t.Logf("Uploaded file: %s (ID: %s)", uploadedFile.Filename, uploadedFile.ID) - - // 2. Prepare Vision context - model doesn't support vision, use agent - ctx := agentContext.New(context.Background(), nil, "test") - - // Capabilities without vision support - capabilities := &openai.Capabilities{ - Vision: nil, // No vision support - force agent usage - } - - // Uses configuration with vision-helper agent - uses := &agentContext.Uses{ - Vision: "tests.vision-helper", // Use vision-helper agent that logs file metadata - } - - messages := []agentContext.Message{ - { - Role: "user", - Content: []agentContext.ContentPart{ - { - Type: agentContext.ContentImageURL, - ImageURL: &agentContext.ImageURL{ - URL: "__" + uploaderName + "://" + uploadedFile.ID, - }, - }, - }, - }, - } - - // 3. Call Vision - should pass file metadata to agent via Space - result, err := content.Vision(ctx, capabilities, messages, uses, false) - if err != nil { - t.Fatalf("Vision function failed: %v", err) - } - - if len(result) != 1 { - t.Fatalf("Expected 1 message, got %d", len(result)) - } - - // 4. Verify result contains file metadata validation info - contentParts, ok := result[0].Content.([]agentContext.ContentPart) - if !ok { - t.Fatalf("Expected content to be []ContentPart, got %T", result[0].Content) - } - - if len(contentParts) != 1 { - t.Fatalf("Expected 1 content part, got %d", len(contentParts)) - } - - if contentParts[0].Type != agentContext.ContentText { - t.Errorf("Expected ContentText (from agent), got: %s", contentParts[0].Type) - } - - // The vision-helper assistant should have logged file metadata from Space - // We can verify this through the response text (which should contain the description) - if contentParts[0].Text == "" { - t.Error("Expected non-empty text from vision agent") - } - - t.Logf("✓ Vision agent processed image with file metadata") - t.Logf("Agent response: %s", contentParts[0].Text) - - // Note: The actual validation of Space data happens in the vision-helper's Next hook - // which logs the file metadata. In a real test, we would need to capture those logs - // or have the agent return structured data that we can verify. -} - -// TestVision_MultipleFilesMetadata tests file metadata handling with multiple attachments -func TestVision_MultipleFilesMetadata(t *testing.T) { - testutils.Prepare(t) - defer testutils.Clean(t) - - // Setup test uploader - uploaderName := "test-vision-multi" - manager := setupTestUploader(t, uploaderName) - defer cleanupTestUploader(uploaderName) - - // 1. Upload two test images - imageData1 := generateTestImage(t) - imageData2 := generateTestImage(t) - - // Upload first image - reader1 := strings.NewReader(string(imageData1)) - fileHeader1 := &attachment.FileHeader{ - FileHeader: &multipart.FileHeader{ - Filename: "test-image-1.png", - Size: int64(len(imageData1)), - Header: make(map[string][]string), - }, - } - fileHeader1.Header.Set("Content-Type", "image/png") - - uploadedFile1, err := manager.Upload(context.Background(), fileHeader1, reader1, attachment.UploadOption{ - Groups: []string{"vision", "test"}, - }) - if err != nil { - t.Fatalf("Failed to upload first image: %v", err) - } - - // Upload second image - reader2 := strings.NewReader(string(imageData2)) - fileHeader2 := &attachment.FileHeader{ - FileHeader: &multipart.FileHeader{ - Filename: "test-image-2.png", - Size: int64(len(imageData2)), - Header: make(map[string][]string), - }, - } - fileHeader2.Header.Set("Content-Type", "image/png") - - uploadedFile2, err := manager.Upload(context.Background(), fileHeader2, reader2, attachment.UploadOption{ - Groups: []string{"vision", "test"}, - }) - if err != nil { - t.Fatalf("Failed to upload second image: %v", err) - } - - t.Logf("Uploaded files: %s (ID: %s), %s (ID: %s)", - uploadedFile1.Filename, uploadedFile1.ID, - uploadedFile2.Filename, uploadedFile2.ID) - - // 2. Prepare Vision context with both images - ctx := agentContext.New(context.Background(), nil, "test") - - capabilities := &openai.Capabilities{ - Vision: nil, // No vision support - } - - uses := &agentContext.Uses{ - Vision: "tests.vision-helper", - } - - messages := []agentContext.Message{ - { - Role: "user", - Content: []agentContext.ContentPart{ - { - Type: agentContext.ContentImageURL, - ImageURL: &agentContext.ImageURL{ - URL: "__" + uploaderName + "://" + uploadedFile1.ID, - }, - }, - { - Type: agentContext.ContentImageURL, - ImageURL: &agentContext.ImageURL{ - URL: "__" + uploaderName + "://" + uploadedFile2.ID, - }, - }, - }, - }, - } - - // 3. Call Vision - should handle multiple files' metadata - result, err := content.Vision(ctx, capabilities, messages, uses, false) - if err != nil { - t.Fatalf("Vision function failed: %v", err) - } - - if len(result) != 1 { - t.Fatalf("Expected 1 message, got %d", len(result)) - } - - // 4. Verify both images were processed - contentParts, ok := result[0].Content.([]agentContext.ContentPart) - if !ok { - t.Fatalf("Expected content to be []ContentPart, got %T", result[0].Content) - } - - // Each image should be processed by the vision agent and return text - if len(contentParts) != 2 { - t.Fatalf("Expected 2 content parts (one per image), got %d", len(contentParts)) - } - - for i, part := range contentParts { - if part.Type != agentContext.ContentText { - t.Errorf("Part %d: expected ContentText, got: %s", i, part.Type) - } - if part.Text == "" { - t.Errorf("Part %d: expected non-empty text", i) - } - } - - t.Logf("✓ Multiple files processed with metadata tracking") - t.Logf("File 1 response: %s", contentParts[0].Text) - t.Logf("File 2 response: %s", contentParts[1].Text) -} diff --git a/agent/content/excel.go b/agent/content/excel.go deleted file mode 100644 index 24a8a94d..00000000 --- a/agent/content/excel.go +++ /dev/null @@ -1,48 +0,0 @@ -package content - -import ( - "fmt" - "strings" - - "github.com/yaoapp/gou/connector/openai" - agentContext "github.com/yaoapp/yao/agent/context" -) - -// ExcelHandler handles Microsoft Excel spreadsheets -type ExcelHandler struct{} - -// CanHandle checks if this handler can handle the content type -func (h *ExcelHandler) CanHandle(contentType string, fileType FileType) bool { - return fileType == FileTypeExcel || - contentType == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" || - contentType == "application/vnd.ms-excel" || - strings.Contains(contentType, "excel") || - strings.Contains(contentType, "spreadsheet") -} - -// Handle processes Excel spreadsheet content -func (h *ExcelHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) { - // TODO: Implement Excel handling - // 1. Extract data from .xlsx or .xls file - // 2. Convert to text format (e.g., CSV-like or structured text) - // 3. Handle multiple sheets - // 4. Return Result with formatted text - return nil, fmt.Errorf("not implemented") -} - -// extractExcelText extracts text from Excel file -func extractExcelText(data []byte, contentType string) (string, error) { - // TODO: Implement Excel text extraction - // Handle both .xls (old format) and .xlsx (new format) - // Consider using libraries like: - // - github.com/360EntSecGroup-Skylar/excelize for .xlsx - // Format output as readable text or CSV - return "", fmt.Errorf("not implemented") -} - -// formatExcelAsText formats Excel data as readable text -func formatExcelAsText(sheets map[string][][]string) string { - // TODO: Format multiple sheets into readable text - // Include sheet names, headers, and data - return "" -} diff --git a/agent/content/fetch.go b/agent/content/fetch.go deleted file mode 100644 index eaf35ab5..00000000 --- a/agent/content/fetch.go +++ /dev/null @@ -1,95 +0,0 @@ -package content - -import ( - "fmt" - - agentContext "github.com/yaoapp/yao/agent/context" - "github.com/yaoapp/yao/attachment" -) - -// DefaultFetcher implements the Fetcher interface -type DefaultFetcher struct{} - -// NewFetcher creates a new default fetcher -func NewFetcher() Fetcher { - return &DefaultFetcher{} -} - -// Fetch retrieves content from HTTP URL or uploader wrapper -func (f *DefaultFetcher) Fetch(ctx *agentContext.Context, source Source, url string) (*Info, error) { - switch source { - case SourceHTTP: - return f.fetchHTTP(ctx, url) - case SourceUploader: - return f.fetchUploader(ctx, url) - default: - return nil, fmt.Errorf("unsupported source: %s", source) - } -} - -// fetchHTTP fetches content from an HTTP(S) URL -func (f *DefaultFetcher) fetchHTTP(ctx *agentContext.Context, url string) (*Info, error) { - // TODO: Implement HTTP fetch logic - // 1. Download file from URL - // 2. Detect content type - // 3. Detect file type based on content type and extension - // 4. Return Info with data - return nil, fmt.Errorf("not implemented") -} - -// fetchUploader fetches content from uploader wrapper (__uploader://fileid) -func (f *DefaultFetcher) fetchUploader(ctx *agentContext.Context, wrapper string) (*Info, error) { - // 1. Parse wrapper to get uploader name and file ID - uploaderName, fileID, ok := attachment.Parse(wrapper) - if !ok { - return nil, fmt.Errorf("invalid uploader wrapper format: %s", wrapper) - } - - // 2. Get attachment manager - var manager attachment.FileManager - var exists bool - - // Try to get manager by name - manager, exists = attachment.Managers[uploaderName] - if !exists { - return nil, fmt.Errorf("uploader '%s' not found", uploaderName) - } - - // 3. Get file info - file, err := manager.Info(ctx.Context, fileID) - if err != nil { - return nil, fmt.Errorf("failed to get file info: %w", err) - } - - // 4. Read file content - data, err := manager.Read(ctx.Context, fileID) - if err != nil { - return nil, fmt.Errorf("failed to read file: %w", err) - } - - // 5. Return Info with data - return &Info{ - Data: data, - ContentType: file.ContentType, - Filename: file.Filename, - FileType: DetectFileType(file.ContentType, file.Filename), - URL: wrapper, - Source: SourceUploader, - UploaderName: uploaderName, - FileID: fileID, - }, nil -} - -// parseUploaderWrapper parses uploader wrapper format: __uploader://fileid -func parseUploaderWrapper(wrapper string) (uploaderName, fileID string, err error) { - // TODO: Implement wrapper parsing - // Format: __uploader://fileid - return "", "", fmt.Errorf("not implemented") -} - -// detectFileType detects file type from content type and data -func detectFileType(contentType string, data []byte) FileType { - // TODO: Implement file type detection - // Based on content type and magic bytes - return FileTypeUnknown -} diff --git a/agent/content/image.go b/agent/content/image.go deleted file mode 100644 index 7f3ad0b3..00000000 --- a/agent/content/image.go +++ /dev/null @@ -1,188 +0,0 @@ -package content - -import ( - "fmt" - "strings" - - "github.com/yaoapp/gou/connector/openai" - agentContext "github.com/yaoapp/yao/agent/context" -) - -// ImageHandler handles image content -type ImageHandler struct{} - -// CanHandle checks if this handler can handle the content type -func (h *ImageHandler) CanHandle(contentType string, fileType FileType) bool { - return fileType == FileTypeImage || strings.HasPrefix(contentType, "image/") -} - -// Handle processes image content -// Logic: -// 1. If forceUses is true and uses.Vision is specified -> use vision tool regardless of model capability -// 2. If model supports vision and forceUses is false -> convert to base64 or image_url format -// 3. If model doesn't support vision -> use agent/MCP specified in uses.Vision -func (h *ImageHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) { - if len(info.Data) == 0 { - return nil, fmt.Errorf("no image data to process") - } - - if capabilities == nil { - return nil, fmt.Errorf("no capabilities provided") - } - - // Check if model supports vision - supportsVision, visionFormat := agentContext.GetVisionSupport(capabilities) - - // If forceUses is true and uses.Vision is specified, use vision tool regardless of model capability - if forceUses && uses != nil && uses.Vision != "" { - text, err := h.handleWithVisionAgent(ctx, info, uses.Vision) - if err != nil { - return nil, fmt.Errorf("failed to handle image with vision agent/MCP (forced): %w", err) - } - return &Result{ - Text: text, - }, nil - } - - if supportsVision { - // Model supports vision - return as image_url ContentPart - contentPart, err := h.handleWithVisionModel(ctx, info, visionFormat) - if err != nil { - return nil, fmt.Errorf("failed to handle image with vision model: %w", err) - } - return &Result{ - ContentPart: contentPart, - }, nil - } - - // Model doesn't support vision - use vision agent/MCP - visionTool := "" - if uses != nil && uses.Vision != "" { - visionTool = uses.Vision - } - - if visionTool == "" { - return nil, fmt.Errorf("model doesn't support vision and no vision tool specified in uses.Vision") - } - - // Call vision agent/MCP to extract text - text, err := h.handleWithVisionAgent(ctx, info, visionTool) - if err != nil { - return nil, fmt.Errorf("failed to handle image with vision agent/MCP: %w", err) - } - - return &Result{ - Text: text, - }, nil -} - -// handleWithVisionModel processes image using model's vision capability -func (h *ImageHandler) handleWithVisionModel(ctx *agentContext.Context, info *Info, format agentContext.VisionFormat) (*agentContext.ContentPart, error) { - // Encode image to base64 - base64Data := encodeImageBase64(info.Data, info.ContentType) - - // Format according to model's vision format - switch format { - case agentContext.VisionFormatOpenAI: - // OpenAI format: image_url with data URI - return &agentContext.ContentPart{ - Type: agentContext.ContentImageURL, - ImageURL: &agentContext.ImageURL{ - URL: base64Data, - Detail: agentContext.DetailAuto, - }, - }, nil - - case agentContext.VisionFormatClaude: - // Claude format: also uses image_url but may have different handling - // For now, use the same format as OpenAI - return &agentContext.ContentPart{ - Type: agentContext.ContentImageURL, - ImageURL: &agentContext.ImageURL{ - URL: base64Data, - Detail: agentContext.DetailAuto, - }, - }, nil - - case agentContext.VisionFormatDefault, "": - // Default format (when Vision: true) - use OpenAI format - return &agentContext.ContentPart{ - Type: agentContext.ContentImageURL, - ImageURL: &agentContext.ImageURL{ - URL: base64Data, - Detail: agentContext.DetailAuto, - }, - }, nil - - default: - return nil, fmt.Errorf("unsupported vision format: %s", format) - } -} - -// handleWithVisionAgent processes image using vision agent or MCP -func (h *ImageHandler) handleWithVisionAgent(ctx *agentContext.Context, info *Info, visionTool string) (string, error) { - // Parse vision tool format - // Format can be: - // - "agent_id" (call agent) - // - "mcp:server_id" (call MCP tool) - if strings.HasPrefix(visionTool, "mcp:") { - // MCP tool - serverID := strings.TrimPrefix(visionTool, "mcp:") - return h.callMCPVisionTool(ctx, serverID, info) - } - - // Agent call - return h.callVisionAgent(ctx, visionTool, info) -} - -// callVisionAgent calls a vision agent to describe the image -func (h *ImageHandler) callVisionAgent(ctx *agentContext.Context, agentID string, info *Info) (string, error) { - // Prepare message with image - base64Data := EncodeToBase64DataURI(info.Data, info.ContentType) - - message := agentContext.Message{ - Role: agentContext.RoleUser, - Content: []agentContext.ContentPart{ - { - Type: agentContext.ContentText, - Text: "Please analyze this image.", - }, - { - Type: agentContext.ContentImageURL, - ImageURL: &agentContext.ImageURL{ - URL: base64Data, - Detail: agentContext.DetailAuto, - }, - }, - }, - } - - // Call agent with file metadata in context - // File info (filename, file_id, etc.) will be available in ctx.Metadata["file_info"] - // This allows hooks (especially Next hook) to access and format file information - return CallAgentWithFileInfo(ctx, agentID, message, info) -} - -// callMCPVisionTool calls an MCP vision tool to describe the image -func (h *ImageHandler) callMCPVisionTool(ctx *agentContext.Context, serverID string, info *Info) (string, error) { - // Prepare base64 encoded image for MCP tool - base64Data := EncodeToBase64DataURI(info.Data, info.ContentType) - - // Prepare arguments for MCP tool - arguments := map[string]interface{}{ - "image": base64Data, - "content_type": info.ContentType, - } - - // Call MCP tool (typically "describe_image" or similar) - return CallMCPTool(ctx, serverID, "describe_image", arguments) -} - -// encodeImageBase64 encodes image data to base64 with data URI prefix -func encodeImageBase64(data []byte, contentType string) string { - // Use the common function - if contentType == "" { - contentType = "image/png" // default for images - } - return EncodeToBase64DataURI(data, contentType) -} diff --git a/agent/content/image/image.go b/agent/content/image/image.go new file mode 100644 index 00000000..d10fdbc3 --- /dev/null +++ b/agent/content/image/image.go @@ -0,0 +1,406 @@ +package image + +import ( + "encoding/base64" + "fmt" + "strings" + + "github.com/yaoapp/yao/agent/content/tools" + "github.com/yaoapp/yao/agent/content/types" + agentContext "github.com/yaoapp/yao/agent/context" + "github.com/yaoapp/yao/agent/i18n" + "github.com/yaoapp/yao/agent/output/message" + searchTypes "github.com/yaoapp/yao/agent/search/types" + "github.com/yaoapp/yao/attachment" +) + +// Image handles image content +type Image struct { + options *types.Options +} + +// New creates a new image handler +func New(options *types.Options) *Image { + return &Image{options: options} +} + +// Parse parses image content +// Logic: +// 1. Check model capabilities first +// 2. If forceUses is true and uses.Vision is specified -> use vision tool regardless of model capability +// 3. If model supports vision -> pass through or convert to base64 format +// 4. If model doesn't support vision -> use vision agent/MCP to extract text +func (h *Image) Parse(ctx *agentContext.Context, content agentContext.ContentPart) (agentContext.ContentPart, []*searchTypes.Reference, error) { + if content.ImageURL == nil || content.ImageURL.URL == "" { + return content, nil, fmt.Errorf("image_url content missing URL") + } + + // Check model capabilities first + supportsVision, visionFormat := agentContext.GetVisionSupport(h.options.Capabilities) + + // Check if we should force using Uses tools + forceUses := h.options.CompletionOptions != nil && h.options.CompletionOptions.ForceUses + + // If forceUses is true and uses.Vision is specified, use vision tool regardless of model capability + if forceUses && h.options.CompletionOptions != nil && h.options.CompletionOptions.Uses != nil && h.options.CompletionOptions.Uses.Vision != "" { + // Check cache first before calling agent + cachedText, found, err := h.readFromCache(ctx, content.ImageURL.URL) + if err == nil && found { + return agentContext.ContentPart{ + Type: agentContext.ContentText, + Text: cachedText, + }, nil, nil + } + return h.agent(ctx, content) + } + + // If model supports vision + if supportsVision { + url := content.ImageURL.URL + // If it's already a data URI (base64), pass through directly + if strings.HasPrefix(url, "data:") { + return content, nil, nil + } + // Convert to base64 format + return h.base64(ctx, content, visionFormat) + } + + // Model doesn't support vision - check cache first, then use vision agent/MCP + // Try to get cached text (from attachment's content_preview) + cachedText, found, err := h.readFromCache(ctx, content.ImageURL.URL) + if err == nil && found { + // Cache hit! Return as text content + return agentContext.ContentPart{ + Type: agentContext.ContentText, + Text: cachedText, + }, nil, nil + } + + // No cache, try to use vision agent/MCP + if h.options.CompletionOptions != nil && h.options.CompletionOptions.Uses != nil && h.options.CompletionOptions.Uses.Vision != "" { + return h.agent(ctx, content) + } + + // No vision support and no vision tool specified, return error + return content, nil, fmt.Errorf("model doesn't support vision and no vision tool specified in uses.Vision") +} + +// base64 encodes image content to base64 (for vision support) +func (h *Image) base64(ctx *agentContext.Context, content agentContext.ContentPart, format agentContext.VisionFormat) (agentContext.ContentPart, []*searchTypes.Reference, error) { + if content.ImageURL == nil || content.ImageURL.URL == "" { + return content, nil, fmt.Errorf("image_url content missing URL") + } + + url := content.ImageURL.URL + + // Read image data from source + data, contentType, err := h.read(ctx, url) + if err != nil { + return content, nil, fmt.Errorf("failed to read image: %w", err) + } + + // Encode to base64 data URI + base64Data := EncodeToBase64DataURI(data, contentType) + + // Return as image_url ContentPart + return agentContext.ContentPart{ + Type: agentContext.ContentImageURL, + ImageURL: &agentContext.ImageURL{ + URL: base64Data, + Detail: content.ImageURL.Detail, + }, + }, nil, nil +} + +// read reads image content from various sources +func (h *Image) read(ctx *agentContext.Context, url string) ([]byte, string, error) { + // Determine source type and read accordingly + if strings.HasPrefix(url, "data:") { + // Data URI format: data:image/png;base64,xxxxx + return h.readFromDataURI(url) + } + + if strings.HasPrefix(url, "__") { + // Uploader wrapper format: __uploader://fileid + return h.readFromUploader(ctx, url) + } + + if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") { + // HTTP URL - for now return error, can be implemented later + return nil, "", fmt.Errorf("HTTP URL fetch not implemented yet: %s", url) + } + + // Unknown source + return nil, "", fmt.Errorf("unsupported image source: %s", url) +} + +// readFromDataURI reads image content from a data URI +func (h *Image) readFromDataURI(dataURI string) ([]byte, string, error) { + // Parse data URI: data:image/png;base64,xxxxx + if !strings.HasPrefix(dataURI, "data:") { + return nil, "", fmt.Errorf("invalid data URI format") + } + + // Find the comma separator + commaIndex := strings.Index(dataURI, ",") + if commaIndex == -1 { + return nil, "", fmt.Errorf("invalid data URI: missing comma separator") + } + + // Extract metadata part (e.g., "image/png;base64") + metadata := dataURI[5:commaIndex] // Skip "data:" + base64Data := dataURI[commaIndex+1:] + + // Parse content type + contentType := "image/png" // default + if strings.Contains(metadata, ";") { + parts := strings.Split(metadata, ";") + if len(parts) > 0 && parts[0] != "" { + contentType = parts[0] + } + } else if metadata != "" && metadata != "base64" { + contentType = metadata + } + + // Decode base64 data + data, err := base64.StdEncoding.DecodeString(base64Data) + if err != nil { + return nil, "", fmt.Errorf("failed to decode base64 data: %w", err) + } + + return data, contentType, nil +} + +// readFromUploader reads image content from file uploader __uploader://fileid +func (h *Image) readFromUploader(ctx *agentContext.Context, wrapper string) ([]byte, string, error) { + // Parse wrapper to get uploader name and file ID + uploaderName, fileID, ok := attachment.Parse(wrapper) + if !ok { + return nil, "", fmt.Errorf("invalid uploader wrapper format: %s", wrapper) + } + + // Get attachment manager + manager, exists := attachment.Managers[uploaderName] + if !exists { + return nil, "", fmt.Errorf("uploader '%s' not found", uploaderName) + } + + // Get file info + file, err := manager.Info(ctx.Context, fileID) + if err != nil { + return nil, "", fmt.Errorf("failed to get file info: %w", err) + } + + // Read file content + data, err := manager.Read(ctx.Context, fileID) + if err != nil { + return nil, "", fmt.Errorf("failed to read file: %w", err) + } + + return data, file.ContentType, nil +} + +// readFromCache reads cached text content for an image +func (h *Image) readFromCache(ctx *agentContext.Context, url string) (string, bool, error) { + // Parse URL to check if it's an uploader wrapper + uploaderName, fileID, isWrapper := attachment.Parse(url) + if !isWrapper { + return "", false, nil // Not an uploader wrapper, no cache + } + + // Try attachment manager's content_preview (cross-call cache) + manager, exists := attachment.Managers[uploaderName] + if !exists { + return "", false, nil + } + + // GetText with fullContent=false to get preview (default) + text, err := manager.GetText(ctx.Context, fileID, false) + if err == nil && text != "" { + return text, true, nil + } + + // No cache found + return "", false, nil +} + +// saveToCache saves processed text to cache +func (h *Image) saveToCache(ctx *agentContext.Context, url string, text string) error { + // Parse URL to get uploader name and file ID + uploaderName, fileID, isWrapper := attachment.Parse(url) + if !isWrapper { + return nil // Not an uploader wrapper, nothing to cache + } + + // Save to attachment manager for future calls + manager, exists := attachment.Managers[uploaderName] + if !exists { + return nil + } + + return manager.SaveText(ctx.Context, fileID, text) +} + +// agent calls image agent to parse image content +// Note: Cache check is done in Parse() before calling this method +func (h *Image) agent(ctx *agentContext.Context, content agentContext.ContentPart) (agentContext.ContentPart, []*searchTypes.Reference, error) { + if content.ImageURL == nil || content.ImageURL.URL == "" { + return content, nil, fmt.Errorf("image_url content missing URL") + } + + url := content.ImageURL.URL + + // Get vision tool from options + visionTool := "" + if h.options.CompletionOptions != nil && h.options.CompletionOptions.Uses != nil { + visionTool = h.options.CompletionOptions.Uses.Vision + } + + if visionTool == "" { + return content, nil, fmt.Errorf("no vision tool specified in uses.Vision") + } + + // Parse vision tool format + // Format can be: + // - "agent_id" (call agent) + // - "mcp:server_id" (call MCP tool) + var text string + var err error + if strings.HasPrefix(visionTool, "mcp:") { + // MCP tool + serverID := strings.TrimPrefix(visionTool, "mcp:") + text, err = h.callMCPVisionTool(ctx, serverID, content) + } else { + // Agent call + text, err = h.callVisionAgent(ctx, visionTool, content) + } + + if err != nil { + return content, nil, fmt.Errorf("failed to process image with vision tool: %w", err) + } + + // Cache the result + if cacheErr := h.saveToCache(ctx, url, text); cacheErr != nil { + // Log error but don't fail the request + fmt.Printf("Warning: failed to cache processed text: %v\n", cacheErr) + } + + // Return as text content + return agentContext.ContentPart{ + Type: agentContext.ContentText, + Text: text, + }, nil, nil +} + +// callVisionAgent calls a vision agent to describe the image +func (h *Image) callVisionAgent(ctx *agentContext.Context, agentID string, content agentContext.ContentPart) (string, error) { + // Read image data and convert to base64 + data, contentType, err := h.read(ctx, content.ImageURL.URL) + if err != nil { + return "", fmt.Errorf("failed to read image: %w", err) + } + + base64Data := EncodeToBase64DataURI(data, contentType) + + // Prepare message with image + message := agentContext.Message{ + Role: agentContext.RoleUser, + Content: []agentContext.ContentPart{ + { + Type: agentContext.ContentText, + Text: "Please analyze this image.", + }, + { + Type: agentContext.ContentImageURL, + ImageURL: &agentContext.ImageURL{ + URL: base64Data, + Detail: agentContext.DetailAuto, + }, + }, + }, + } + + // Send loading message + loadingID := h.sendLoading(ctx, i18n.T(ctx.Locale, "content.image.analyzing")) + + // Call agent using the tools package + result, err := tools.CallAgent(ctx, agentID, message) + + // Send done message + h.sendLoadingDone(ctx, loadingID) + + return result, err +} + +// callMCPVisionTool calls an MCP vision tool to describe the image +func (h *Image) callMCPVisionTool(ctx *agentContext.Context, serverID string, content agentContext.ContentPart) (string, error) { + // Read image data and convert to base64 + data, contentType, err := h.read(ctx, content.ImageURL.URL) + if err != nil { + return "", fmt.Errorf("failed to read image: %w", err) + } + + base64Data := EncodeToBase64DataURI(data, contentType) + + // Prepare arguments for MCP tool + arguments := map[string]interface{}{ + "image": base64Data, + "content_type": contentType, + } + + // Send loading message + loadingID := h.sendLoading(ctx, i18n.T(ctx.Locale, "content.image.analyzing")) + + // Call MCP tool (typically "describe_image" or similar) + result, err := tools.CallMCPTool(ctx, serverID, "describe_image", arguments) + + // Send done message + h.sendLoadingDone(ctx, loadingID) + + return result, err +} + +// sendLoading sends a loading message and returns the message ID +func (h *Image) sendLoading(ctx *agentContext.Context, msg string) string { + loadingMsg := &message.Message{ + Type: message.TypeLoading, + Props: map[string]interface{}{ + "message": msg, + }, + } + + msgID, err := ctx.SendStream(loadingMsg) + if err != nil { + return "" + } + return msgID +} + +// sendLoadingDone marks the loading message as done +func (h *Image) sendLoadingDone(ctx *agentContext.Context, loadingID string) { + if loadingID == "" { + return + } + + doneMsg := &message.Message{ + MessageID: loadingID, + Delta: true, + DeltaAction: message.DeltaReplace, + Type: message.TypeLoading, + Props: map[string]interface{}{ + "done": true, + }, + } + + ctx.Send(doneMsg) +} + +// EncodeToBase64DataURI encodes data to base64 with data URI prefix +func EncodeToBase64DataURI(data []byte, contentType string) string { + if contentType == "" { + contentType = "image/png" // default for images + } + + encoded := base64.StdEncoding.EncodeToString(data) + return fmt.Sprintf("data:%s;base64,%s", contentType, encoded) +} diff --git a/agent/content/image/image_test.go b/agent/content/image/image_test.go new file mode 100644 index 00000000..f7764e8d --- /dev/null +++ b/agent/content/image/image_test.go @@ -0,0 +1,374 @@ +package image_test + +import ( + stdContext "context" + "encoding/base64" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/gou/connector/openai" + "github.com/yaoapp/yao/agent/content/image" + contentTypes "github.com/yaoapp/yao/agent/content/types" + agentContext "github.com/yaoapp/yao/agent/context" + "github.com/yaoapp/yao/agent/output/message" + "github.com/yaoapp/yao/agent/testutils" + oauthTypes "github.com/yaoapp/yao/openapi/oauth/types" +) + +// newTestContext creates a Context for testing with commonly used fields pre-populated +func newTestContext(capabilities *openai.Capabilities) *agentContext.Context { + authorized := &oauthTypes.AuthorizedInfo{ + Subject: "test-user", + ClientID: "test-client-id", + UserID: "test-user-123", + TeamID: "test-team-456", + TenantID: "test-tenant-789", + } + + ctx := agentContext.New(stdContext.Background(), authorized, "test-chat") + ctx.AssistantID = "test-assistant" + ctx.Locale = "en-us" + ctx.Theme = "light" + ctx.Client = agentContext.Client{ + Type: "web", + UserAgent: "TestAgent/1.0", + IP: "127.0.0.1", + } + ctx.Referer = agentContext.RefererAPI + ctx.Accept = agentContext.AcceptWebCUI + ctx.Route = "" + ctx.Metadata = make(map[string]interface{}) + ctx.Capabilities = capabilities + ctx.IDGenerator = message.NewIDGenerator() + return ctx +} + +// newTestOptions creates test options with the given capabilities +func newTestOptions(capabilities *openai.Capabilities, completionOptions *agentContext.CompletionOptions) *contentTypes.Options { + return &contentTypes.Options{ + Capabilities: capabilities, + CompletionOptions: completionOptions, + } +} + +// TestParseWithVisionSupport tests parsing image when model supports vision +func TestParseWithVisionSupport(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + // Create capabilities with vision support + capabilities := &openai.Capabilities{ + Vision: "openai", + } + + options := newTestOptions(capabilities, nil) + ctx := newTestContext(capabilities) + + // Create test image content with data URI + base64Data := "data:image/png;base64," + base64.StdEncoding.EncodeToString(createTestPNG()) + content := agentContext.ContentPart{ + Type: agentContext.ContentImageURL, + ImageURL: &agentContext.ImageURL{ + URL: base64Data, + Detail: agentContext.DetailAuto, + }, + } + + handler := image.New(options) + result, refs, err := handler.Parse(ctx, content) + + assert.NoError(t, err) + assert.Nil(t, refs) + assert.Equal(t, agentContext.ContentImageURL, result.Type) + assert.NotNil(t, result.ImageURL) + assert.Equal(t, base64Data, result.ImageURL.URL) // Should pass through unchanged +} + +// TestParseWithoutVisionSupport tests parsing image when model doesn't support vision +func TestParseWithoutVisionSupport(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + // Create capabilities WITHOUT vision support + capabilities := &openai.Capabilities{ + Vision: nil, + } + + options := newTestOptions(capabilities, nil) + ctx := newTestContext(capabilities) + + // Create test image content + base64Data := "data:image/png;base64," + base64.StdEncoding.EncodeToString(createTestPNG()) + content := agentContext.ContentPart{ + Type: agentContext.ContentImageURL, + ImageURL: &agentContext.ImageURL{ + URL: base64Data, + Detail: agentContext.DetailAuto, + }, + } + + handler := image.New(options) + _, _, err := handler.Parse(ctx, content) + + // Should return error because no vision support and no vision tool specified + assert.Error(t, err) + assert.Contains(t, err.Error(), "no vision tool specified") +} + +// TestParseWithEmptyURL tests parsing image with empty URL +func TestParseWithEmptyURL(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + capabilities := &openai.Capabilities{ + Vision: "openai", + } + + options := newTestOptions(capabilities, nil) + ctx := newTestContext(capabilities) + + // Create content with empty URL + content := agentContext.ContentPart{ + Type: agentContext.ContentImageURL, + ImageURL: &agentContext.ImageURL{ + URL: "", + }, + } + + handler := image.New(options) + _, _, err := handler.Parse(ctx, content) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "missing URL") +} + +// TestParseWithNilImageURL tests parsing image with nil ImageURL +func TestParseWithNilImageURL(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + capabilities := &openai.Capabilities{ + Vision: "openai", + } + + options := newTestOptions(capabilities, nil) + ctx := newTestContext(capabilities) + + // Create content with nil ImageURL + content := agentContext.ContentPart{ + Type: agentContext.ContentImageURL, + ImageURL: nil, + } + + handler := image.New(options) + _, _, err := handler.Parse(ctx, content) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "missing URL") +} + +// TestEncodeToBase64DataURI tests base64 encoding +func TestEncodeToBase64DataURI(t *testing.T) { + tests := []struct { + name string + data []byte + contentType string + wantPrefix string + }{ + { + name: "PNG image", + data: []byte{0x89, 0x50, 0x4E, 0x47}, + contentType: "image/png", + wantPrefix: "data:image/png;base64,", + }, + { + name: "JPEG image", + data: []byte{0xFF, 0xD8, 0xFF}, + contentType: "image/jpeg", + wantPrefix: "data:image/jpeg;base64,", + }, + { + name: "Empty content type defaults to PNG", + data: []byte{0x01, 0x02, 0x03}, + contentType: "", + wantPrefix: "data:image/png;base64,", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := image.EncodeToBase64DataURI(tt.data, tt.contentType) + + // Check prefix + assert.True(t, strings.HasPrefix(result, tt.wantPrefix)) + + // Verify base64 encoding by decoding + base64Part := result[len(tt.wantPrefix):] + decoded, err := base64.StdEncoding.DecodeString(base64Part) + assert.NoError(t, err) + + // Verify decoded data matches original + assert.Equal(t, tt.data, decoded) + }) + } +} + +// TestParseDataURIPassthrough tests that data URI images pass through unchanged when vision is supported +func TestParseDataURIPassthrough(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + capabilities := &openai.Capabilities{ + Vision: "openai", + } + + options := newTestOptions(capabilities, nil) + ctx := newTestContext(capabilities) + + // Create test image content with data URI + originalURL := "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + content := agentContext.ContentPart{ + Type: agentContext.ContentImageURL, + ImageURL: &agentContext.ImageURL{ + URL: originalURL, + Detail: agentContext.DetailHigh, + }, + } + + handler := image.New(options) + result, refs, err := handler.Parse(ctx, content) + + assert.NoError(t, err) + assert.Nil(t, refs) + assert.Equal(t, agentContext.ContentImageURL, result.Type) + assert.NotNil(t, result.ImageURL) + assert.Equal(t, originalURL, result.ImageURL.URL) + assert.Equal(t, agentContext.DetailHigh, result.ImageURL.Detail) +} + +// TestParseWithVisionAgent tests parsing image using a vision agent when model doesn't support vision +func TestParseWithVisionAgent(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + // Create capabilities WITHOUT vision support + capabilities := &openai.Capabilities{ + Vision: nil, // Model doesn't support vision + } + + // Configure to use vision agent + completionOptions := &agentContext.CompletionOptions{ + Uses: &agentContext.Uses{ + Vision: "tests.vision-test", // Use our test vision agent + }, + } + + options := newTestOptions(capabilities, completionOptions) + ctx := newTestContext(capabilities) + + // Create test image content with data URI (1x1 red PNG) + base64Data := "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" + content := agentContext.ContentPart{ + Type: agentContext.ContentImageURL, + ImageURL: &agentContext.ImageURL{ + URL: base64Data, + Detail: agentContext.DetailAuto, + }, + } + + handler := image.New(options) + result, refs, err := handler.Parse(ctx, content) + + // Should succeed and return text content (image description from agent) + assert.NoError(t, err) + assert.Nil(t, refs) + assert.Equal(t, agentContext.ContentText, result.Type) + assert.NotEmpty(t, result.Text) // Agent should return some description + t.Logf("Vision agent response: %s", result.Text) +} + +// TestParseWithForceUsesVisionAgent tests forceUses flag with vision agent +func TestParseWithForceUsesVisionAgent(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + // Create capabilities WITH vision support + capabilities := &openai.Capabilities{ + Vision: "openai", // Model supports vision + } + + // Configure to FORCE use vision agent (even though model supports vision) + completionOptions := &agentContext.CompletionOptions{ + ForceUses: true, // Force using the vision tool + Uses: &agentContext.Uses{ + Vision: "tests.vision-test", // Use our test vision agent + }, + } + + options := newTestOptions(capabilities, completionOptions) + ctx := newTestContext(capabilities) + + // Create test image content with data URI + base64Data := "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" + content := agentContext.ContentPart{ + Type: agentContext.ContentImageURL, + ImageURL: &agentContext.ImageURL{ + URL: base64Data, + Detail: agentContext.DetailAuto, + }, + } + + handler := image.New(options) + result, refs, err := handler.Parse(ctx, content) + + // Should succeed and return text content (forced to use agent even though model supports vision) + assert.NoError(t, err) + assert.Nil(t, refs) + assert.Equal(t, agentContext.ContentText, result.Type) + assert.NotEmpty(t, result.Text) // Agent should return some description + t.Logf("Vision agent (forced) response: %s", result.Text) +} + +// createTestPNG creates a minimal valid PNG image (1x1 red pixel) +func createTestPNG() []byte { + return []byte{ + 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, // PNG signature + 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52, // IHDR chunk + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, // 1x1 dimensions + 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53, + 0xDE, 0x00, 0x00, 0x00, 0x0C, 0x49, 0x44, 0x41, // IDAT chunk + 0x54, 0x08, 0xD7, 0x63, 0xF8, 0xCF, 0xC0, 0x00, + 0x00, 0x03, 0x01, 0x01, 0x00, 0x18, 0xDD, 0x8D, + 0xB4, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, // IEND chunk + 0x44, 0xAE, 0x42, 0x60, 0x82, + } +} diff --git a/agent/content/image_test.go b/agent/content/image_test.go deleted file mode 100644 index cc0c18d4..00000000 --- a/agent/content/image_test.go +++ /dev/null @@ -1,261 +0,0 @@ -package content - -import ( - stdContext "context" - "encoding/base64" - "os" - "strings" - "testing" - - "github.com/yaoapp/gou/connector/openai" - agentContext "github.com/yaoapp/yao/agent/context" - "github.com/yaoapp/yao/config" - "github.com/yaoapp/yao/openapi/oauth/types" - "github.com/yaoapp/yao/test" -) - -func TestMain(m *testing.M) { - // Setup test environment - test.Prepare(nil, config.Conf) - defer test.Clean() - - // Run tests - code := m.Run() - os.Exit(code) -} - -// newTestContext creates a Context for testing with commonly used fields pre-populated -func newTestContext(capabilities *openai.Capabilities) *agentContext.Context { - authorized := &types.AuthorizedInfo{ - Subject: "test-user", - ClientID: "test-client-id", - UserID: "test-user-123", - TeamID: "test-team-456", - TenantID: "test-tenant-789", - } - - ctx := agentContext.New(stdContext.Background(), authorized, "test-chat") - ctx.AssistantID = "test-assistant" - ctx.Locale = "en-us" - ctx.Theme = "light" - ctx.Client = agentContext.Client{ - Type: "web", - UserAgent: "TestAgent/1.0", - IP: "127.0.0.1", - } - ctx.Referer = agentContext.RefererAPI - ctx.Accept = agentContext.AcceptWebCUI - ctx.Route = "" - ctx.Metadata = make(map[string]interface{}) - ctx.Capabilities = capabilities - return ctx -} - -func TestImageHandler_CanHandle(t *testing.T) { - handler := &ImageHandler{} - - tests := []struct { - name string - contentType string - fileType FileType - want bool - }{ - {"PNG image", "image/png", FileTypeImage, true}, - {"JPEG image", "image/jpeg", FileTypeImage, true}, - {"GIF image", "image/gif", FileTypeImage, true}, - {"WebP image", "image/webp", FileTypeImage, true}, - {"Text (should not handle)", "text/plain", FileTypeText, false}, - {"PDF (should not handle)", "application/pdf", FileTypePDF, false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := handler.CanHandle(tt.contentType, tt.fileType) - if got != tt.want { - t.Errorf("CanHandle(%q, %q) = %v, want %v", tt.contentType, tt.fileType, got, tt.want) - } - }) - } -} - -func TestImageHandler_Handle_WithVisionSupport(t *testing.T) { - test.Prepare(t, config.Conf) - defer test.Clean() - - handler := &ImageHandler{} - - // Create a simple test image (1x1 red PNG) - pngData := createTestPNG() - - // Create capabilities with vision support - capabilities := &openai.Capabilities{ - Vision: "openai", // Vision is enabled with OpenAI format - } - - // Create test context - ctx := newTestContext(capabilities) - - info := &Info{ - FileType: FileTypeImage, - ContentType: "image/png", - Data: pngData, - } - - result, err := handler.Handle(ctx, info, capabilities, nil, false) - if err != nil { - t.Fatalf("Handle() error = %v", err) - } - - if result == nil { - t.Fatal("Expected non-nil result") - } - - if result.ContentPart == nil { - t.Fatal("Expected ContentPart for vision-supported model") - } - - if result.ContentPart.Type != agentContext.ContentImageURL { - t.Errorf("Expected ContentPart type = %v, got %v", agentContext.ContentImageURL, result.ContentPart.Type) - } - - if result.ContentPart.ImageURL == nil { - t.Fatal("Expected ImageURL to be set") - } - - // Verify base64 encoding - if result.ContentPart.ImageURL.URL == "" { - t.Error("Expected non-empty URL") - } - - // Should be data URI format - if len(result.ContentPart.ImageURL.URL) < 20 { - t.Error("Expected data URI to be longer") - } -} - -func TestImageHandler_Handle_WithoutVisionSupport(t *testing.T) { - test.Prepare(t, config.Conf) - defer test.Clean() - - handler := &ImageHandler{} - - // Create a simple test image - pngData := createTestPNG() - - // Create capabilities WITHOUT vision support - capabilities := &openai.Capabilities{ - Vision: nil, // No vision support - } - - // Create test context - ctx := newTestContext(capabilities) - - info := &Info{ - FileType: FileTypeImage, - ContentType: "image/png", - Data: pngData, - } - - // Should return error because no vision support and no tool - _, err := handler.Handle(ctx, info, capabilities, nil, false) - if err == nil { - t.Error("Expected error when no vision support and no tool specified") - } -} - -func TestImageHandler_Handle_EmptyData(t *testing.T) { - test.Prepare(t, config.Conf) - defer test.Clean() - - handler := &ImageHandler{} - - capabilities := &openai.Capabilities{ - Vision: "openai", - } - - // Create test context - ctx := newTestContext(capabilities) - - info := &Info{ - FileType: FileTypeImage, - ContentType: "image/png", - Data: []byte{}, // Empty data - } - - _, err := handler.Handle(ctx, info, capabilities, nil, false) - if err == nil { - t.Error("Expected error for empty image data") - } -} - -func TestEncodeImageBase64(t *testing.T) { - tests := []struct { - name string - data []byte - contentType string - wantPrefix string - }{ - { - name: "PNG image", - data: []byte{0x89, 0x50, 0x4E, 0x47}, // PNG magic number - contentType: "image/png", - wantPrefix: "data:image/png;base64,", - }, - { - name: "JPEG image", - data: []byte{0xFF, 0xD8, 0xFF}, // JPEG magic number - contentType: "image/jpeg", - wantPrefix: "data:image/jpeg;base64,", - }, - { - name: "Empty content type defaults to PNG", - data: []byte{0x01, 0x02, 0x03}, - contentType: "", - wantPrefix: "data:image/png;base64,", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := encodeImageBase64(tt.data, tt.contentType) - - // Check prefix - if !strings.HasPrefix(result, tt.wantPrefix) { - t.Errorf("Expected prefix %q, got %q", tt.wantPrefix, result[:len(tt.wantPrefix)]) - } - - // Verify base64 encoding by decoding - base64Part := result[len(tt.wantPrefix):] - decoded, err := base64.StdEncoding.DecodeString(base64Part) - if err != nil { - t.Errorf("Failed to decode base64: %v", err) - } - - // Verify decoded data matches original - if len(decoded) != len(tt.data) { - t.Errorf("Decoded length = %d, want %d", len(decoded), len(tt.data)) - } - for i := range decoded { - if decoded[i] != tt.data[i] { - t.Errorf("Decoded byte[%d] = %x, want %x", i, decoded[i], tt.data[i]) - } - } - }) - } -} - -// createTestPNG creates a minimal valid PNG image (1x1 red pixel) -func createTestPNG() []byte { - // This is a minimal valid 1x1 red PNG image - return []byte{ - 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, // PNG signature - 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52, // IHDR chunk - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, // 1x1 dimensions - 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53, - 0xDE, 0x00, 0x00, 0x00, 0x0C, 0x49, 0x44, 0x41, // IDAT chunk - 0x54, 0x08, 0xD7, 0x63, 0xF8, 0xCF, 0xC0, 0x00, - 0x00, 0x03, 0x01, 0x01, 0x00, 0x18, 0xDD, 0x8D, - 0xB4, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, // IEND chunk - 0x44, 0xAE, 0x42, 0x60, 0x82, - } -} diff --git a/agent/content/interfaces.go b/agent/content/interfaces.go deleted file mode 100644 index b6dc5a05..00000000 --- a/agent/content/interfaces.go +++ /dev/null @@ -1,26 +0,0 @@ -package content - -import ( - "github.com/yaoapp/gou/connector/openai" - agentContext "github.com/yaoapp/yao/agent/context" -) - -// Handler defines the interface for handling different content types -// Converts content (images, documents, etc.) to text or standard formats -type Handler interface { - // CanHandle checks if this handler can handle the given content type - CanHandle(contentType string, fileType FileType) bool - - // Handle converts the content and returns processed result - // ctx: agent context (passed from Vision function) - // capabilities: model capabilities (for vision/audio support detection) - // uses: configuration for external tools (agents/MCP servers) - // forceUses: if true, force using Uses tools even when model has native capabilities - Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) -} - -// Fetcher defines the interface for fetching content from different sources -type Fetcher interface { - // Fetch retrieves content from a URL or file ID - Fetch(ctx *agentContext.Context, source Source, url string) (*Info, error) -} diff --git a/agent/content/link/link.go b/agent/content/link/link.go new file mode 100644 index 00000000..cf8e7d53 --- /dev/null +++ b/agent/content/link/link.go @@ -0,0 +1 @@ +package link diff --git a/agent/content/pdf.go b/agent/content/pdf.go deleted file mode 100644 index 5d14bb53..00000000 --- a/agent/content/pdf.go +++ /dev/null @@ -1,50 +0,0 @@ -package content - -import ( - "fmt" - "strings" - - "github.com/yaoapp/gou/connector/openai" - agentContext "github.com/yaoapp/yao/agent/context" -) - -// PDFHandler handles PDF documents -type PDFHandler struct{} - -// CanHandle checks if this handler can handle the content type -func (h *PDFHandler) CanHandle(contentType string, fileType FileType) bool { - return fileType == FileTypePDF || - contentType == "application/pdf" || - strings.Contains(contentType, "pdf") -} - -// Handle processes PDF content -// Logic: -// 1. Check if uses.Vision is specified and supports PDF -// 2. If yes, use vision tool to handle PDF (images + text) -// 3. If no, extract text directly from PDF -func (h *PDFHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) { - // TODO: Implement PDF handling - // 1. Check if vision tool supports PDF - // 2. If yes: - // - Call vision tool to handle PDF (handles both text and images) - // 3. If no: - // - Extract text from PDF using default library - // 4. Return Result with extracted text - return nil, fmt.Errorf("not implemented") -} - -// extractPDFText extracts text content from PDF -func extractPDFText(data []byte) (string, error) { - // TODO: Implement PDF text extraction - // Use a PDF library to extract text - // Consider preserving layout/structure - return "", fmt.Errorf("not implemented") -} - -// handleWithVisionTool processes PDF using vision tool (for PDFs with images) -func handleWithVisionTool(ctx *agentContext.Context, data []byte, visionTool string) (string, error) { - // TODO: Implement vision tool PDF processing - // Some vision tools can handle PDF directly and extract both text and images - return "", fmt.Errorf("not implemented") -} diff --git a/agent/content/pdf/pdf.go b/agent/content/pdf/pdf.go new file mode 100644 index 00000000..163ed8f3 --- /dev/null +++ b/agent/content/pdf/pdf.go @@ -0,0 +1,49 @@ +package pdf + +import ( + "fmt" + + "github.com/yaoapp/yao/agent/content/types" + agentContext "github.com/yaoapp/yao/agent/context" + searchTypes "github.com/yaoapp/yao/agent/search/types" +) + +// PDF handlePDF content +type PDF struct { + options *types.Options +} + +// New creates a new image handler +func New(options *types.Options) *PDF { + return &PDF{options: options} +} + +// Parse parses pdf content +func (h *PDF) Parse(ctx *agentContext.Context, content agentContext.ContentPart) (agentContext.ContentPart, []*searchTypes.Reference, error) { + return content, nil, nil +} + +// asImages converts pdf content to images then parse as image content +func (h *PDF) asImages(ctx *agentContext.Context, content agentContext.ContentPart) (agentContext.ContentPart, []*searchTypes.Reference, error) { + return agentContext.ContentPart{}, nil, nil +} + +// base64 encodes image content to base64 ( for PDF support ) +func (h *PDF) base64(ctx *agentContext.Context, content agentContext.ContentPart) (agentContext.ContentPart, []*searchTypes.Reference, error) { + return agentContext.ContentPart{}, nil, nil +} + +// read image content from file uploader __uploader://fileid return agentContext.ContentPart +func (h *PDF) read(ctx *agentContext.Context, content agentContext.ContentPart) (agentContext.ContentPart, []*searchTypes.Reference, error) { + return agentContext.ContentPart{}, nil, fmt.Errorf("not implemented") +} + +// read image content from file uploader __uploader://fileid return agentContext.ContentPart +func (h *PDF) readFromCache(ctx *agentContext.Context, content agentContext.ContentPart) (agentContext.ContentPart, []*searchTypes.Reference, error) { + return agentContext.ContentPart{}, nil, fmt.Errorf("not implemented") +} + +// read image content from file uploader __uploader://fileid return base64 encoded string +func (h *PDF) readFromUploader(ctx *agentContext.Context, content agentContext.ContentPart) (string, error) { + return "", fmt.Errorf("not implemented") +} diff --git a/agent/content/registry.go b/agent/content/registry.go deleted file mode 100644 index 77c98d23..00000000 --- a/agent/content/registry.go +++ /dev/null @@ -1,47 +0,0 @@ -package content - -import ( - "fmt" - - "github.com/yaoapp/gou/connector/openai" - agentContext "github.com/yaoapp/yao/agent/context" -) - -// Registry holds all registered content handlers -type Registry struct { - handlers []Handler -} - -// NewRegistry creates a new handler registry with default handlers -func NewRegistry() *Registry { - return &Registry{ - handlers: []Handler{ - &ImageHandler{}, - &AudioHandler{}, - &PDFHandler{}, - &WordHandler{}, - &ExcelHandler{}, - &TextHandler{}, - }, - } -} - -// GetHandler finds the appropriate handler for the given content -func (r *Registry) GetHandler(contentType string, fileType FileType) Handler { - for _, handler := range r.handlers { - if handler.CanHandle(contentType, fileType) { - return handler - } - } - return nil -} - -// Handle processes content using the appropriate handler -func (r *Registry) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) { - handler := r.GetHandler(info.ContentType, info.FileType) - if handler == nil { - return nil, fmt.Errorf("no handler found for content type: %s, file type: %s", info.ContentType, info.FileType) - } - - return handler.Handle(ctx, info, capabilities, uses, forceUses) -} diff --git a/agent/content/text.go b/agent/content/text.go deleted file mode 100644 index 6f64d03b..00000000 --- a/agent/content/text.go +++ /dev/null @@ -1,123 +0,0 @@ -package content - -import ( - "encoding/json" - "fmt" - "strings" - - "github.com/yaoapp/gou/connector/openai" - agentContext "github.com/yaoapp/yao/agent/context" -) - -// TextHandler handles plain text, code files, CSV, JSON, XML, Markdown, etc. -type TextHandler struct{} - -// CanHandle checks if this handler can handle the content type -func (h *TextHandler) CanHandle(contentType string, fileType FileType) bool { - // Handle explicit text file types - if fileType == FileTypeText || fileType == FileTypeCSV || fileType == FileTypeJSON { - return true - } - - // Handle text-based MIME types - if strings.HasPrefix(contentType, "text/") { - return true - } - - // Handle common text-based content types - textContentTypes := []string{ - "application/json", - "application/xml", - "application/javascript", - "application/typescript", - "application/x-yaml", - "application/yaml", - "application/toml", - "application/x-sh", - "application/x-python", - "application/x-ruby", - "application/x-perl", - } - - for _, ct := range textContentTypes { - if contentType == ct || strings.Contains(contentType, ct) { - return true - } - } - - return false -} - -// Handle processes text content -func (h *TextHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) { - if len(info.Data) == 0 { - return nil, fmt.Errorf("no data to process") - } - - var text string - var err error - - // Handle different text formats - switch { - case info.FileType == FileTypeCSV || strings.Contains(info.ContentType, "csv"): - // Format CSV as readable text (for now, just return as-is, can enhance later) - text = string(info.Data) - - case info.FileType == FileTypeJSON || - info.ContentType == "application/json" || - strings.Contains(info.ContentType, "json"): - // Pretty print JSON - text, err = formatJSONAsText(info.Data) - if err != nil { - // If JSON parsing fails, return raw text - text = string(info.Data) - } - - case info.ContentType == "application/xml" || - strings.Contains(info.ContentType, "xml"): - // For now, return XML as-is (can enhance formatting later) - text = string(info.Data) - - default: - // Plain text, code files, markdown, etc. - text, err = readTextContent(info.Data, info.ContentType) - if err != nil { - return nil, fmt.Errorf("failed to read text content: %w", err) - } - } - - return &Result{ - Text: text, - }, nil -} - -// readTextContent reads text content from data -func readTextContent(data []byte, contentType string) (string, error) { - // For now, assume UTF-8 encoding - // TODO: Add encoding detection if needed (e.g., using golang.org/x/text/encoding) - return string(data), nil -} - -// formatCSVAsText formats CSV data as readable text -func formatCSVAsText(data []byte) (string, error) { - // TODO: Parse CSV and format as readable table - // Consider using encoding/csv package - // For now, just return as-is - return string(data), nil -} - -// formatJSONAsText formats JSON data as readable text -func formatJSONAsText(data []byte) (string, error) { - // Pretty print JSON with indentation - var obj interface{} - if err := json.Unmarshal(data, &obj); err != nil { - return "", err - } - - pretty, err := json.MarshalIndent(obj, "", " ") - if err != nil { - return "", err - } - - return string(pretty), nil -} diff --git a/agent/content/text_test.go b/agent/content/text_test.go deleted file mode 100644 index 029a4625..00000000 --- a/agent/content/text_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package content - -import ( - "testing" -) - -func TestTextHandler_CanHandle(t *testing.T) { - handler := &TextHandler{} - - tests := []struct { - name string - contentType string - fileType FileType - want bool - }{ - {"Plain text", "text/plain", FileTypeText, true}, - {"Markdown", "text/markdown", FileTypeText, true}, - {"HTML", "text/html", FileTypeText, true}, - {"JSON", "application/json", FileTypeJSON, true}, - {"JavaScript", "application/javascript", FileTypeText, true}, - {"TypeScript", "application/typescript", FileTypeText, true}, - {"YAML", "application/yaml", FileTypeText, true}, - {"CSV", "text/csv", FileTypeCSV, true}, - {"XML", "application/xml", FileTypeText, true}, - {"PDF (should not handle)", "application/pdf", FileTypePDF, false}, - {"Image (should not handle)", "image/png", FileTypeImage, false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := handler.CanHandle(tt.contentType, tt.fileType) - if got != tt.want { - t.Errorf("CanHandle(%q, %q) = %v, want %v", tt.contentType, tt.fileType, got, tt.want) - } - }) - } -} - -func TestTextHandler_Handle(t *testing.T) { - handler := &TextHandler{} - - tests := []struct { - name string - info *Info - wantErr bool - checkResult func(*testing.T, *Result) - }{ - { - name: "Plain text", - info: &Info{ - FileType: FileTypeText, - ContentType: "text/plain", - Data: []byte("Hello, World!"), - }, - wantErr: false, - checkResult: func(t *testing.T, r *Result) { - if r.Text != "Hello, World!" { - t.Errorf("Expected 'Hello, World!', got %q", r.Text) - } - }, - }, - { - name: "JSON with pretty print", - info: &Info{ - FileType: FileTypeJSON, - ContentType: "application/json", - Data: []byte(`{"name":"test","value":123}`), - }, - wantErr: false, - checkResult: func(t *testing.T, r *Result) { - // Should be pretty printed - if len(r.Text) <= len(`{"name":"test","value":123}`) { - t.Errorf("JSON should be pretty printed, got: %q", r.Text) - } - }, - }, - { - name: "Code file (Go)", - info: &Info{ - FileType: FileTypeText, - ContentType: "text/plain", - Data: []byte("package main\n\nfunc main() {\n\tprintln(\"Hello\")\n}"), - }, - wantErr: false, - checkResult: func(t *testing.T, r *Result) { - if r.Text == "" { - t.Error("Expected non-empty text for Go code") - } - }, - }, - { - name: "Empty data", - info: &Info{ - FileType: FileTypeText, - ContentType: "text/plain", - Data: []byte{}, - }, - wantErr: true, - }, - } - - // Create test context - testCtx := newTestContext(nil) - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result, err := handler.Handle(testCtx, tt.info, nil, nil, false) - if (err != nil) != tt.wantErr { - t.Errorf("Handle() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !tt.wantErr && tt.checkResult != nil { - tt.checkResult(t, result) - } - }) - } -} - -func TestDetectFileType(t *testing.T) { - tests := []struct { - name string - contentType string - filename string - want FileType - }{ - {"Go file", "text/plain", "main.go", FileTypeText}, - {"Python file", "text/plain", "script.py", FileTypeText}, - {"JavaScript file", "application/javascript", "app.js", FileTypeText}, - {"TypeScript file", "text/plain", "index.ts", FileTypeText}, - {"Markdown file", "text/markdown", "README.md", FileTypeText}, - {"JSON file", "application/json", "config.json", FileTypeJSON}, - {"YAML file", "text/plain", "config.yml", FileTypeText}, - {"PDF file", "application/pdf", "document.pdf", FileTypePDF}, - {"Image file", "image/png", "photo.png", FileTypeImage}, - {"CSV file", "text/csv", "data.csv", FileTypeCSV}, - {"XML file", "application/xml", "config.xml", FileTypeXML}, - {"Shell script", "text/plain", "script.sh", FileTypeText}, - {"Dockerfile", "text/plain", "Dockerfile", FileTypeText}, - {"gitignore", "text/plain", ".gitignore", FileTypeText}, - {"HTML", "text/html", "index.html", FileTypeText}, - {"CSS", "text/css", "styles.css", FileTypeText}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := DetectFileType(tt.contentType, tt.filename) - if got != tt.want { - t.Errorf("DetectFileType(%q, %q) = %v, want %v", tt.contentType, tt.filename, got, tt.want) - } - }) - } -} diff --git a/agent/content/tools.go b/agent/content/tools.go deleted file mode 100644 index d0de3806..00000000 --- a/agent/content/tools.go +++ /dev/null @@ -1,240 +0,0 @@ -package content - -import ( - "context" - "encoding/base64" - "fmt" - "sync" - - jsoniter "github.com/json-iterator/go" - "github.com/yaoapp/gou/mcp" - "github.com/yaoapp/kun/log" - "github.com/yaoapp/yao/agent/caller" - agentContext "github.com/yaoapp/yao/agent/context" -) - -// fileInfoMutex protects concurrent access to files_info list in Space -var fileInfoMutex sync.Mutex - -// CallAgent calls an agent to process content (vision, audio, etc.) -// This is a generic function that can be used by any handler -func CallAgent(ctx *agentContext.Context, agentID string, message agentContext.Message) (string, error) { - if caller.AgentGetterFunc == nil { - return "", fmt.Errorf("AgentGetterFunc not initialized") - } - - // Load the agent by ID using the injected function - agent, err := caller.AgentGetterFunc(agentID) - if err != nil { - return "", fmt.Errorf("failed to load agent %s: %w", agentID, err) - } - - // Call the agent with the message - messages := []agentContext.Message{message} - - // Note: Connector is now in Options (call-level parameter), not Context - // For A2A calls, skip history and output (we only need the response data) - opts := &agentContext.Options{Skip: &agentContext.Skip{History: true, Output: true}} // Skip history and output - response, err := agent.Stream(ctx, messages, opts) - if err != nil { - return "", fmt.Errorf("failed to call agent %s: %w", agentID, err) - } - - // Extract text from agent response - // Two formats are supported: - // 1. Custom Hook response (from Next hook) - response.Next - // 2. Standard Agent Stream response (LLM completion) - response.Completion - - return extractTextFromResponse(response) -} - -// CallAgentWithFileInfo calls an agent to process content with file metadata -// The file metadata is passed via ctx.Memory.Context for access by hooks (especially Next hook) -// Uses Memory.Context (request-scoped) to avoid creating context copies and ensure proper cleanup -// -// Memory Keys (with agent ID as namespace prefix to avoid conflicts between different agents): -// - {agentID}:files_info - List of all files being processed by this agent (array) -// - {agentID}:current_file - Currently processing file (single object) -func CallAgentWithFileInfo(ctx *agentContext.Context, agentID string, message agentContext.Message, info *Info) (string, error) { - // Store file information in Memory.Context if available - if info != nil && ctx.Memory != nil && ctx.Memory.Context != nil { - fileInfo := map[string]interface{}{ - "url": info.URL, - "filename": info.Filename, - "content_type": info.ContentType, - "file_type": string(info.FileType), - "source": string(info.Source), - } - - // Add uploader-specific information if available - if info.UploaderName != "" { - fileInfo["uploader_name"] = info.UploaderName - } - if info.FileID != "" { - fileInfo["file_id"] = info.FileID - } - - // Use agent ID as namespace prefix for Memory keys - filesListKey := agentID + ":files_info" - currentFileKey := agentID + ":current_file" - - // Thread-safe: append current file to files list - fileInfoMutex.Lock() - var filesList []map[string]interface{} - if existing, ok := ctx.Memory.Context.Get(filesListKey); ok { - // Convert existing data to []map[string]interface{} - if existingList, ok := existing.([]interface{}); ok { - for _, item := range existingList { - if itemMap, ok := item.(map[string]interface{}); ok { - filesList = append(filesList, itemMap) - } - } - } else if existingList, ok := existing.([]map[string]interface{}); ok { - filesList = existingList - } - } - // Append current file to list - filesList = append(filesList, fileInfo) - ctx.Memory.Context.Set(filesListKey, filesList, 0) - fileInfoMutex.Unlock() - - // Store current file in Memory.Context - if err := ctx.Memory.Context.Set(currentFileKey, fileInfo, 0); err != nil { - log.Trace("[Content] Failed to set current file info in Memory.Context: %v", err) - } - - // Ensure cleanup after agent call completes - defer func() { - // Clean up current file - if err := ctx.Memory.Context.Del(currentFileKey); err != nil { - log.Trace("[Content] Failed to delete current file info from Memory.Context: %v", err) - } - // Clean up files list (reset for next call) - if err := ctx.Memory.Context.Del(filesListKey); err != nil { - log.Trace("[Content] Failed to delete files list from Memory.Context: %v", err) - } - }() - } - - // Call the agent with the original context - return CallAgent(ctx, agentID, message) -} - -// extractTextFromResponse extracts text from agent response -// Now that agent.Stream() returns *agentContext.Response directly, -// we can access fields without type assertions or JSON conversion. -// -// Priority: -// 1. Check response.Next (custom hook data) → return complete data -// 2. Check response.Completion (standard LLM response) → extract text only -func extractTextFromResponse(response *agentContext.Response) (string, error) { - if response == nil { - return "", fmt.Errorf("agent returned nil response") - } - - // Priority 1: Check Next field (custom hook data) - // If Next hook returns custom data, return the complete structure - if response.Next != nil { - // If next is a string, return directly - if nextStr, ok := response.Next.(string); ok { - return nextStr, nil - } - // Otherwise, JSON stringify to preserve complete structure - jsonBytes, err := jsoniter.Marshal(response.Next) - if err != nil { - return "", fmt.Errorf("failed to serialize next hook data: %w", err) - } - return string(jsonBytes), nil - } - - // Priority 2: Check Completion field (standard LLM response) - // Extract text content from the LLM completion - if response.Completion != nil { - // Content can be string or []ContentPart (multimodal) - switch v := response.Completion.Content.(type) { - case string: - // Simple text content - return v, nil - case []interface{}: - // Multimodal content array - extract all text parts - var text string - for _, part := range v { - if partMap, ok := part.(map[string]interface{}); ok { - if partType, _ := partMap["type"].(string); partType == "text" { - if textContent, ok := partMap["text"].(string); ok { - text += textContent - } - } - } - } - if text != "" { - return text, nil - } - // No text found in content parts - return "", fmt.Errorf("no text content found in completion content parts") - } - } - - // No content found - return "", fmt.Errorf("no content found in agent response") -} - -// CallMCPTool calls an MCP tool to process content -// This is a generic function that can be used by any handler -func CallMCPTool(ctx *agentContext.Context, serverID string, toolName string, arguments map[string]interface{}) (string, error) { - // Get MCP context for cancellation/timeout control - mcpCtx := ctx.Context - if mcpCtx == nil { - mcpCtx = context.Background() - } - - // Get MCP client - client, err := mcp.Select(serverID) - if err != nil { - return "", fmt.Errorf("failed to select MCP client '%s': %w", serverID, err) - } - - // Call the tool - log.Trace("[Content] Calling MCP tool: %s (server: %s)", toolName, serverID) - callResult, err := client.CallTool(mcpCtx, toolName, arguments) - if err != nil { - return "", fmt.Errorf("MCP tool call failed: %w", err) - } - - // Check if result is an error - if callResult.IsError { - return "", fmt.Errorf("MCP tool returned error: %v", callResult.Content) - } - - // Extract text content from result - // callResult.Content is []ToolContent - var text string - for _, content := range callResult.Content { - if content.Type == "text" { - text += content.Text - } - // Can also handle other types like image, resource if needed - } - - if text == "" { - // If no text content found, return error - return "", fmt.Errorf("MCP tool returned no text content") - } - - return text, nil -} - -// EncodeToBase64DataURI encodes data to base64 with data URI prefix -// This is useful for encoding images, audio, or other binary data -func EncodeToBase64DataURI(data []byte, contentType string) string { - // Ensure we have a valid content type - if contentType == "" { - contentType = "application/octet-stream" // default - } - - // Encode to base64 - encoded := base64.StdEncoding.EncodeToString(data) - - // Return data URI format - return fmt.Sprintf("data:%s;base64,%s", contentType, encoded) -} diff --git a/agent/content/tools/tools.go b/agent/content/tools/tools.go new file mode 100644 index 00000000..cf75ecd1 --- /dev/null +++ b/agent/content/tools/tools.go @@ -0,0 +1,125 @@ +package tools + +import ( + "context" + "fmt" + + jsoniter "github.com/json-iterator/go" + "github.com/yaoapp/gou/mcp" + "github.com/yaoapp/kun/log" + "github.com/yaoapp/yao/agent/caller" + agentContext "github.com/yaoapp/yao/agent/context" +) + +// CallAgent calls an agent to process content (vision, audio, etc.) +func CallAgent(ctx *agentContext.Context, agentID string, message agentContext.Message) (string, error) { + if caller.AgentGetterFunc == nil { + return "", fmt.Errorf("AgentGetterFunc not initialized") + } + + // Load the agent by ID using the injected function + agent, err := caller.AgentGetterFunc(agentID) + if err != nil { + return "", fmt.Errorf("failed to load agent %s: %w", agentID, err) + } + + // Call the agent with the message + messages := []agentContext.Message{message} + + // For A2A calls, skip history and output (we only need the response data) + opts := &agentContext.Options{Skip: &agentContext.Skip{History: true, Output: true}} + response, err := agent.Stream(ctx, messages, opts) + if err != nil { + return "", fmt.Errorf("failed to call agent %s: %w", agentID, err) + } + + // Extract text from agent response + return ExtractTextFromResponse(response) +} + +// ExtractTextFromResponse extracts text from agent response +func ExtractTextFromResponse(response *agentContext.Response) (string, error) { + if response == nil { + return "", fmt.Errorf("agent returned nil response") + } + + // Priority 1: Check Next field (custom hook data) + if response.Next != nil { + if nextStr, ok := response.Next.(string); ok { + return nextStr, nil + } + // Otherwise, JSON stringify to preserve complete structure + jsonBytes, err := jsoniter.Marshal(response.Next) + if err != nil { + return "", fmt.Errorf("failed to serialize next hook data: %w", err) + } + return string(jsonBytes), nil + } + + // Priority 2: Check Completion field (standard LLM response) + if response.Completion != nil { + switch v := response.Completion.Content.(type) { + case string: + return v, nil + case []interface{}: + // Multimodal content array - extract all text parts + var text string + for _, part := range v { + if partMap, ok := part.(map[string]interface{}); ok { + if partType, _ := partMap["type"].(string); partType == "text" { + if textContent, ok := partMap["text"].(string); ok { + text += textContent + } + } + } + } + if text != "" { + return text, nil + } + return "", fmt.Errorf("no text content found in completion content parts") + } + } + + return "", fmt.Errorf("no content found in agent response") +} + +// CallMCPTool calls an MCP tool to process content +func CallMCPTool(ctx *agentContext.Context, serverID string, toolName string, arguments map[string]interface{}) (string, error) { + // Get MCP context for cancellation/timeout control + mcpCtx := ctx.Context + if mcpCtx == nil { + mcpCtx = context.Background() + } + + // Get MCP client + client, err := mcp.Select(serverID) + if err != nil { + return "", fmt.Errorf("failed to select MCP client '%s': %w", serverID, err) + } + + // Call the tool + log.Trace("[Content] Calling MCP tool: %s (server: %s)", toolName, serverID) + callResult, err := client.CallTool(mcpCtx, toolName, arguments) + if err != nil { + return "", fmt.Errorf("MCP tool call failed: %w", err) + } + + // Check if result is an error + if callResult.IsError { + return "", fmt.Errorf("MCP tool returned error: %v", callResult.Content) + } + + // Extract text content from result + var text string + for _, content := range callResult.Content { + if content.Type == "text" { + text += content.Text + } + } + + if text == "" { + return "", fmt.Errorf("MCP tool returned no text content") + } + + return text, nil +} diff --git a/agent/content/types.go b/agent/content/types.go deleted file mode 100644 index b9d4fd70..00000000 --- a/agent/content/types.go +++ /dev/null @@ -1,262 +0,0 @@ -package content - -import "github.com/yaoapp/yao/agent/context" - -// FileType represents the type of file content -type FileType string - -const ( - // Image types - FileTypeImage FileType = "image" - - // Audio types - FileTypeAudio FileType = "audio" - - // Document types - FileTypeText FileType = "text" - FileTypePDF FileType = "pdf" - FileTypeWord FileType = "word" - FileTypeExcel FileType = "excel" - FileTypePPT FileType = "ppt" - FileTypeCSV FileType = "csv" - - // Data types - FileTypeJSON FileType = "json" - FileTypeXML FileType = "xml" - - // Binary - FileTypeBinary FileType = "binary" - - // Other - FileTypeUnknown FileType = "unknown" -) - -// Source represents where the content comes from -type Source string - -const ( - SourceHTTP Source = "http" // HTTP(S) URL - SourceUploader Source = "uploader" // Uploader wrapper: __uploader://fileid - SourceBase64 Source = "base64" // Base64 encoded data - SourceLocal Source = "local" // Local file path -) - -// Result represents the result of content handling -type Result struct { - Text string // Extracted text content - ContentPart *context.ContentPart // Processed ContentPart (for model input) - Metadata map[string]interface{} // Additional metadata - Error error // Error if handling failed -} - -// Info holds information about a content part to be handled -type Info struct { - Source Source // Where the content comes from - FileType FileType // Type of the file - ContentType string // MIME content type - URL string // Original URL or file ID - Filename string // Original filename (if available) - Data []byte // File data (if already fetched) - - // For uploader wrapper - UploaderName string // Uploader name from wrapper - FileID string // File ID from wrapper -} - -// DetectFileType detects file type from content type, filename, and file extension -func DetectFileType(contentType, filename string) FileType { - // Check by content type first - switch { - case contentType == "application/pdf": - return FileTypePDF - case contentType == "application/json": - return FileTypeJSON - case contentType == "application/xml" || contentType == "text/xml": - return FileTypeXML - case contentType == "text/csv": - return FileTypeCSV - case contentType == "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - contentType == "application/msword": - return FileTypeWord - case contentType == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - contentType == "application/vnd.ms-excel": - return FileTypeExcel - case contentType == "application/vnd.openxmlformats-officedocument.presentationml.presentation", - contentType == "application/vnd.ms-powerpoint": - return FileTypePPT - } - - // Check image types - if isImageContentType(contentType) { - return FileTypeImage - } - - // Check audio types - if isAudioContentType(contentType) { - return FileTypeAudio - } - - // Check text types - if isTextContentType(contentType) { - return FileTypeText - } - - // If content type doesn't help, check file extension - if filename != "" { - if ext := getFileExtension(filename); ext != "" { - return detectTypeByExtension(ext) - } - } - - return FileTypeUnknown -} - -// isImageContentType checks if content type is an image -func isImageContentType(contentType string) bool { - return contentType != "" && - (contentType == "image/png" || - contentType == "image/jpeg" || - contentType == "image/jpg" || - contentType == "image/gif" || - contentType == "image/webp" || - contentType == "image/svg+xml" || - contentType == "image/bmp") -} - -// isAudioContentType checks if content type is audio -func isAudioContentType(contentType string) bool { - return contentType != "" && - (contentType == "audio/mpeg" || - contentType == "audio/mp3" || - contentType == "audio/wav" || - contentType == "audio/ogg" || - contentType == "audio/flac" || - contentType == "audio/aac") -} - -// isTextContentType checks if content type is text-based -func isTextContentType(contentType string) bool { - if contentType == "" { - return false - } - - // Common text MIME types - textTypes := []string{ - "text/plain", - "text/html", - "text/css", - "text/javascript", - "text/markdown", - "text/x-markdown", - "application/javascript", - "application/typescript", - "application/x-yaml", - "application/yaml", - "application/toml", - "application/x-sh", - "application/x-python", - "application/x-ruby", - "application/x-perl", - "application/x-php", - "application/x-go", - } - - for _, t := range textTypes { - if contentType == t { - return true - } - } - - return false -} - -// getFileExtension extracts file extension from filename (without dot) -func getFileExtension(filename string) string { - for i := len(filename) - 1; i >= 0; i-- { - if filename[i] == '.' { - return filename[i+1:] - } - if filename[i] == '/' || filename[i] == '\\' { - break - } - } - return "" -} - -// detectTypeByExtension detects file type by file extension -func detectTypeByExtension(ext string) FileType { - // Normalize to lowercase - ext = toLower(ext) - - // Image extensions - imageExts := []string{"png", "jpg", "jpeg", "gif", "webp", "svg", "bmp", "ico"} - for _, e := range imageExts { - if ext == e { - return FileTypeImage - } - } - - // Audio extensions - audioExts := []string{"mp3", "wav", "ogg", "flac", "aac", "m4a"} - for _, e := range audioExts { - if ext == e { - return FileTypeAudio - } - } - - // Document extensions - switch ext { - case "pdf": - return FileTypePDF - case "doc", "docx": - return FileTypeWord - case "xls", "xlsx": - return FileTypeExcel - case "ppt", "pptx": - return FileTypePPT - case "csv": - return FileTypeCSV - case "json": - return FileTypeJSON - case "xml": - return FileTypeXML - } - - // Code and text file extensions (very comprehensive list) - textExts := []string{ - "txt", "text", "md", "markdown", "rst", - // Programming languages - "go", "py", "js", "ts", "jsx", "tsx", "java", "c", "cpp", "h", "hpp", - "cs", "rb", "php", "pl", "swift", "kt", "rs", "scala", "clj", - // Web - "html", "htm", "css", "scss", "sass", "less", - // Config - "yaml", "yml", "toml", "ini", "conf", "config", - // Shell - "sh", "bash", "zsh", "fish", - // Data - "sql", "graphql", "proto", - // Others - "log", "gitignore", "env", "dockerfile", - } - for _, e := range textExts { - if ext == e { - return FileTypeText - } - } - - return FileTypeUnknown -} - -// toLower converts ASCII string to lowercase (simple version) -func toLower(s string) string { - result := make([]byte, len(s)) - for i := 0; i < len(s); i++ { - c := s[i] - if c >= 'A' && c <= 'Z' { - c += 'a' - 'A' - } - result[i] = c - } - return string(result) -} diff --git a/agent/content/types/types.go b/agent/content/types/types.go new file mode 100644 index 00000000..eb7ba32e --- /dev/null +++ b/agent/content/types/types.go @@ -0,0 +1,23 @@ +package types + +import ( + "github.com/yaoapp/gou/connector" + "github.com/yaoapp/gou/connector/openai" + agentContext "github.com/yaoapp/yao/agent/context" +) + +// Options represents the options for the content +type Options struct { + + // Connector, Current connector instance + Connector connector.Connector + + // Capabilities, Current capabilities instance + Capabilities *openai.Capabilities + + // CompletionOptions, Current completion options instance + CompletionOptions *agentContext.CompletionOptions + + // StreamOptions, Current stream options instance + StreamOptions *agentContext.StreamOptions +} diff --git a/agent/content/word.go b/agent/content/word.go deleted file mode 100644 index 9b438097..00000000 --- a/agent/content/word.go +++ /dev/null @@ -1,39 +0,0 @@ -package content - -import ( - "fmt" - "strings" - - "github.com/yaoapp/gou/connector/openai" - agentContext "github.com/yaoapp/yao/agent/context" -) - -// WordHandler handles Microsoft Word documents -type WordHandler struct{} - -// CanHandle checks if this handler can handle the content type -func (h *WordHandler) CanHandle(contentType string, fileType FileType) bool { - return fileType == FileTypeWord || - contentType == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || - contentType == "application/msword" || - strings.Contains(contentType, "word") -} - -// Handle processes Word document content -func (h *WordHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) { - // TODO: Implement Word document handling - // 1. Extract text from .docx or .doc file - // 2. Preserve formatting information if needed - // 3. Return Result with extracted text - return nil, fmt.Errorf("not implemented") -} - -// extractWordText extracts text from Word document -func extractWordText(data []byte, contentType string) (string, error) { - // TODO: Implement Word text extraction - // Handle both .doc (old format) and .docx (new format) - // Consider using libraries like: - // - github.com/unidoc/unioffice for .docx - // - Other libraries for .doc - return "", fmt.Errorf("not implemented") -} diff --git a/agent/context/types.go b/agent/context/types.go index 3f501a66..900dd86f 100644 --- a/agent/context/types.go +++ b/agent/context/types.go @@ -302,6 +302,9 @@ type Options struct { // Agent mode, use to select the mode of the request, default is "chat" Mode string `json:"mode,omitempty"` // Agent mode, use to select the mode of the request, default is "chat" + // Uses configuration, allow hook to override wrapper configurations for vision, audio, search, and fetch + Uses *Uses `json:"uses,omitempty"` // Uses configuration, allow hook to override wrapper configurations for vision, audio, search, and fetch + // Metadata for passing custom data to hooks (e.g., scenario selection) Metadata map[string]any `json:"metadata,omitempty"` // Custom metadata passed to Create/Next hooks } diff --git a/agent/i18n/builtin.go b/agent/i18n/builtin.go index e6aee7bb..45ba220f 100644 --- a/agent/i18n/builtin.go +++ b/agent/i18n/builtin.go @@ -99,6 +99,9 @@ func init() { "kb.chat.name": "Chat Knowledge Base", "kb.chat.description": "Auto-created knowledge base collection for chat sessions", + // Content: content/image/image.go - Image processing messages + "content.image.analyzing": "Analyzing image...", + // Search: assistant/search.go - Output messages "search.loading": "Searching...", "search.success": "Found %d references", @@ -192,6 +195,9 @@ func init() { "kb.chat.name": "聊天知识库", "kb.chat.description": "自动为聊天会话创建的知识库集合", + // Content: content/image/image.go - Image processing messages + "content.image.analyzing": "正在分析图片...", + // Search: assistant/search.go - Output messages "search.loading": "正在搜索...", "search.success": "找到 %d 条参考资料", @@ -313,6 +319,9 @@ func init() { "kb.chat.name": "聊天知识库", "kb.chat.description": "自动为聊天会话创建的知识库集合", + // Content: content/image/image.go - Image processing messages + "content.image.analyzing": "正在分析图片...", + // Search: assistant/search.go - Output messages "search.loading": "正在搜索...", "search.success": "找到 %d 条参考资料",