124bd38f7a
- Added global uses configuration to the assistant, allowing for centralized management of vision, audio, search, and fetch settings. - Updated the Assistant struct and related methods to support the new Uses configuration, improving flexibility in assistant operations. - Refactored the Stream method to utilize the new CompletionResponse type, enhancing response handling. - Introduced new methods for building requests and managing capabilities, streamlining the assistant's interaction with various connectors.
47 lines
1.4 KiB
Go
47 lines
1.4 KiB
Go
package vision
|
|
|
|
import (
|
|
"github.com/yaoapp/yao/agent/context"
|
|
)
|
|
|
|
// PreprocessVisionMessages preprocess messages to handle vision content
|
|
// Removes or converts vision content for models that don't support it
|
|
func PreprocessVisionMessages(messages []context.Message, supportsVision bool) []context.Message {
|
|
// TODO: Implement vision message preprocessing
|
|
// If supportsVision is false:
|
|
// - Remove image_url content parts
|
|
// - Convert to text-only messages
|
|
// - Optionally add image descriptions from vision API
|
|
// If supportsVision is true:
|
|
// - Validate image URLs
|
|
// - Ensure proper format
|
|
return messages
|
|
}
|
|
|
|
// ConvertImageToText convert image content to text description
|
|
// Used when model doesn't support vision
|
|
func ConvertImageToText(imageURL string) (string, error) {
|
|
// TODO: Implement image to text conversion
|
|
// - Call vision API (if configured)
|
|
// - Generate description
|
|
// - Return as text content
|
|
return "", nil
|
|
}
|
|
|
|
// ValidateImageURL validate image URL format
|
|
func ValidateImageURL(imageURL string) error {
|
|
// TODO: Implement image URL validation
|
|
// - Check URL format
|
|
// - Validate image type
|
|
// - Check accessibility
|
|
return nil
|
|
}
|
|
|
|
// ExtractImagesFromMessages extract all image URLs from messages
|
|
func ExtractImagesFromMessages(messages []context.Message) []string {
|
|
// TODO: Implement image extraction
|
|
// - Iterate through messages
|
|
// - Find ContentPart with type="image_url"
|
|
// - Collect all image URLs
|
|
return nil
|
|
}
|