feat(agent): support native audio input for multimodal LLMs

# Conflicts:
#	pkg/providers/protocoltypes/types.go
This commit is contained in:
Daniel 2026-04-23 18:41:22 +03:00
parent 0c0a582559
commit 58ba107af8
2 changed files with 10 additions and 1 deletions

View file

@ -75,6 +75,14 @@ func resolveMediaRefs(messages []providers.Message, store media.MediaStore, maxS
continue continue
} }
if strings.HasPrefix(mime, "audio/") {
dataURL := encodeImageToDataURL(localPath, mime, info, maxSize)
if dataURL != "" {
result[i].Audio = append(result[i].Audio, dataURL)
}
continue
}
pathTags = append(pathTags, buildPathTag(mime, localPath)) pathTags = append(pathTags, buildPathTag(mime, localPath))
} }

View file

@ -76,6 +76,7 @@ type Message struct {
Content string `json:"content"` Content string `json:"content"`
Media []string `json:"media,omitempty"` Media []string `json:"media,omitempty"`
Attachments []Attachment `json:"attachments,omitempty"` Attachments []Attachment `json:"attachments,omitempty"`
Audio []string `json:"audio,omitempty"`
ReasoningContent string `json:"reasoning_content,omitempty"` ReasoningContent string `json:"reasoning_content,omitempty"`
SystemParts []ContentBlock `json:"system_parts,omitempty"` // structured system blocks for cache-aware adapters SystemParts []ContentBlock `json:"system_parts,omitempty"` // structured system blocks for cache-aware adapters
ToolCalls []ToolCall `json:"tool_calls,omitempty"` ToolCalls []ToolCall `json:"tool_calls,omitempty"`
@ -91,4 +92,4 @@ type ToolFunctionDefinition struct {
Name string `json:"name"` Name string `json:"name"`
Description string `json:"description"` Description string `json:"description"`
Parameters map[string]any `json:"parameters"` Parameters map[string]any `json:"parameters"`
} }