feat: add Media field to Message struct and implement serializeMessages for vision API support
- Add Media []string field to Message struct for image/media URLs - Implement serializeMessages() to format messages with image_url content parts - Enables OpenAI-compatible vision APIs to receive image attachments
This commit is contained in:
parent
26d1b8e374
commit
3d54a77c40
2 changed files with 43 additions and 1 deletions
|
|
@ -116,7 +116,7 @@ func (p *Provider) Chat(
|
||||||
|
|
||||||
requestBody := map[string]any{
|
requestBody := map[string]any{
|
||||||
"model": model,
|
"model": model,
|
||||||
"messages": stripSystemParts(messages),
|
"messages": serializeMessages(messages),
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(tools) > 0 {
|
if len(tools) > 0 {
|
||||||
|
|
@ -195,6 +195,47 @@ func (p *Provider) Chat(
|
||||||
return parseResponse(body)
|
return parseResponse(body)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func serializeMessages(messages []Message) []map[string]interface{} {
|
||||||
|
result := make([]map[string]interface{}, 0, len(messages))
|
||||||
|
for _, m := range messages {
|
||||||
|
if len(m.Media) == 0 {
|
||||||
|
msg := map[string]interface{}{
|
||||||
|
"role": m.Role,
|
||||||
|
"content": m.Content,
|
||||||
|
}
|
||||||
|
if m.ToolCallID != "" {
|
||||||
|
msg["tool_call_id"] = m.ToolCallID
|
||||||
|
}
|
||||||
|
if len(m.ToolCalls) > 0 {
|
||||||
|
msg["tool_calls"] = m.ToolCalls
|
||||||
|
}
|
||||||
|
result = append(result, msg)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
parts := make([]map[string]interface{}, 0, 1+len(m.Media))
|
||||||
|
if m.Content != "" {
|
||||||
|
parts = append(parts, map[string]interface{}{
|
||||||
|
"type": "text",
|
||||||
|
"text": m.Content,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for _, mediaURL := range m.Media {
|
||||||
|
parts = append(parts, map[string]interface{}{
|
||||||
|
"type": "image_url",
|
||||||
|
"image_url": map[string]interface{}{
|
||||||
|
"url": mediaURL,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
result = append(result, map[string]interface{}{
|
||||||
|
"role": m.Role,
|
||||||
|
"content": parts,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
func parseResponse(body []byte) (*LLMResponse, error) {
|
func parseResponse(body []byte) (*LLMResponse, error) {
|
||||||
var apiResponse struct {
|
var apiResponse struct {
|
||||||
Choices []struct {
|
Choices []struct {
|
||||||
|
|
|
||||||
|
|
@ -65,6 +65,7 @@ type ContentBlock struct {
|
||||||
type Message struct {
|
type Message struct {
|
||||||
Role string `json:"role"`
|
Role string `json:"role"`
|
||||||
Content string `json:"content"`
|
Content string `json:"content"`
|
||||||
|
Media []string `json:"media,omitempty"` // URLs of images or other media attachments
|
||||||
ReasoningContent string `json:"reasoning_content,omitempty"`
|
ReasoningContent string `json:"reasoning_content,omitempty"`
|
||||||
SystemParts []ContentBlock `json:"system_parts,omitempty"` // structured system blocks for cache-aware adapters
|
SystemParts []ContentBlock `json:"system_parts,omitempty"` // structured system blocks for cache-aware adapters
|
||||||
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue