feat(anthropic): add vision/image support for native Anthropic provider
The merged #1020 added vision support to the openai_compat provider but left the native Anthropic provider without image handling. This patch closes the gap: when a Message carries Media (base64 data URLs), the Anthropic adapter now builds NewImageBlockBase64 content blocks using the official SDK, giving users of `anthropic/` prefixed models the same vision capability. Changes: - buildParams: handle msg.Media in user messages, convert data URLs to anthropic.NewImageBlockBase64 content blocks - parseDataURL: extract media type and base64 data from data URLs - Tests for parseDataURL (8 cases) and media message building
This commit is contained in:
parent
df1b53fdf9
commit
019a5e6c80
2 changed files with 98 additions and 0 deletions
|
|
@ -132,6 +132,20 @@ func buildParams(
|
||||||
anthropicMessages = append(anthropicMessages,
|
anthropicMessages = append(anthropicMessages,
|
||||||
anthropic.NewUserMessage(anthropic.NewToolResultBlock(msg.ToolCallID, msg.Content, false)),
|
anthropic.NewUserMessage(anthropic.NewToolResultBlock(msg.ToolCallID, msg.Content, false)),
|
||||||
)
|
)
|
||||||
|
} else if len(msg.Media) > 0 {
|
||||||
|
// Multimodal message with text + images
|
||||||
|
var blocks []anthropic.ContentBlockParamUnion
|
||||||
|
if msg.Content != "" {
|
||||||
|
blocks = append(blocks, anthropic.NewTextBlock(msg.Content))
|
||||||
|
}
|
||||||
|
for _, mediaURL := range msg.Media {
|
||||||
|
if mediaType, b64Data, ok := parseDataURL(mediaURL); ok {
|
||||||
|
blocks = append(blocks, anthropic.NewImageBlockBase64(mediaType, b64Data))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(blocks) > 0 {
|
||||||
|
anthropicMessages = append(anthropicMessages, anthropic.NewUserMessage(blocks...))
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
anthropicMessages = append(anthropicMessages,
|
anthropicMessages = append(anthropicMessages,
|
||||||
anthropic.NewUserMessage(anthropic.NewTextBlock(msg.Content)),
|
anthropic.NewUserMessage(anthropic.NewTextBlock(msg.Content)),
|
||||||
|
|
@ -257,6 +271,23 @@ func parseResponse(resp *anthropic.Message) *LLMResponse {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseDataURL extracts the media type and base64 data from a data URL.
|
||||||
|
// Expected format: "data:<mediaType>;base64,<data>"
|
||||||
|
func parseDataURL(url string) (mediaType string, b64Data string, ok bool) {
|
||||||
|
// data:image/jpeg;base64,/9j/4AAQ...
|
||||||
|
if !strings.HasPrefix(url, "data:") {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
rest := url[len("data:"):]
|
||||||
|
idx := strings.Index(rest, ";base64,")
|
||||||
|
if idx < 0 {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
mediaType = rest[:idx]
|
||||||
|
b64Data = rest[idx+len(";base64,"):]
|
||||||
|
return mediaType, b64Data, mediaType != "" && b64Data != ""
|
||||||
|
}
|
||||||
|
|
||||||
func normalizeBaseURL(apiBase string) string {
|
func normalizeBaseURL(apiBase string) string {
|
||||||
base := strings.TrimSpace(apiBase)
|
base := strings.TrimSpace(apiBase)
|
||||||
if base == "" {
|
if base == "" {
|
||||||
|
|
|
||||||
|
|
@ -262,6 +262,73 @@ func TestProvider_ChatUsesTokenSource(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseDataURL(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
url string
|
||||||
|
wantType string
|
||||||
|
wantData string
|
||||||
|
wantOK bool
|
||||||
|
}{
|
||||||
|
{"valid jpeg", "data:image/jpeg;base64,/9j/4AAQ", "image/jpeg", "/9j/4AAQ", true},
|
||||||
|
{"valid png", "data:image/png;base64,iVBOR", "image/png", "iVBOR", true},
|
||||||
|
{"valid gif", "data:image/gif;base64,R0lG", "image/gif", "R0lG", true},
|
||||||
|
{"valid webp", "data:image/webp;base64,UklG", "image/webp", "UklG", true},
|
||||||
|
{"no data prefix", "https://example.com/image.jpg", "", "", false},
|
||||||
|
{"no base64 marker", "data:image/jpeg,rawdata", "", "", false},
|
||||||
|
{"empty media type", "data:;base64,abc", "", "", false},
|
||||||
|
{"empty data", "data:image/jpeg;base64,", "", "", false},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
gotType, gotData, gotOK := parseDataURL(tt.url)
|
||||||
|
if gotOK != tt.wantOK {
|
||||||
|
t.Errorf("ok = %v, want %v", gotOK, tt.wantOK)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if gotOK {
|
||||||
|
if gotType != tt.wantType {
|
||||||
|
t.Errorf("type = %q, want %q", gotType, tt.wantType)
|
||||||
|
}
|
||||||
|
if gotData != tt.wantData {
|
||||||
|
t.Errorf("data = %q, want %q", gotData, tt.wantData)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildParams_MediaMessage(t *testing.T) {
|
||||||
|
messages := []Message{
|
||||||
|
{
|
||||||
|
Role: "user",
|
||||||
|
Content: "What's in this image?",
|
||||||
|
Media: []string{"data:image/jpeg;base64,/9j/4AAQ"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
params, err := buildParams(messages, nil, "claude-sonnet-4.6", map[string]any{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("buildParams() error: %v", err)
|
||||||
|
}
|
||||||
|
if len(params.Messages) != 1 {
|
||||||
|
t.Fatalf("len(Messages) = %d, want 1", len(params.Messages))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildParams_MediaMessageTextOnly(t *testing.T) {
|
||||||
|
// Message without media should still work as text-only
|
||||||
|
messages := []Message{
|
||||||
|
{Role: "user", Content: "Hello"},
|
||||||
|
}
|
||||||
|
params, err := buildParams(messages, nil, "claude-sonnet-4.6", map[string]any{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("buildParams() error: %v", err)
|
||||||
|
}
|
||||||
|
if len(params.Messages) != 1 {
|
||||||
|
t.Fatalf("len(Messages) = %d, want 1", len(params.Messages))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func createAnthropicTestClient(baseURL, token string) *anthropic.Client {
|
func createAnthropicTestClient(baseURL, token string) *anthropic.Client {
|
||||||
c := anthropic.NewClient(
|
c := anthropic.NewClient(
|
||||||
anthropicoption.WithAuthToken(token),
|
anthropicoption.WithAuthToken(token),
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue