feat(anthropic): add vision/image support for native Anthropic provider
The merged #1020 added vision support to the openai_compat provider but left the native Anthropic provider without image handling. This patch closes the gap: when a Message carries Media (base64 data URLs), the Anthropic adapter now builds NewImageBlockBase64 content blocks using the official SDK, giving users of `anthropic/` prefixed models the same vision capability. Changes: - buildParams: handle msg.Media in user messages, convert data URLs to anthropic.NewImageBlockBase64 content blocks - parseDataURL: extract media type and base64 data from data URLs - Tests for parseDataURL (8 cases) and media message building
This commit is contained in:
parent
df1b53fdf9
commit
019a5e6c80
2 changed files with 98 additions and 0 deletions
|
|
@ -132,6 +132,20 @@ func buildParams(
|
|||
anthropicMessages = append(anthropicMessages,
|
||||
anthropic.NewUserMessage(anthropic.NewToolResultBlock(msg.ToolCallID, msg.Content, false)),
|
||||
)
|
||||
} else if len(msg.Media) > 0 {
|
||||
// Multimodal message with text + images
|
||||
var blocks []anthropic.ContentBlockParamUnion
|
||||
if msg.Content != "" {
|
||||
blocks = append(blocks, anthropic.NewTextBlock(msg.Content))
|
||||
}
|
||||
for _, mediaURL := range msg.Media {
|
||||
if mediaType, b64Data, ok := parseDataURL(mediaURL); ok {
|
||||
blocks = append(blocks, anthropic.NewImageBlockBase64(mediaType, b64Data))
|
||||
}
|
||||
}
|
||||
if len(blocks) > 0 {
|
||||
anthropicMessages = append(anthropicMessages, anthropic.NewUserMessage(blocks...))
|
||||
}
|
||||
} else {
|
||||
anthropicMessages = append(anthropicMessages,
|
||||
anthropic.NewUserMessage(anthropic.NewTextBlock(msg.Content)),
|
||||
|
|
@ -257,6 +271,23 @@ func parseResponse(resp *anthropic.Message) *LLMResponse {
|
|||
}
|
||||
}
|
||||
|
||||
// parseDataURL extracts the media type and base64 data from a data URL.
|
||||
// Expected format: "data:<mediaType>;base64,<data>"
|
||||
func parseDataURL(url string) (mediaType string, b64Data string, ok bool) {
|
||||
// data:image/jpeg;base64,/9j/4AAQ...
|
||||
if !strings.HasPrefix(url, "data:") {
|
||||
return "", "", false
|
||||
}
|
||||
rest := url[len("data:"):]
|
||||
idx := strings.Index(rest, ";base64,")
|
||||
if idx < 0 {
|
||||
return "", "", false
|
||||
}
|
||||
mediaType = rest[:idx]
|
||||
b64Data = rest[idx+len(";base64,"):]
|
||||
return mediaType, b64Data, mediaType != "" && b64Data != ""
|
||||
}
|
||||
|
||||
func normalizeBaseURL(apiBase string) string {
|
||||
base := strings.TrimSpace(apiBase)
|
||||
if base == "" {
|
||||
|
|
|
|||
|
|
@ -262,6 +262,73 @@ func TestProvider_ChatUsesTokenSource(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestParseDataURL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
url string
|
||||
wantType string
|
||||
wantData string
|
||||
wantOK bool
|
||||
}{
|
||||
{"valid jpeg", "data:image/jpeg;base64,/9j/4AAQ", "image/jpeg", "/9j/4AAQ", true},
|
||||
{"valid png", "data:image/png;base64,iVBOR", "image/png", "iVBOR", true},
|
||||
{"valid gif", "data:image/gif;base64,R0lG", "image/gif", "R0lG", true},
|
||||
{"valid webp", "data:image/webp;base64,UklG", "image/webp", "UklG", true},
|
||||
{"no data prefix", "https://example.com/image.jpg", "", "", false},
|
||||
{"no base64 marker", "data:image/jpeg,rawdata", "", "", false},
|
||||
{"empty media type", "data:;base64,abc", "", "", false},
|
||||
{"empty data", "data:image/jpeg;base64,", "", "", false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
gotType, gotData, gotOK := parseDataURL(tt.url)
|
||||
if gotOK != tt.wantOK {
|
||||
t.Errorf("ok = %v, want %v", gotOK, tt.wantOK)
|
||||
return
|
||||
}
|
||||
if gotOK {
|
||||
if gotType != tt.wantType {
|
||||
t.Errorf("type = %q, want %q", gotType, tt.wantType)
|
||||
}
|
||||
if gotData != tt.wantData {
|
||||
t.Errorf("data = %q, want %q", gotData, tt.wantData)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildParams_MediaMessage(t *testing.T) {
|
||||
messages := []Message{
|
||||
{
|
||||
Role: "user",
|
||||
Content: "What's in this image?",
|
||||
Media: []string{"data:image/jpeg;base64,/9j/4AAQ"},
|
||||
},
|
||||
}
|
||||
params, err := buildParams(messages, nil, "claude-sonnet-4.6", map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatalf("buildParams() error: %v", err)
|
||||
}
|
||||
if len(params.Messages) != 1 {
|
||||
t.Fatalf("len(Messages) = %d, want 1", len(params.Messages))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildParams_MediaMessageTextOnly(t *testing.T) {
|
||||
// Message without media should still work as text-only
|
||||
messages := []Message{
|
||||
{Role: "user", Content: "Hello"},
|
||||
}
|
||||
params, err := buildParams(messages, nil, "claude-sonnet-4.6", map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatalf("buildParams() error: %v", err)
|
||||
}
|
||||
if len(params.Messages) != 1 {
|
||||
t.Fatalf("len(Messages) = %d, want 1", len(params.Messages))
|
||||
}
|
||||
}
|
||||
|
||||
func createAnthropicTestClient(baseURL, token string) *anthropic.Client {
|
||||
c := anthropic.NewClient(
|
||||
anthropicoption.WithAuthToken(token),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue