From e2d9bb5038a23ec9365282dac9ffd90089db4ee3 Mon Sep 17 00:00:00 2001 From: Emanuel Casco Date: Fri, 8 May 2026 21:04:23 +0200 Subject: [PATCH] Fix image input support --- cmd/ocgo/main.go | 269 +++++++++++++++++++++++++++++++++++++++++- cmd/ocgo/main_test.go | 120 ++++++++++++++++++- 2 files changed, 382 insertions(+), 7 deletions(-) diff --git a/cmd/ocgo/main.go b/cmd/ocgo/main.go index 6e9e0d6..57ff458 100644 --- a/cmd/ocgo/main.go +++ b/cmd/ocgo/main.go @@ -89,12 +89,23 @@ type ResponseTool struct { type OAIMessage struct { Role string `json:"role"` - Content string `json:"content,omitempty"` + Content any `json:"content,omitempty"` ToolCalls []OAIToolCall `json:"tool_calls,omitempty"` ToolCallID string `json:"tool_call_id,omitempty"` ReasoningContent string `json:"reasoning_content,omitempty"` } +type OAIContentPart struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ImageURL *OAIImageURL `json:"image_url,omitempty"` +} + +type OAIImageURL struct { + URL string `json:"url"` + Detail string `json:"detail,omitempty"` +} + type OAITool struct { Type string `json:"type"` Function OAIFunction `json:"function"` @@ -171,6 +182,22 @@ func knownModelIDs() []string { return []string{"glm-5.1", "glm-5", "kimi-k2.6", "kimi-k2.5", "mimo-v2.5-pro", "mimo-v2.5", "mimo-v2-pro", "mimo-v2-omni", "minimax-m2.7", "minimax-m2.5", "deepseek-v4-pro", "deepseek-v4-flash", "qwen3.6-plus", "qwen3.5-plus"} } +func modelSupportsImages(model string) bool { + switch model { + case "kimi-k2.6", "kimi-k2.5", "mimo-v2-omni": + return true + default: + return false + } +} + +func modelInputModalities(model string) []string { + if modelSupportsImages(model) { + return []string{"text", "image"} + } + return []string{"text"} +} + func launchCmd() *cobra.Command { var model string var yes bool @@ -334,6 +361,10 @@ func proxyMessages(w http.ResponseWriter, r *http.Request, cfg Config) { return } or := convertRequest(ar) + if err := validateImageSupport(or); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } body, _ := json.Marshal(or) req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, openAIURL, bytes.NewReader(body)) if err != nil { @@ -370,6 +401,11 @@ func proxyChatCompletions(w http.ResponseWriter, r *http.Request, cfg Config) { http.Error(w, "invalid request", http.StatusBadRequest) return } + body, err = prepareChatBody(body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, openAIURL, bytes.NewReader(body)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -399,6 +435,10 @@ func proxyResponses(w http.ResponseWriter, r *http.Request, cfg Config) { return } or := responsesToChat(rr) + if err := validateImageSupport(or); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } body, _ := json.Marshal(or) req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, openAIURL, bytes.NewReader(body)) if err != nil { @@ -433,6 +473,88 @@ func copyHeaders(dst, src http.Header) { } } +func prepareChatBody(body []byte) ([]byte, error) { + var req map[string]any + if json.Unmarshal(body, &req) != nil { + return body, nil + } + model, _ := req["model"].(string) + if !rawChatBodyHasImages(req) { + return body, nil + } + if !modelSupportsImages(model) { + return nil, unsupportedImageModelError(model) + } + changed := stripRawChatImageDetails(req) + if !changed { + return body, nil + } + out, err := json.Marshal(req) + if err != nil { + return body, nil + } + return out, nil +} + +func rawChatBodyHasImages(req map[string]any) bool { + messages, _ := req["messages"].([]any) + for _, item := range messages { + msg, ok := item.(map[string]any) + if !ok { + continue + } + if contentHasImage(msg["content"]) { + return true + } + } + return false +} + +func validateImageSupport(or OAIRequest) error { + if requestHasImages(or) && !modelSupportsImages(or.Model) { + return unsupportedImageModelError(or.Model) + } + return nil +} + +func unsupportedImageModelError(model string) error { + if model == "" { + model = "unknown" + } + return fmt.Errorf("model %s does not support image inputs", model) +} + +func stripRawChatImageDetails(req map[string]any) bool { + changed := false + messages, _ := req["messages"].([]any) + for _, item := range messages { + msg, ok := item.(map[string]any) + if !ok { + continue + } + parts, _ := msg["content"].([]any) + for _, part := range parts { + p, ok := part.(map[string]any) + if !ok { + continue + } + if _, ok := p["detail"]; ok { + delete(p, "detail") + changed = true + } + image, ok := p["image_url"].(map[string]any) + if !ok { + continue + } + if _, ok := image["detail"]; ok { + delete(image, "detail") + changed = true + } + } + } + return changed +} + func convertRequest(ar AnthropicRequest) OAIRequest { model := ar.Model if model == "" || strings.HasPrefix(model, "claude-") { @@ -469,6 +591,37 @@ func responsesToChat(rr ResponsesRequest) OAIRequest { return out } +func requestHasImages(or OAIRequest) bool { + for _, m := range or.Messages { + if contentHasImage(m.Content) { + return true + } + } + return false +} + +func contentHasImage(content any) bool { + switch v := content.(type) { + case []OAIContentPart: + for _, part := range v { + if part.Type == "image_url" && part.ImageURL != nil && part.ImageURL.URL != "" { + return true + } + } + case []any: + for _, item := range v { + m, ok := item.(map[string]any) + if !ok { + continue + } + if typ, _ := m["type"].(string); typ == "image_url" || typ == "input_image" { + return true + } + } + } + return false +} + func responsesInputToMessages(raw json.RawMessage) []OAIMessage { if len(raw) == 0 { return nil @@ -495,7 +648,7 @@ func responsesInputToMessages(raw json.RawMessage) []OAIMessage { if role == "" { role = "user" } - out = append(out, OAIMessage{Role: role, Content: responsesContentText(item["content"])}) + out = append(out, OAIMessage{Role: role, Content: responsesContent(item["content"])}) case "function_call": var id, callID, name, args string _ = json.Unmarshal(item["id"], &id) @@ -557,6 +710,66 @@ func cacheReasoningContent(calls []OAIToolCall, reasoning string) { } } +func responsesContent(raw json.RawMessage) any { + if len(raw) == 0 { + return "" + } + var s string + if json.Unmarshal(raw, &s) == nil { + return s + } + var parts []map[string]json.RawMessage + if json.Unmarshal(raw, &parts) != nil { + return string(raw) + } + var text strings.Builder + var out []OAIContentPart + hasImage := false + for _, p := range parts { + var typ string + _ = json.Unmarshal(p["type"], &typ) + switch typ { + case "input_text", "output_text", "text": + for _, key := range []string{"text", "output_text"} { + var v string + if json.Unmarshal(p[key], &v) == nil { + text.WriteString(v) + out = append(out, OAIContentPart{Type: "text", Text: v}) + break + } + } + case "input_image", "image_url": + if image := responsesImageURL(p); image != nil { + hasImage = true + out = append(out, OAIContentPart{Type: "image_url", ImageURL: image}) + } + } + } + if hasImage { + return out + } + return text.String() +} + +func responsesImageURL(p map[string]json.RawMessage) *OAIImageURL { + var url string + if json.Unmarshal(p["image_url"], &url) != nil { + var obj struct { + URL string `json:"url"` + } + if json.Unmarshal(p["image_url"], &obj) == nil { + url = obj.URL + } + } + if url == "" { + _ = json.Unmarshal(p["url"], &url) + } + if url == "" { + return nil + } + return &OAIImageURL{URL: url} +} + func responsesContentText(raw json.RawMessage) string { if len(raw) == 0 { return "" @@ -591,6 +804,8 @@ func contentToOpenAI(m AMessage) []OAIMessage { return []OAIMessage{{Role: m.Role, Content: string(m.Content)}} } var text strings.Builder + var parts []OAIContentPart + hasImage := false var calls []OAIToolCall var toolMsgs []OAIMessage for _, b := range blocks { @@ -601,6 +816,14 @@ func contentToOpenAI(m AMessage) []OAIMessage { var v string _ = json.Unmarshal(b["text"], &v) text.WriteString(v) + if v != "" { + parts = append(parts, OAIContentPart{Type: "text", Text: v}) + } + case "image": + if image := anthropicImageURL(b); image != nil { + hasImage = true + parts = append(parts, OAIContentPart{Type: "image_url", ImageURL: image}) + } case "tool_use": var id, name string _ = json.Unmarshal(b["id"], &id) @@ -618,7 +841,7 @@ func contentToOpenAI(m AMessage) []OAIMessage { } if len(calls) > 0 { msg := assistantToolCallsMessage(calls) - msg.Content = text.String() + msg.Content = openAIContentValue(text.String(), parts, hasImage) return []OAIMessage{msg} } if len(toolMsgs) > 0 { @@ -631,7 +854,43 @@ func contentToOpenAI(m AMessage) []OAIMessage { } return out } - return []OAIMessage{{Role: m.Role, Content: text.String()}} + return []OAIMessage{{Role: m.Role, Content: openAIContentValue(text.String(), parts, hasImage)}} +} + +func openAIContentValue(text string, parts []OAIContentPart, hasImage bool) any { + if hasImage { + return parts + } + return text +} + +func anthropicImageURL(b map[string]json.RawMessage) *OAIImageURL { + var source struct { + Type string `json:"type"` + MediaType string `json:"media_type"` + Data string `json:"data"` + URL string `json:"url"` + } + if json.Unmarshal(b["source"], &source) != nil { + return nil + } + if source.URL != "" || source.Type == "url" { + if source.URL == "" { + return nil + } + return &OAIImageURL{URL: source.URL} + } + if source.Data == "" { + return nil + } + if strings.HasPrefix(source.Data, "data:") { + return &OAIImageURL{URL: source.Data} + } + mediaType := source.MediaType + if mediaType == "" { + mediaType = "image/png" + } + return &OAIImageURL{URL: "data:" + mediaType + ";base64," + source.Data} } func systemText(raw json.RawMessage) string { @@ -1150,7 +1409,7 @@ func writeCodexModelCatalog(path string) error { "auto_compact_token_limit": nil, "effective_context_window_percent": 95, "experimental_supported_tools": []any{}, - "input_modalities": []string{"text"}, + "input_modalities": modelInputModalities(id), "supports_search_tool": false, }) } diff --git a/cmd/ocgo/main_test.go b/cmd/ocgo/main_test.go index 4183b87..4621517 100644 --- a/cmd/ocgo/main_test.go +++ b/cmd/ocgo/main_test.go @@ -69,13 +69,34 @@ func TestWriteCodexModelCatalog(t *testing.T) { t.Fatal(err) } content := string(b) - for _, want := range []string{`"models"`, `"slug": "deepseek-v4-pro"`, `"context_window": 128000`, `"truncation_policy"`} { + for _, want := range []string{`"models"`, `"slug": "deepseek-v4-pro"`, `"context_window": 128000`, `"truncation_policy"`, `"supports_image_detail_original": false`, `"image"`} { if !strings.Contains(content, want) { t.Fatalf("missing %q in:\n%s", want, content) } } } +func TestCodexModelCatalogAllowsImagesForKnownVisionModels(t *testing.T) { + if !modelSupportsImages("kimi-k2.6") { + t.Fatal("kimi-k2.6 should support image inputs") + } + if modelSupportsImages("deepseek-v4-pro") { + t.Fatal("deepseek-v4-pro should not support image inputs") + } + for _, tc := range []struct { + model string + want []string + }{ + {model: "kimi-k2.6", want: []string{"text", "image"}}, + {model: "deepseek-v4-pro", want: []string{"text"}}, + } { + got := modelInputModalities(tc.model) + if strings.Join(got, ",") != strings.Join(tc.want, ",") { + t.Fatalf("%s modalities = %+v, want %+v", tc.model, got, tc.want) + } + } +} + func TestCompareVersions(t *testing.T) { if compareVersions("0.80.9", "0.81.0") >= 0 { t.Fatal("0.80.9 should be older") @@ -141,11 +162,106 @@ func TestAnthropicToolResultPreservesFollowingUserText(t *testing.T) { if messages[0].Role != "tool" || messages[0].ToolCallID != "call_123" || messages[0].Content != "09:33:16" { t.Fatalf("bad tool result conversion: %+v", messages[0]) } - if messages[1].Role != "user" || !strings.Contains(messages[1].Content, "figma.example") { + if messages[1].Role != "user" || !strings.Contains(contentString(messages[1].Content), "figma.example") { t.Fatalf("following user text was not preserved: %+v", messages[1]) } } +func TestResponsesInputPreservesImages(t *testing.T) { + messages := responsesInputToMessages([]byte(`[{"type":"message","role":"user","content":[{"type":"input_text","text":"describe this"},{"type":"input_image","image_url":"data:image/png;base64,abc","detail":"high"}]}]`)) + if len(messages) != 1 { + t.Fatalf("got %d messages", len(messages)) + } + parts, ok := messages[0].Content.([]OAIContentPart) + if !ok { + t.Fatalf("content should be multimodal parts: %+v", messages[0].Content) + } + if len(parts) != 2 || parts[0].Type != "text" || parts[0].Text != "describe this" { + t.Fatalf("bad text part: %+v", parts) + } + if parts[1].Type != "image_url" || parts[1].ImageURL == nil || parts[1].ImageURL.URL != "data:image/png;base64,abc" || parts[1].ImageURL.Detail != "" { + t.Fatalf("bad image part: %+v", parts[1]) + } +} + +func TestResponsesImageKeepsKimiModel(t *testing.T) { + req := ResponsesRequest{Model: "kimi-k2.6", Input: []byte(`[{"type":"message","role":"user","content":[{"type":"input_text","text":"describe this"},{"type":"input_image","image_url":"data:image/png;base64,abc"}]}]`)} + out := responsesToChat(req) + if out.Model != "kimi-k2.6" { + t.Fatalf("image request should keep Kimi model, got %q", out.Model) + } + if err := validateImageSupport(out); err != nil { + t.Fatalf("Kimi image request should validate: %v", err) + } +} + +func TestResponsesImageRejectsUnsupportedModel(t *testing.T) { + req := ResponsesRequest{Model: "deepseek-v4-pro", Input: []byte(`[{"type":"message","role":"user","content":[{"type":"input_text","text":"describe this"},{"type":"input_image","image_url":"data:image/png;base64,abc"}]}]`)} + out := responsesToChat(req) + if err := validateImageSupport(out); err == nil || !strings.Contains(err.Error(), "deepseek-v4-pro") { + t.Fatalf("DeepSeek image request should be rejected, got %v", err) + } +} + +func TestRawChatImageKeepsKimiAndStripsDetail(t *testing.T) { + body, err := prepareChatBody([]byte(`{"model":"kimi-k2.6","messages":[{"role":"user","content":[{"type":"text","text":"describe this"},{"type":"image_url","image_url":{"url":"data:image/png;base64,abc","detail":"high"}}]}]}`)) + if err != nil { + t.Fatalf("Kimi image request should validate: %v", err) + } + if !strings.Contains(string(body), `"model":"kimi-k2.6"`) { + t.Fatalf("image chat body should keep Kimi model: %s", string(body)) + } + if strings.Contains(string(body), `"detail"`) { + t.Fatalf("image detail should be stripped for compatibility: %s", string(body)) + } +} + +func TestRawChatImageRejectsUnsupportedModel(t *testing.T) { + _, err := prepareChatBody([]byte(`{"model":"deepseek-v4-pro","messages":[{"role":"user","content":[{"type":"text","text":"describe this"},{"type":"image_url","image_url":{"url":"data:image/png;base64,abc"}}]}]}`)) + if err == nil || !strings.Contains(err.Error(), "deepseek-v4-pro") { + t.Fatalf("DeepSeek image request should be rejected, got %v", err) + } +} + +func TestAnthropicContentPreservesImages(t *testing.T) { + messages := contentToOpenAI(AMessage{Role: "user", Content: []byte(`[{"type":"text","text":"what is this?"},{"type":"image","source":{"type":"base64","media_type":"image/jpeg","data":"abc"}}]`)}) + if len(messages) != 1 { + t.Fatalf("got %d messages", len(messages)) + } + parts, ok := messages[0].Content.([]OAIContentPart) + if !ok { + t.Fatalf("content should be multimodal parts: %+v", messages[0].Content) + } + if len(parts) != 2 || parts[0].Text != "what is this?" { + t.Fatalf("bad text part: %+v", parts) + } + if parts[1].ImageURL == nil || parts[1].ImageURL.URL != "data:image/jpeg;base64,abc" { + t.Fatalf("bad image part: %+v", parts[1]) + } +} + +func TestAnthropicImageKeepsKimiModel(t *testing.T) { + out := convertRequest(AnthropicRequest{Model: "kimi-k2.6", Messages: []AMessage{{Role: "user", Content: []byte(`[{"type":"text","text":"what is this?"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"abc"}}]`)}}}) + if out.Model != "kimi-k2.6" { + t.Fatalf("image request should keep Kimi model, got %q", out.Model) + } + if err := validateImageSupport(out); err != nil { + t.Fatalf("Kimi image request should validate: %v", err) + } +} + +func TestAnthropicImageRejectsUnsupportedModel(t *testing.T) { + out := convertRequest(AnthropicRequest{Model: "deepseek-v4-pro", Messages: []AMessage{{Role: "user", Content: []byte(`[{"type":"text","text":"what is this?"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"abc"}}]`)}}}) + if err := validateImageSupport(out); err == nil || !strings.Contains(err.Error(), "deepseek-v4-pro") { + t.Fatalf("DeepSeek image request should be rejected, got %v", err) + } +} + +func contentString(v any) string { + s, _ := v.(string) + return s +} + func TestStreamAnthropicForwardsToolCalls(t *testing.T) { reasoningContentCache.Lock() reasoningContentCache.byCallID = map[string]string{}