diff --git a/pkg/agent/agent_media.go b/pkg/agent/agent_media.go index a773d2ebb..84f386f2d 100644 --- a/pkg/agent/agent_media.go +++ b/pkg/agent/agent_media.go @@ -66,6 +66,7 @@ func resolveMediaRefs(messages []providers.Message, store media.MediaStore, maxS } mime := detectMIME(localPath, meta) + pathTags = append(pathTags, buildPathTag(mime, localPath)) if strings.HasPrefix(mime, "image/") { dataURL := encodeImageToDataURL(localPath, mime, info, maxSize) @@ -74,8 +75,6 @@ func resolveMediaRefs(messages []providers.Message, store media.MediaStore, maxS } continue } - - pathTags = append(pathTags, buildPathTag(mime, localPath)) } result[i].Media = resolved @@ -182,6 +181,8 @@ func encodeImageToDataURL(localPath, mime string, info os.FileInfo, maxSize int) // Tag type is derived from MIME: [audio:/path], [video:/path], or [file:/path]. func buildPathTag(mime, localPath string) string { switch { + case strings.HasPrefix(mime, "image/"): + return "[image:" + localPath + "]" case strings.HasPrefix(mime, "audio/"): return "[audio:" + localPath + "]" case strings.HasPrefix(mime, "video/"): @@ -197,6 +198,8 @@ func injectPathTags(content string, tags []string) string { for _, tag := range tags { var generic string switch { + case strings.HasPrefix(tag, "[image:"): + generic = "[image: photo]" case strings.HasPrefix(tag, "[audio:"): generic = "[audio]" case strings.HasPrefix(tag, "[video:"): diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go index 4047ab74d..14136aa38 100644 --- a/pkg/agent/agent_test.go +++ b/pkg/agent/agent_test.go @@ -1781,17 +1781,22 @@ func (m *artifactThenSendProvider) Chat( if messages[i].Role != "tool" { continue } - start := strings.Index(messages[i].Content, "[file:") - if start < 0 { - continue + for _, prefix := range []string{"[image:", "[file:", "[audio:", "[video:"} { + start := strings.Index(messages[i].Content, prefix) + if start < 0 { + continue + } + rest := messages[i].Content[start+len(prefix):] + end := strings.Index(rest, "]") + if end < 0 { + continue + } + artifactPath = rest[:end] + break } - rest := messages[i].Content[start+len("[file:"):] - end := strings.Index(rest, "]") - if end < 0 { - continue + if artifactPath != "" { + break } - artifactPath = rest[:end] - break } if artifactPath == "" { return nil, fmt.Errorf("provider did not receive artifact path in tool result") @@ -4656,7 +4661,7 @@ func TestRun_PicoToolFeedbackSuppressesDuplicateInterimAssistantContent(t *testi } } -func TestResolveMediaRefs_ResolvesToBase64(t *testing.T) { +func TestResolveMediaRefs_ImageBase64AndPathTag(t *testing.T) { store := media.NewFileMediaStore() dir := t.TempDir() @@ -4690,9 +4695,14 @@ func TestResolveMediaRefs_ResolvesToBase64(t *testing.T) { if !strings.HasPrefix(result[0].Media[0], "data:image/png;base64,") { t.Fatalf("expected data:image/png;base64, prefix, got %q", result[0].Media[0][:40]) } + localPath, _, _ := store.ResolveWithMeta(ref) + expectedContent := "describe this [image:" + localPath + "]" + if result[0].Content != expectedContent { + t.Fatalf("expected content %q, got %q", expectedContent, result[0].Content) + } } -func TestResolveMediaRefs_SkipsOversizedFile(t *testing.T) { +func TestResolveMediaRefs_OversizedImageSkipsBase64KeepsPathTag(t *testing.T) { store := media.NewFileMediaStore() dir := t.TempDir() @@ -4714,6 +4724,11 @@ func TestResolveMediaRefs_SkipsOversizedFile(t *testing.T) { if len(result[0].Media) != 0 { t.Fatalf("expected 0 media (oversized), got %d", len(result[0].Media)) } + localPath, _, _ := store.ResolveWithMeta(ref) + expected := "hi [image:" + localPath + "]" + if result[0].Content != expected { + t.Fatalf("expected content %q, got %q", expected, result[0].Content) + } } func TestResolveMediaRefs_UnknownTypeInjectsPath(t *testing.T) { @@ -4797,6 +4812,11 @@ func TestResolveMediaRefs_UsesMetaContentType(t *testing.T) { if !strings.HasPrefix(result[0].Media[0], "data:image/jpeg;base64,") { t.Fatalf("expected jpeg prefix, got %q", result[0].Media[0][:30]) } + localPath, _, _ := store.ResolveWithMeta(ref) + expectedContent := "hi [image:" + localPath + "]" + if result[0].Content != expectedContent { + t.Fatalf("expected content %q, got %q", expectedContent, result[0].Content) + } } func TestResolveMediaRefs_PDFInjectsFilePath(t *testing.T) { @@ -4929,12 +4949,14 @@ func TestResolveMediaRefs_MixedImageAndFile(t *testing.T) { result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize) if len(result[0].Media) != 1 { - t.Fatalf("expected 1 media (image only), got %d", len(result[0].Media)) + t.Fatalf("expected 1 media (image base64 only), got %d", len(result[0].Media)) } if !strings.HasPrefix(result[0].Media[0], "data:image/png;base64,") { t.Fatal("expected image to be base64 encoded") } - expectedContent := "check these [file:" + pdfPath + "]" + imgLocalPath, _, _ := store.ResolveWithMeta(imgRef) + pdfLocalPath, _, _ := store.ResolveWithMeta(fileRef) + expectedContent := "check these [file:" + pdfLocalPath + "] [image:" + imgLocalPath + "]" if result[0].Content != expectedContent { t.Fatalf("expected content %q, got %q", expectedContent, result[0].Content) } diff --git a/pkg/agent/steering_test.go b/pkg/agent/steering_test.go index bba988672..9330ec238 100644 --- a/pkg/agent/steering_test.go +++ b/pkg/agent/steering_test.go @@ -1051,16 +1051,18 @@ func TestAgentLoop_Continue_PreservesSteeringMedia(t *testing.T) { foundResolvedMedia := false for _, msg := range msgs { - if msg.Role != "user" || msg.Content != "describe this image" || len(msg.Media) != 1 { + if msg.Role != "user" { continue } - if strings.HasPrefix(msg.Media[0], "data:image/png;base64,") { + hasBase64 := len(msg.Media) > 0 && strings.HasPrefix(msg.Media[0], "data:image/png;base64,") + hasPathTag := strings.Contains(msg.Content, "[image:") + if hasBase64 && hasPathTag { foundResolvedMedia = true break } } if !foundResolvedMedia { - t.Fatal("expected continue path to inject steering media into the provider request") + t.Fatal("expected continue path to inject both base64 media and image path tag") } defaultAgent := al.registry.GetDefaultAgent()