feat(media): add image path tags alongside base64 for LLM file access

Images are still base64-encoded into msg.Media for multimodal LLMs,
but now also get [image:path] tags injected into message content so
the LLM knows the local file path for save/forward operations.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Guoguo 2026-04-28 23:15:25 -07:00
parent 0420d1dcb1
commit 4fedc16086
3 changed files with 45 additions and 18 deletions

View file

@ -66,6 +66,7 @@ func resolveMediaRefs(messages []providers.Message, store media.MediaStore, maxS
} }
mime := detectMIME(localPath, meta) mime := detectMIME(localPath, meta)
pathTags = append(pathTags, buildPathTag(mime, localPath))
if strings.HasPrefix(mime, "image/") { if strings.HasPrefix(mime, "image/") {
dataURL := encodeImageToDataURL(localPath, mime, info, maxSize) dataURL := encodeImageToDataURL(localPath, mime, info, maxSize)
@ -74,8 +75,6 @@ func resolveMediaRefs(messages []providers.Message, store media.MediaStore, maxS
} }
continue continue
} }
pathTags = append(pathTags, buildPathTag(mime, localPath))
} }
result[i].Media = resolved result[i].Media = resolved
@ -182,6 +181,8 @@ func encodeImageToDataURL(localPath, mime string, info os.FileInfo, maxSize int)
// Tag type is derived from MIME: [audio:/path], [video:/path], or [file:/path]. // Tag type is derived from MIME: [audio:/path], [video:/path], or [file:/path].
func buildPathTag(mime, localPath string) string { func buildPathTag(mime, localPath string) string {
switch { switch {
case strings.HasPrefix(mime, "image/"):
return "[image:" + localPath + "]"
case strings.HasPrefix(mime, "audio/"): case strings.HasPrefix(mime, "audio/"):
return "[audio:" + localPath + "]" return "[audio:" + localPath + "]"
case strings.HasPrefix(mime, "video/"): case strings.HasPrefix(mime, "video/"):
@ -197,6 +198,8 @@ func injectPathTags(content string, tags []string) string {
for _, tag := range tags { for _, tag := range tags {
var generic string var generic string
switch { switch {
case strings.HasPrefix(tag, "[image:"):
generic = "[image: photo]"
case strings.HasPrefix(tag, "[audio:"): case strings.HasPrefix(tag, "[audio:"):
generic = "[audio]" generic = "[audio]"
case strings.HasPrefix(tag, "[video:"): case strings.HasPrefix(tag, "[video:"):

View file

@ -1781,11 +1781,12 @@ func (m *artifactThenSendProvider) Chat(
if messages[i].Role != "tool" { if messages[i].Role != "tool" {
continue continue
} }
start := strings.Index(messages[i].Content, "[file:") for _, prefix := range []string{"[image:", "[file:", "[audio:", "[video:"} {
start := strings.Index(messages[i].Content, prefix)
if start < 0 { if start < 0 {
continue continue
} }
rest := messages[i].Content[start+len("[file:"):] rest := messages[i].Content[start+len(prefix):]
end := strings.Index(rest, "]") end := strings.Index(rest, "]")
if end < 0 { if end < 0 {
continue continue
@ -1793,6 +1794,10 @@ func (m *artifactThenSendProvider) Chat(
artifactPath = rest[:end] artifactPath = rest[:end]
break break
} }
if artifactPath != "" {
break
}
}
if artifactPath == "" { if artifactPath == "" {
return nil, fmt.Errorf("provider did not receive artifact path in tool result") return nil, fmt.Errorf("provider did not receive artifact path in tool result")
} }
@ -4656,7 +4661,7 @@ func TestRun_PicoToolFeedbackSuppressesDuplicateInterimAssistantContent(t *testi
} }
} }
func TestResolveMediaRefs_ResolvesToBase64(t *testing.T) { func TestResolveMediaRefs_ImageBase64AndPathTag(t *testing.T) {
store := media.NewFileMediaStore() store := media.NewFileMediaStore()
dir := t.TempDir() dir := t.TempDir()
@ -4690,9 +4695,14 @@ func TestResolveMediaRefs_ResolvesToBase64(t *testing.T) {
if !strings.HasPrefix(result[0].Media[0], "data:image/png;base64,") { if !strings.HasPrefix(result[0].Media[0], "data:image/png;base64,") {
t.Fatalf("expected data:image/png;base64, prefix, got %q", result[0].Media[0][:40]) t.Fatalf("expected data:image/png;base64, prefix, got %q", result[0].Media[0][:40])
} }
localPath, _, _ := store.ResolveWithMeta(ref)
expectedContent := "describe this [image:" + localPath + "]"
if result[0].Content != expectedContent {
t.Fatalf("expected content %q, got %q", expectedContent, result[0].Content)
}
} }
func TestResolveMediaRefs_SkipsOversizedFile(t *testing.T) { func TestResolveMediaRefs_OversizedImageSkipsBase64KeepsPathTag(t *testing.T) {
store := media.NewFileMediaStore() store := media.NewFileMediaStore()
dir := t.TempDir() dir := t.TempDir()
@ -4714,6 +4724,11 @@ func TestResolveMediaRefs_SkipsOversizedFile(t *testing.T) {
if len(result[0].Media) != 0 { if len(result[0].Media) != 0 {
t.Fatalf("expected 0 media (oversized), got %d", len(result[0].Media)) t.Fatalf("expected 0 media (oversized), got %d", len(result[0].Media))
} }
localPath, _, _ := store.ResolveWithMeta(ref)
expected := "hi [image:" + localPath + "]"
if result[0].Content != expected {
t.Fatalf("expected content %q, got %q", expected, result[0].Content)
}
} }
func TestResolveMediaRefs_UnknownTypeInjectsPath(t *testing.T) { func TestResolveMediaRefs_UnknownTypeInjectsPath(t *testing.T) {
@ -4797,6 +4812,11 @@ func TestResolveMediaRefs_UsesMetaContentType(t *testing.T) {
if !strings.HasPrefix(result[0].Media[0], "data:image/jpeg;base64,") { if !strings.HasPrefix(result[0].Media[0], "data:image/jpeg;base64,") {
t.Fatalf("expected jpeg prefix, got %q", result[0].Media[0][:30]) t.Fatalf("expected jpeg prefix, got %q", result[0].Media[0][:30])
} }
localPath, _, _ := store.ResolveWithMeta(ref)
expectedContent := "hi [image:" + localPath + "]"
if result[0].Content != expectedContent {
t.Fatalf("expected content %q, got %q", expectedContent, result[0].Content)
}
} }
func TestResolveMediaRefs_PDFInjectsFilePath(t *testing.T) { func TestResolveMediaRefs_PDFInjectsFilePath(t *testing.T) {
@ -4929,12 +4949,14 @@ func TestResolveMediaRefs_MixedImageAndFile(t *testing.T) {
result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize) result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize)
if len(result[0].Media) != 1 { if len(result[0].Media) != 1 {
t.Fatalf("expected 1 media (image only), got %d", len(result[0].Media)) t.Fatalf("expected 1 media (image base64 only), got %d", len(result[0].Media))
} }
if !strings.HasPrefix(result[0].Media[0], "data:image/png;base64,") { if !strings.HasPrefix(result[0].Media[0], "data:image/png;base64,") {
t.Fatal("expected image to be base64 encoded") t.Fatal("expected image to be base64 encoded")
} }
expectedContent := "check these [file:" + pdfPath + "]" imgLocalPath, _, _ := store.ResolveWithMeta(imgRef)
pdfLocalPath, _, _ := store.ResolveWithMeta(fileRef)
expectedContent := "check these [file:" + pdfLocalPath + "] [image:" + imgLocalPath + "]"
if result[0].Content != expectedContent { if result[0].Content != expectedContent {
t.Fatalf("expected content %q, got %q", expectedContent, result[0].Content) t.Fatalf("expected content %q, got %q", expectedContent, result[0].Content)
} }

View file

@ -1051,16 +1051,18 @@ func TestAgentLoop_Continue_PreservesSteeringMedia(t *testing.T) {
foundResolvedMedia := false foundResolvedMedia := false
for _, msg := range msgs { for _, msg := range msgs {
if msg.Role != "user" || msg.Content != "describe this image" || len(msg.Media) != 1 { if msg.Role != "user" {
continue continue
} }
if strings.HasPrefix(msg.Media[0], "data:image/png;base64,") { hasBase64 := len(msg.Media) > 0 && strings.HasPrefix(msg.Media[0], "data:image/png;base64,")
hasPathTag := strings.Contains(msg.Content, "[image:")
if hasBase64 && hasPathTag {
foundResolvedMedia = true foundResolvedMedia = true
break break
} }
} }
if !foundResolvedMedia { if !foundResolvedMedia {
t.Fatal("expected continue path to inject steering media into the provider request") t.Fatal("expected continue path to inject both base64 media and image path tag")
} }
defaultAgent := al.registry.GetDefaultAgent() defaultAgent := al.registry.GetDefaultAgent()