opencode/picoclaw
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(media): add image path tags alongside base64 for LLM file access

Browse source 
Images are still base64-encoded into msg.Media for multimodal LLMs,
but now also get [image:path] tags injected into message content so
the LLM knows the local file path for save/forward operations.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Guoguo 2026-04-28 23:15:25 -07:00
parent 0420d1dcb1
commit 4fedc16086
3 changed files with 45 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
pkg/agent/agent_media.go
View file

@ -66,6 +66,7 @@ func resolveMediaRefs(messages []providers.Message, store media.MediaStore, maxS
}
mime := detectMIME(localPath, meta)
pathTags = append(pathTags, buildPathTag(mime, localPath))
if strings.HasPrefix(mime, "image/") {
dataURL := encodeImageToDataURL(localPath, mime, info, maxSize)
@ -74,8 +75,6 @@ func resolveMediaRefs(messages []providers.Message, store media.MediaStore, maxS
}
continue
}
pathTags = append(pathTags, buildPathTag(mime, localPath))
}
result[i].Media = resolved
@ -182,6 +181,8 @@ func encodeImageToDataURL(localPath, mime string, info os.FileInfo, maxSize int)
// Tag type is derived from MIME: [audio:/path], [video:/path], or [file:/path].
func buildPathTag(mime, localPath string) string {
switch {
case strings.HasPrefix(mime, "image/"):
return "[image:" + localPath + "]"
case strings.HasPrefix(mime, "audio/"):
return "[audio:" + localPath + "]"
case strings.HasPrefix(mime, "video/"):
@ -197,6 +198,8 @@ func injectPathTags(content string, tags []string) string {
for _, tag := range tags {
var generic string
switch {
case strings.HasPrefix(tag, "[image:"):
generic = "[image: photo]"
case strings.HasPrefix(tag, "[audio:"):
generic = "[audio]"
case strings.HasPrefix(tag, "[video:"):

48
pkg/agent/agent_test.go
View file

@ -1781,17 +1781,22 @@ func (m *artifactThenSendProvider) Chat(
if messages[i].Role != "tool" {
continue
}
start := strings.Index(messages[i].Content, "[file:")
if start < 0 {
continue
for _, prefix := range []string{"[image:", "[file:", "[audio:", "[video:"} {
start := strings.Index(messages[i].Content, prefix)
if start < 0 {
continue
}
rest := messages[i].Content[start+len(prefix):]
end := strings.Index(rest, "]")
if end < 0 {
continue
}
artifactPath = rest[:end]
break
}
rest := messages[i].Content[start+len("[file:"):]
end := strings.Index(rest, "]")
if end < 0 {
continue
if artifactPath != "" {
break
}
artifactPath = rest[:end]
break
}
if artifactPath == "" {
return nil, fmt.Errorf("provider did not receive artifact path in tool result")
@ -4656,7 +4661,7 @@ func TestRun_PicoToolFeedbackSuppressesDuplicateInterimAssistantContent(t *testi
}
}
func TestResolveMediaRefs_ResolvesToBase64(t *testing.T) {
func TestResolveMediaRefs_ImageBase64AndPathTag(t *testing.T) {
store := media.NewFileMediaStore()
dir := t.TempDir()
@ -4690,9 +4695,14 @@ func TestResolveMediaRefs_ResolvesToBase64(t *testing.T) {
if !strings.HasPrefix(result[0].Media[0], "data:image/png;base64,") {
t.Fatalf("expected data:image/png;base64, prefix, got %q", result[0].Media[0][:40])
}
localPath, _, _ := store.ResolveWithMeta(ref)
expectedContent := "describe this [image:" + localPath + "]"
if result[0].Content != expectedContent {
t.Fatalf("expected content %q, got %q", expectedContent, result[0].Content)
}
}
func TestResolveMediaRefs_SkipsOversizedFile(t *testing.T) {
func TestResolveMediaRefs_OversizedImageSkipsBase64KeepsPathTag(t *testing.T) {
store := media.NewFileMediaStore()
dir := t.TempDir()
@ -4714,6 +4724,11 @@ func TestResolveMediaRefs_SkipsOversizedFile(t *testing.T) {
if len(result[0].Media) != 0 {
t.Fatalf("expected 0 media (oversized), got %d", len(result[0].Media))
}
localPath, _, _ := store.ResolveWithMeta(ref)
expected := "hi [image:" + localPath + "]"
if result[0].Content != expected {
t.Fatalf("expected content %q, got %q", expected, result[0].Content)
}
}
func TestResolveMediaRefs_UnknownTypeInjectsPath(t *testing.T) {
@ -4797,6 +4812,11 @@ func TestResolveMediaRefs_UsesMetaContentType(t *testing.T) {
if !strings.HasPrefix(result[0].Media[0], "data:image/jpeg;base64,") {
t.Fatalf("expected jpeg prefix, got %q", result[0].Media[0][:30])
}
localPath, _, _ := store.ResolveWithMeta(ref)
expectedContent := "hi [image:" + localPath + "]"
if result[0].Content != expectedContent {
t.Fatalf("expected content %q, got %q", expectedContent, result[0].Content)
}
}
func TestResolveMediaRefs_PDFInjectsFilePath(t *testing.T) {
@ -4929,12 +4949,14 @@ func TestResolveMediaRefs_MixedImageAndFile(t *testing.T) {
result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize)
if len(result[0].Media) != 1 {
t.Fatalf("expected 1 media (image only), got %d", len(result[0].Media))
t.Fatalf("expected 1 media (image base64 only), got %d", len(result[0].Media))
}
if !strings.HasPrefix(result[0].Media[0], "data:image/png;base64,") {
t.Fatal("expected image to be base64 encoded")
}
expectedContent := "check these [file:" + pdfPath + "]"
imgLocalPath, _, _ := store.ResolveWithMeta(imgRef)
pdfLocalPath, _, _ := store.ResolveWithMeta(fileRef)
expectedContent := "check these [file:" + pdfLocalPath + "] [image:" + imgLocalPath + "]"
if result[0].Content != expectedContent {
t.Fatalf("expected content %q, got %q", expectedContent, result[0].Content)
}

8
pkg/agent/steering_test.go
View file

@ -1051,16 +1051,18 @@ func TestAgentLoop_Continue_PreservesSteeringMedia(t *testing.T) {
foundResolvedMedia := false
for _, msg := range msgs {
if msg.Role != "user" || msg.Content != "describe this image" || len(msg.Media) != 1 {
if msg.Role != "user" {
continue
}
if strings.HasPrefix(msg.Media[0], "data:image/png;base64,") {
hasBase64 := len(msg.Media) > 0 && strings.HasPrefix(msg.Media[0], "data:image/png;base64,")
hasPathTag := strings.Contains(msg.Content, "[image:")
if hasBase64 && hasPathTag {
foundResolvedMedia = true
break
}
}
if !foundResolvedMedia {
t.Fatal("expected continue path to inject steering media into the provider request")
t.Fatal("expected continue path to inject both base64 media and image path tag")
}
defaultAgent := al.registry.GetDefaultAgent()