feat(feishu): support downloading external images from interactive cards
Previously only Feishu-hosted images (img_key, icon_key) could be downloaded. Now external URLs in src field are also downloaded via HTTP and made available to the LLM. - extractCardImageKeys now returns two slices: Feishu keys and external URLs - Add downloadExternalImage to download images from HTTP URLs - Update downloadInboundMedia to handle both Feishu API and HTTP downloads - Update tests for new function signature
This commit is contained in:
parent
be4808dafe
commit
40e5510b77
3 changed files with 204 additions and 75 deletions
|
|
@ -87,66 +87,61 @@ func stripMentionPlaceholders(content string, mentions []*larkim.MentionEvent) s
|
||||||
|
|
||||||
// extractCardImageKeys recursively extracts all image keys from a Feishu interactive card.
|
// extractCardImageKeys recursively extracts all image keys from a Feishu interactive card.
|
||||||
// Image keys are used to download images from Feishu API.
|
// Image keys are used to download images from Feishu API.
|
||||||
// Only Feishu-hosted keys are returned (img_xxx, icon_xxx); external URLs are filtered out.
|
// Returns two slices: Feishu-hosted keys and external URLs.
|
||||||
func extractCardImageKeys(rawContent string) []string {
|
func extractCardImageKeys(rawContent string) (feishuKeys []string, externalURLs []string) {
|
||||||
if rawContent == "" {
|
if rawContent == "" {
|
||||||
return nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var card map[string]any
|
var card map[string]any
|
||||||
if err := json.Unmarshal([]byte(rawContent), &card); err != nil {
|
if err := json.Unmarshal([]byte(rawContent), &card); err != nil {
|
||||||
return nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var keys []string
|
extractImageKeysRecursive(card, &feishuKeys, &externalURLs)
|
||||||
extractImageKeysRecursive(card, &keys)
|
return feishuKeys, externalURLs
|
||||||
return keys
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// isFeishuImageKey returns true if the string is a Feishu-hosted image key
|
// isExternalURL returns true if the string is an external HTTP/HTTPS URL.
|
||||||
// (not an external URL). Feishu keys typically start with img_, icon_, or file_.
|
func isExternalURL(s string) bool {
|
||||||
func isFeishuImageKey(s string) bool {
|
return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://")
|
||||||
if s == "" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
// Filter out external URLs
|
|
||||||
if strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// extractImageKeysRecursive traverses card structure to find all image keys.
|
// extractImageKeysRecursive traverses card structure to find all image keys.
|
||||||
// Only Feishu-hosted keys are collected; external URLs are skipped.
|
// Collects both Feishu-hosted keys and external URLs separately.
|
||||||
func extractImageKeysRecursive(v any, keys *[]string) {
|
func extractImageKeysRecursive(v any, feishuKeys, externalURLs *[]string) {
|
||||||
switch val := v.(type) {
|
switch val := v.(type) {
|
||||||
case map[string]any:
|
case map[string]any:
|
||||||
// Check if this is an img element
|
// Check if this is an img element
|
||||||
if tag, ok := val["tag"].(string); ok {
|
if tag, ok := val["tag"].(string); ok {
|
||||||
switch tag {
|
switch tag {
|
||||||
case "img":
|
case "img":
|
||||||
// Try img_key first (most common, always Feishu-hosted)
|
// Try img_key first (always Feishu-hosted)
|
||||||
if imgKey, ok := val["img_key"].(string); ok && imgKey != "" {
|
if imgKey, ok := val["img_key"].(string); ok && imgKey != "" {
|
||||||
*keys = append(*keys, imgKey)
|
*feishuKeys = append(*feishuKeys, imgKey)
|
||||||
}
|
}
|
||||||
// Also try src, but only if it's a Feishu-hosted key (not external URL)
|
// Check src - could be Feishu key or external URL
|
||||||
if src, ok := val["src"].(string); ok && src != "" && isFeishuImageKey(src) {
|
if src, ok := val["src"].(string); ok && src != "" {
|
||||||
*keys = append(*keys, src)
|
if isExternalURL(src) {
|
||||||
|
*externalURLs = append(*externalURLs, src)
|
||||||
|
} else {
|
||||||
|
*feishuKeys = append(*feishuKeys, src)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
case "icon":
|
case "icon":
|
||||||
// Icon elements use icon_key
|
// Icon elements use icon_key
|
||||||
if iconKey, ok := val["icon_key"].(string); ok && iconKey != "" {
|
if iconKey, ok := val["icon_key"].(string); ok && iconKey != "" {
|
||||||
*keys = append(*keys, iconKey)
|
*feishuKeys = append(*feishuKeys, iconKey)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Recurse into all nested structures
|
// Recurse into all nested structures
|
||||||
for _, child := range val {
|
for _, child := range val {
|
||||||
extractImageKeysRecursive(child, keys)
|
extractImageKeysRecursive(child, feishuKeys, externalURLs)
|
||||||
}
|
}
|
||||||
case []any:
|
case []any:
|
||||||
for _, item := range val {
|
for _, item := range val {
|
||||||
extractImageKeysRecursive(item, keys)
|
extractImageKeysRecursive(item, feishuKeys, externalURLs)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -293,82 +293,114 @@ func TestStripMentionPlaceholders(t *testing.T) {
|
||||||
|
|
||||||
func TestExtractCardImageKeys(t *testing.T) {
|
func TestExtractCardImageKeys(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
content string
|
content string
|
||||||
want []string
|
wantFeishuKeys []string
|
||||||
|
wantExternalURLs []string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "empty content",
|
name: "empty content",
|
||||||
content: "",
|
content: "",
|
||||||
want: nil,
|
wantFeishuKeys: nil,
|
||||||
|
wantExternalURLs: nil,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "invalid JSON",
|
name: "invalid JSON",
|
||||||
content: "not json",
|
content: "not json",
|
||||||
want: nil,
|
wantFeishuKeys: nil,
|
||||||
|
wantExternalURLs: nil,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "card with no images",
|
name: "card with no images",
|
||||||
content: `{"schema":"2.0","body":{"elements":[{"tag":"markdown","content":"text"}]}}`,
|
content: `{"schema":"2.0","body":{"elements":[{"tag":"markdown","content":"text"}]}}`,
|
||||||
want: nil,
|
wantFeishuKeys: nil,
|
||||||
|
wantExternalURLs: nil,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "single image with img_key",
|
name: "single image with img_key",
|
||||||
content: `{"elements":[{"tag":"img","img_key":"img_abc123"}]}`,
|
content: `{"elements":[{"tag":"img","img_key":"img_abc123"}]}`,
|
||||||
want: []string{"img_abc123"},
|
wantFeishuKeys: []string{"img_abc123"},
|
||||||
|
wantExternalURLs: nil,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "single image with src",
|
name: "single image with src as Feishu key",
|
||||||
content: `{"elements":[{"tag":"img","src":"img_xyz789"}]}`,
|
content: `{"elements":[{"tag":"img","src":"img_xyz789"}]}`,
|
||||||
want: []string{"img_xyz789"},
|
wantFeishuKeys: []string{"img_xyz789"},
|
||||||
|
wantExternalURLs: nil,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "multiple images",
|
name: "multiple images",
|
||||||
content: `{"elements":[{"tag":"img","img_key":"img_1"},{"tag":"div","text":{"content":"text"}},{"tag":"img","img_key":"img_2"}]}`,
|
content: `{"elements":[{"tag":"img","img_key":"img_1"},{"tag":"div","text":{"content":"text"}},{"tag":"img","img_key":"img_2"}]}`,
|
||||||
want: []string{"img_1", "img_2"},
|
wantFeishuKeys: []string{"img_1", "img_2"},
|
||||||
|
wantExternalURLs: nil,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "nested image in columns",
|
name: "nested image in columns",
|
||||||
content: `{"elements":[{"tag":"div","columns":[{"tag":"img","img_key":"img_col1"},{"tag":"img","img_key":"img_col2"}]}]}`,
|
content: `{"elements":[{"tag":"div","columns":[{"tag":"img","img_key":"img_col1"},{"tag":"img","img_key":"img_col2"}]}]}`,
|
||||||
want: []string{"img_col1", "img_col2"},
|
wantFeishuKeys: []string{"img_col1", "img_col2"},
|
||||||
|
wantExternalURLs: nil,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "image in action",
|
name: "image in action",
|
||||||
content: `{"elements":[{"tag":"action","actions":[{"tag":"img","img_key":"img_action"}]}]}`,
|
content: `{"elements":[{"tag":"action","actions":[{"tag":"img","img_key":"img_action"}]}]}`,
|
||||||
want: []string{"img_action"},
|
wantFeishuKeys: []string{"img_action"},
|
||||||
|
wantExternalURLs: nil,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "icon element",
|
name: "icon element",
|
||||||
content: `{"elements":[{"tag":"icon","icon_key":"icon_123"}]}`,
|
content: `{"elements":[{"tag":"icon","icon_key":"icon_123"}]}`,
|
||||||
want: []string{"icon_123"},
|
wantFeishuKeys: []string{"icon_123"},
|
||||||
|
wantExternalURLs: nil,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "complex card with text and images",
|
name: "complex card with text and images",
|
||||||
content: `{"header":{"title":{"content":"Title"}},"elements":[{"tag":"div","text":{"content":"Description"}},{"tag":"img","img_key":"img_main"}]}`,
|
content: `{"header":{"title":{"content":"Title"}},"elements":[{"tag":"div","text":{"content":"Description"}},{"tag":"img","img_key":"img_main"}]}`,
|
||||||
want: []string{"img_main"},
|
wantFeishuKeys: []string{"img_main"},
|
||||||
|
wantExternalURLs: nil,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "external URL in src is filtered out",
|
name: "external URL in src",
|
||||||
content: `{"elements":[{"tag":"img","src":"https://example.com/image.png"}]}`,
|
content: `{"elements":[{"tag":"img","src":"https://example.com/image.png"}]}`,
|
||||||
want: nil,
|
wantFeishuKeys: nil,
|
||||||
|
wantExternalURLs: []string{"https://example.com/image.png"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "mixed Feishu keys and external URLs",
|
name: "mixed Feishu keys and external URLs",
|
||||||
content: `{"elements":[{"tag":"img","img_key":"img_feishu"},{"tag":"img","src":"https://cdn.example.com/external.jpg"},{"tag":"img","src":"img_another"}]}`,
|
content: `{"elements":[{"tag":"img","img_key":"img_feishu"},{"tag":"img","src":"https://cdn.example.com/external.jpg"},{"tag":"img","src":"img_another"}]}`,
|
||||||
want: []string{"img_feishu", "img_another"},
|
wantFeishuKeys: []string{"img_feishu", "img_another"},
|
||||||
|
wantExternalURLs: []string{"https://cdn.example.com/external.jpg"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "multiple external URLs",
|
||||||
|
content: `{"elements":[{"tag":"img","src":"https://a.com/1.png"},{"tag":"img","src":"http://b.com/2.jpg"}]}`,
|
||||||
|
wantFeishuKeys: nil,
|
||||||
|
wantExternalURLs: []string{"https://a.com/1.png", "http://b.com/2.jpg"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
got := extractCardImageKeys(tt.content)
|
gotFeishuKeys, gotExternalURLs := extractCardImageKeys(tt.content)
|
||||||
if len(got) != len(tt.want) {
|
|
||||||
t.Errorf("extractCardImageKeys() = %v, want %v", got, tt.want)
|
// Compare Feishu keys
|
||||||
|
if len(gotFeishuKeys) != len(tt.wantFeishuKeys) {
|
||||||
|
t.Errorf("extractCardImageKeys() feishuKeys = %v, want %v", gotFeishuKeys, tt.wantFeishuKeys)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
for i, v := range got {
|
for i, v := range gotFeishuKeys {
|
||||||
if v != tt.want[i] {
|
if v != tt.wantFeishuKeys[i] {
|
||||||
t.Errorf("extractCardImageKeys()[%d] = %q, want %q", i, v, tt.want[i])
|
t.Errorf("extractCardImageKeys() feishuKeys[%d] = %q, want %q", i, v, tt.wantFeishuKeys[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compare external URLs
|
||||||
|
if len(gotExternalURLs) != len(tt.wantExternalURLs) {
|
||||||
|
t.Errorf("extractCardImageKeys() externalURLs = %v, want %v", gotExternalURLs, tt.wantExternalURLs)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for i, v := range gotExternalURLs {
|
||||||
|
if v != tt.wantExternalURLs[i] {
|
||||||
|
t.Errorf("extractCardImageKeys() externalURLs[%d] = %q, want %q", i, v, tt.wantExternalURLs[i])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -550,13 +550,21 @@ func (c *FeishuChannel) downloadInboundMedia(
|
||||||
|
|
||||||
case larkim.MsgTypeInteractive:
|
case larkim.MsgTypeInteractive:
|
||||||
// Extract and download images embedded in interactive cards
|
// Extract and download images embedded in interactive cards
|
||||||
imageKeys := extractCardImageKeys(rawContent)
|
feishuKeys, externalURLs := extractCardImageKeys(rawContent)
|
||||||
for _, imageKey := range imageKeys {
|
// Download Feishu-hosted images via API
|
||||||
|
for _, imageKey := range feishuKeys {
|
||||||
ref := c.downloadResource(ctx, messageID, imageKey, "image", ".jpg", store, scope)
|
ref := c.downloadResource(ctx, messageID, imageKey, "image", ".jpg", store, scope)
|
||||||
if ref != "" {
|
if ref != "" {
|
||||||
refs = append(refs, ref)
|
refs = append(refs, ref)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Download external images via HTTP
|
||||||
|
for _, imageURL := range externalURLs {
|
||||||
|
ref := c.downloadExternalImage(ctx, imageURL, store, scope)
|
||||||
|
if ref != "" {
|
||||||
|
refs = append(refs, ref)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
case larkim.MsgTypeFile, larkim.MsgTypeAudio, larkim.MsgTypeMedia:
|
case larkim.MsgTypeFile, larkim.MsgTypeAudio, larkim.MsgTypeMedia:
|
||||||
fileKey := extractFileKey(rawContent)
|
fileKey := extractFileKey(rawContent)
|
||||||
|
|
@ -676,6 +684,100 @@ func (c *FeishuChannel) downloadResource(
|
||||||
return ref
|
return ref
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// downloadExternalImage downloads an image from an external URL and stores it in MediaStore.
|
||||||
|
// Returns the media reference on success, or empty string on failure.
|
||||||
|
func (c *FeishuChannel) downloadExternalImage(
|
||||||
|
ctx context.Context,
|
||||||
|
imageURL string,
|
||||||
|
store media.MediaStore,
|
||||||
|
scope string,
|
||||||
|
) string {
|
||||||
|
// Create HTTP request with context
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorCF("feishu", "Failed to create request for external image", map[string]any{
|
||||||
|
"url": imageURL,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download image
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorCF("feishu", "Failed to download external image", map[string]any{
|
||||||
|
"url": imageURL,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
logger.ErrorCF("feishu", "External image download failed with status", map[string]any{
|
||||||
|
"url": imageURL,
|
||||||
|
"status": resp.StatusCode,
|
||||||
|
})
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// Determine filename from URL path
|
||||||
|
filename := filepath.Base(imageURL)
|
||||||
|
if filename == "" || filename == "." || filename == "/" {
|
||||||
|
filename = "external_image"
|
||||||
|
}
|
||||||
|
// Ensure we have an extension for images
|
||||||
|
if filepath.Ext(filename) == "" {
|
||||||
|
filename += ".jpg"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write to the shared picoclaw_media directory using a unique name to avoid collisions.
|
||||||
|
mediaDir := filepath.Join(os.TempDir(), "picoclaw_media")
|
||||||
|
if mkdirErr := os.MkdirAll(mediaDir, 0o700); mkdirErr != nil {
|
||||||
|
logger.ErrorCF("feishu", "Failed to create media directory", map[string]any{
|
||||||
|
"error": mkdirErr.Error(),
|
||||||
|
})
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
ext := filepath.Ext(filename)
|
||||||
|
localPath := filepath.Join(mediaDir, utils.SanitizeFilename(fmt.Sprintf("ext_%d%s", rand.Int63(), ext)))
|
||||||
|
|
||||||
|
out, err := os.Create(localPath)
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorCF("feishu", "Failed to create temp file for external image", map[string]any{
|
||||||
|
"path": localPath,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := io.Copy(out, resp.Body); err != nil {
|
||||||
|
out.Close()
|
||||||
|
os.Remove(localPath)
|
||||||
|
logger.ErrorCF("feishu", "Failed to write external image to file", map[string]any{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
out.Close()
|
||||||
|
|
||||||
|
// Store in MediaStore
|
||||||
|
ref, err := store.Store(localPath, media.MediaMeta{
|
||||||
|
Filename: filename,
|
||||||
|
Source: "feishu_external",
|
||||||
|
}, scope)
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorCF("feishu", "Failed to store external image", map[string]any{
|
||||||
|
"url": imageURL,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
os.Remove(localPath)
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
return ref
|
||||||
|
}
|
||||||
|
|
||||||
// appendMediaTags appends media type tags to content (like Telegram's "[image: photo]").
|
// appendMediaTags appends media type tags to content (like Telegram's "[image: photo]").
|
||||||
// For interactive cards, media tags are not appended because content is raw JSON
|
// For interactive cards, media tags are not appended because content is raw JSON
|
||||||
// and appending would produce invalid JSON format.
|
// and appending would produce invalid JSON format.
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue