From bc45e312279789433159d10db8eb0e9aa2e62627 Mon Sep 17 00:00:00 2001 From: dj-oyu <68707227+dj-oyu@users.noreply.github.com> Date: Thu, 19 Mar 2026 22:39:26 +0900 Subject: [PATCH 1/8] feat: add PDF OCR processing with yomitoku CLI integration PDF files sent via chat are now processed with an external OCR command (configurable via config.json `ocr` section). The OCR output (markdown) is cached in media_cache.db with a first-page preview stored inline. The LLM receives a [document: preview (full: path, N pages)] tag and can use read_file to access the complete OCR result on demand. - OCRConfig: command, args, env, timeout in AgentDefaults - PDFPageCount: lightweight /Count N parser (fallback to "?" on failure) - processPDFsInMessages: replaces [file:*.pdf] tags with OCR results - progressIndicator: upgraded to dynamic labels via atomic.Value for real-time page progress from stderr (TextDetector cycle counting) - ocrPDF: exec.CommandContext with stderr parsing, cache integration - OCR output stored in workspace/.ocr_cache/ Co-Authored-By: Claude Opus 4.6 (1M context) --- pkg/agent/loop_media.go | 306 +++++++++++++++++++++++++++++++++-- pkg/agent/loop_media_test.go | 73 +++++++++ pkg/agent/loop_run.go | 5 + pkg/config/config.go | 17 ++ pkg/mediacache/pdf.go | 52 ++++++ pkg/mediacache/pdf_test.go | 81 ++++++++++ 6 files changed, 522 insertions(+), 12 deletions(-) create mode 100644 pkg/mediacache/pdf.go create mode 100644 pkg/mediacache/pdf_test.go diff --git a/pkg/agent/loop_media.go b/pkg/agent/loop_media.go index 768f7ef42..3086b44cb 100644 --- a/pkg/agent/loop_media.go +++ b/pkg/agent/loop_media.go @@ -7,18 +7,23 @@ package agent import ( + "bufio" "bytes" "context" "encoding/base64" "fmt" "io" "os" + "os/exec" + "path/filepath" "strings" + "sync/atomic" "time" "github.com/h2non/filetype" "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/media" "github.com/sipeed/picoclaw/pkg/mediacache" @@ -216,8 +221,8 @@ func (al *AgentLoop) describeImagesInMessages( if imageCount > 1 { label = fmt.Sprintf("Processing %d images...", imageCount) } - stopIndicator := al.processingIndicator(ctx, channel, chatID, label) - defer stopIndicator() + indicator := al.processingIndicator(ctx, channel, chatID, label) + defer indicator.Stop() for i, m := range result { if len(m.Media) == 0 { @@ -326,15 +331,38 @@ func (al *AgentLoop) describeImage( // a smooth rotating animation when displayed sequentially. var brailleSpinnerFrames = [...]string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"} +// progressIndicator manages a braille spinner with a dynamically updatable label. +// Call UpdateLabel to change the displayed text during processing. +type progressIndicator struct { + label atomic.Value // string + done chan struct{} +} + +// UpdateLabel changes the label shown alongside the spinner. +func (p *progressIndicator) UpdateLabel(label string) { + p.label.Store(label) +} + +// Stop terminates the spinner goroutine. +func (p *progressIndicator) Stop() { + select { + case <-p.done: + default: + close(p.done) + } +} + // processingIndicator publishes draft status messages with a braille spinner -// animation to indicate active processing. It runs until the returned stop -// function is called. The label describes what is being processed. -func (al *AgentLoop) processingIndicator(ctx context.Context, channel, chatID, label string) (stop func()) { +// animation to indicate active processing. It runs until Stop is called. +// Use UpdateLabel to change the displayed text during long operations. +func (al *AgentLoop) processingIndicator(ctx context.Context, channel, chatID, label string) *progressIndicator { + p := &progressIndicator{done: make(chan struct{})} + p.label.Store(label) + if al.bus == nil || channel == "" || chatID == "" { - return func() {} + return p } - done := make(chan struct{}) go func() { ticker := time.NewTicker(150 * time.Millisecond) defer ticker.Stop() @@ -342,12 +370,13 @@ func (al *AgentLoop) processingIndicator(ctx context.Context, channel, chatID, l frame := 0 for { select { - case <-done: + case <-p.done: return case <-ctx.Done(): return case <-ticker.C: - content := fmt.Sprintf("%s %s", brailleSpinnerFrames[frame%len(brailleSpinnerFrames)], label) + lbl, _ := p.label.Load().(string) + content := fmt.Sprintf("%s %s", brailleSpinnerFrames[frame%len(brailleSpinnerFrames)], lbl) _ = al.bus.PublishOutbound(ctx, bus.OutboundMessage{ Channel: channel, ChatID: chatID, @@ -359,9 +388,7 @@ func (al *AgentLoop) processingIndicator(ctx context.Context, channel, chatID, l } }() - return func() { - close(done) - } + return p } // injectImageDescriptions replaces "[image: photo]" tags in content with @@ -379,3 +406,258 @@ func injectImageDescriptions(content string, descriptions []string) string { } return content } + +// maxPreviewRunes is the maximum number of runes to store as preview +// in the media cache for PDF OCR results. +const maxPreviewRunes = 500 + +// processPDFsInMessages finds [file:/path.pdf] tags in messages and replaces +// them with [document: preview... (full: /path/to.md, N pages)] tags after +// running OCR. A braille spinner with page progress is shown during processing. +func (al *AgentLoop) processPDFsInMessages( + ctx context.Context, messages []providers.Message, ocrCfg *config.OCRConfig, + channel, chatID string, +) []providers.Message { + result := make([]providers.Message, len(messages)) + copy(result, messages) + + for i, m := range result { + if !strings.Contains(m.Content, "[file:") { + continue + } + result[i].Content = al.replacePDFTags(ctx, m.Content, ocrCfg, channel, chatID) + } + + return result +} + +// pdfTagPrefix is the file tag pattern for PDF files injected by resolveMediaRefs. +const pdfTagPrefix = "[file:" + +// replacePDFTags finds [file:*.pdf] tags and replaces them with OCR results. +func (al *AgentLoop) replacePDFTags( + ctx context.Context, content string, ocrCfg *config.OCRConfig, + channel, chatID string, +) string { + var out strings.Builder + rest := content + + for { + idx := strings.Index(rest, pdfTagPrefix) + if idx < 0 { + out.WriteString(rest) + break + } + endRel := strings.Index(rest[idx:], "]") + if endRel < 0 { + out.WriteString(rest) + break + } + end := idx + endRel + 1 + + tag := rest[idx:end] + path := tag[len(pdfTagPrefix) : len(tag)-1] + + out.WriteString(rest[:idx]) + + if strings.HasSuffix(strings.ToLower(path), ".pdf") { + out.WriteString(al.ocrPDF(ctx, path, ocrCfg, channel, chatID)) + } else { + out.WriteString(tag) + } + + rest = rest[end:] + } + + return out.String() +} + +// ocrPDF runs OCR on a PDF file and returns a document tag with preview. +// Uses the media cache to avoid redundant OCR runs. +func (al *AgentLoop) ocrPDF( + ctx context.Context, pdfPath string, ocrCfg *config.OCRConfig, + channel, chatID string, +) string { + // Hash the file content for cache lookup + pdfData, err := os.ReadFile(pdfPath) + if err != nil { + logger.WarnCF("agent", "Failed to read PDF", map[string]any{"path": pdfPath, "error": err.Error()}) + return fmt.Sprintf("[file:%s]", pdfPath) + } + hash := mediacache.HashData(pdfData) + + // Check cache + if al.mediaCache != nil { + if entry, ok := al.mediaCache.GetEntry(hash, mediacache.TypePDFOCR); ok { + logger.DebugCF("agent", "PDF OCR cache hit", map[string]any{"hash": hash}) + return formatDocumentTag(entry.Result, entry.FilePath, entry.Pages) + } + } + + // Get page count for progress display + totalPages := mediacache.PDFPageCount(pdfPath) + totalStr := mediacache.FormatPageCount(totalPages) + + // Start progress indicator + indicator := al.processingIndicator(ctx, channel, chatID, + fmt.Sprintf("Processing PDF (0/%s)...", totalStr)) + defer indicator.Stop() + + // Determine output directory for OCR results + outputDir := al.ocrOutputDir() + os.MkdirAll(outputDir, 0o755) + + // Build command + timeout := time.Duration(ocrCfg.GetOCRTimeout()) * time.Second + cmdCtx, cmdCancel := context.WithTimeout(ctx, timeout) + defer cmdCancel() + + args := make([]string, 0, len(ocrCfg.Args)+4) + args = append(args, ocrCfg.Args...) + args = append(args, pdfPath, "-o", outputDir) + + cmd := exec.CommandContext(cmdCtx, ocrCfg.Command, args...) + + // Set environment + if len(ocrCfg.Env) > 0 { + cmd.Env = append(os.Environ(), ocrEnvSlice(ocrCfg.Env)...) + } + + // Pipe stderr for progress tracking + stderrPipe, err := cmd.StderrPipe() + if err != nil { + logger.WarnCF("agent", "Failed to create stderr pipe", map[string]any{"error": err.Error()}) + return fmt.Sprintf("[file:%s]", pdfPath) + } + + logger.InfoCF("agent", "Starting PDF OCR", map[string]any{ + "path": pdfPath, + "pages": totalStr, + "cmd": ocrCfg.Command, + }) + + if err := cmd.Start(); err != nil { + logger.WarnCF("agent", "Failed to start OCR command", map[string]any{"error": err.Error()}) + return fmt.Sprintf("[file:%s]", pdfPath) + } + + // Track progress via stderr + page := 0 + scanner := bufio.NewScanner(stderrPipe) + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(line, "TextDetector __call__") { + page++ + indicator.UpdateLabel(fmt.Sprintf("Processing PDF (%d/%s)...", page, totalStr)) + } + } + + if err := cmd.Wait(); err != nil { + logger.WarnCF("agent", "OCR command failed", map[string]any{ + "path": pdfPath, + "error": err.Error(), + }) + return fmt.Sprintf("[file:%s]", pdfPath) + } + + // Find the output markdown file + mdPath := findOCROutput(outputDir, pdfPath) + if mdPath == "" { + logger.WarnCF("agent", "OCR output not found", map[string]any{"output_dir": outputDir}) + return fmt.Sprintf("[file:%s]", pdfPath) + } + + // Read preview from first part of the markdown + mdData, err := os.ReadFile(mdPath) + if err != nil { + logger.WarnCF("agent", "Failed to read OCR output", map[string]any{"path": mdPath, "error": err.Error()}) + return fmt.Sprintf("[file:%s]", pdfPath) + } + + preview := extractPreview(string(mdData), maxPreviewRunes) + if totalPages == 0 { + totalPages = page // use detected page count as fallback + } + + // Store in cache + if al.mediaCache != nil { + _ = al.mediaCache.PutEntry(hash, mediacache.TypePDFOCR, mediacache.Entry{ + Result: preview, + FilePath: mdPath, + Pages: totalPages, + }) + } + + logger.InfoCF("agent", "PDF OCR completed", map[string]any{ + "path": pdfPath, + "pages": totalPages, + "md_path": mdPath, + }) + + return formatDocumentTag(preview, mdPath, totalPages) +} + +// formatDocumentTag creates the tag injected into message content. +func formatDocumentTag(preview, mdPath string, pages int) string { + pagesStr := mediacache.FormatPageCount(pages) + return fmt.Sprintf("[document: %s\n full: %s (%s pages)\n Use read_file to see the complete document.]", + preview, mdPath, pagesStr) +} + +// extractPreview returns the first maxRunes runes of text, appending "..." if truncated. +func extractPreview(text string, maxRunes int) string { + runes := []rune(strings.TrimSpace(text)) + if len(runes) <= maxRunes { + return string(runes) + } + return string(runes[:maxRunes]) + "..." +} + +// ocrOutputDir returns the directory for storing OCR markdown output files. +func (al *AgentLoop) ocrOutputDir() string { + registry := al.GetRegistry() + if agent := registry.GetDefaultAgent(); agent != nil { + return filepath.Join(agent.Workspace, ".ocr_cache") + } + return filepath.Join(os.TempDir(), "picoclaw-ocr") +} + +// findOCROutput locates the markdown file generated by yomitoku. +// yomitoku names output as .md or _combined.md in the output dir. +func findOCROutput(outputDir, pdfPath string) string { + base := strings.TrimSuffix(filepath.Base(pdfPath), filepath.Ext(pdfPath)) + + // Try common yomitoku output patterns + candidates := []string{ + filepath.Join(outputDir, base+".md"), + filepath.Join(outputDir, base+"_combined.md"), + } + + for _, c := range candidates { + if _, err := os.Stat(c); err == nil { + return c + } + } + + // Fallback: find any .md file in the output directory + entries, err := os.ReadDir(outputDir) + if err != nil { + return "" + } + for _, e := range entries { + if !e.IsDir() && strings.HasSuffix(e.Name(), ".md") { + return filepath.Join(outputDir, e.Name()) + } + } + + return "" +} + +// ocrEnvSlice converts a map to "KEY=VALUE" slice for exec.Cmd.Env. +func ocrEnvSlice(env map[string]string) []string { + result := make([]string, 0, len(env)) + for k, v := range env { + result = append(result, k+"="+v) + } + return result +} diff --git a/pkg/agent/loop_media_test.go b/pkg/agent/loop_media_test.go index 9afd647cb..e21e89b3b 100644 --- a/pkg/agent/loop_media_test.go +++ b/pkg/agent/loop_media_test.go @@ -1,6 +1,7 @@ package agent import ( + "strings" "testing" "github.com/sipeed/picoclaw/pkg/config" @@ -104,3 +105,75 @@ func TestResolveImageModel_FallsToPlanModel(t *testing.T) { t.Errorf("got %q, want %q", model, "openai/gpt-5.4-nano") } } + +func TestExtractPreview_Short(t *testing.T) { + text := "Hello world" + result := extractPreview(text, 500) + if result != "Hello world" { + t.Errorf("got %q", result) + } +} + +func TestExtractPreview_Truncated(t *testing.T) { + text := strings.Repeat("a", 600) + result := extractPreview(text, 500) + if len([]rune(result)) != 503 { // 500 + "..." + t.Errorf("len = %d, want 503", len([]rune(result))) + } + if !strings.HasSuffix(result, "...") { + t.Error("should end with ...") + } +} + +func TestFormatDocumentTag(t *testing.T) { + tag := formatDocumentTag("preview text", "/path/to/doc.md", 18) + if !strings.Contains(tag, "preview text") { + t.Error("should contain preview") + } + if !strings.Contains(tag, "/path/to/doc.md") { + t.Error("should contain file path") + } + if !strings.Contains(tag, "18 pages") { + t.Error("should contain page count") + } + if !strings.Contains(tag, "read_file") { + t.Error("should contain read_file hint") + } +} + +func TestFormatDocumentTag_UnknownPages(t *testing.T) { + tag := formatDocumentTag("preview", "/path.md", 0) + if !strings.Contains(tag, "? pages") { + t.Error("should show ? for unknown page count") + } +} + +func TestReplacePDFTags_NoPDF(t *testing.T) { + al := &AgentLoop{} + content := "Check this out [file:/path/to/audio.mp3]" + result := al.replacePDFTags(t.Context(), content, &config.OCRConfig{Command: "echo"}, "", "") + if result != content { + t.Errorf("non-PDF should be unchanged, got %q", result) + } +} + +func TestReplacePDFTags_NoTags(t *testing.T) { + al := &AgentLoop{} + content := "Hello world" + result := al.replacePDFTags(t.Context(), content, &config.OCRConfig{Command: "echo"}, "", "") + if result != content { + t.Errorf("no tags should be unchanged, got %q", result) + } +} + +func TestProcessPDFs_NilOCR(t *testing.T) { + // When OCR is not configured, messages should pass through unchanged + messages := []providers.Message{ + {Role: "user", Content: "Check [file:/tmp/doc.pdf]"}, + } + al := &AgentLoop{} + result := al.processPDFsInMessages(t.Context(), messages, nil, "", "") + if result[0].Content != messages[0].Content { + t.Errorf("content should be unchanged when OCR config is nil") + } +} diff --git a/pkg/agent/loop_run.go b/pkg/agent/loop_run.go index d0df75f43..66d4bed82 100644 --- a/pkg/agent/loop_run.go +++ b/pkg/agent/loop_run.go @@ -256,6 +256,11 @@ func (al *AgentLoop) runAgentLoop(ctx context.Context, agent *AgentInstance, opt messages = al.describeImagesInMessages(ctx, messages, agent, opts.Channel, opts.ChatID) } + // Process PDFs with OCR when configured + if ocrCfg := cfg.Agents.Defaults.OCR; ocrCfg != nil && ocrCfg.Command != "" { + messages = al.processPDFsInMessages(ctx, messages, ocrCfg, opts.Channel, opts.ChatID) + } + // 2b. Interview staleness nudge: if MEMORY.md hasn't been updated for // several consecutive turns, inject a reminder so the AI writes its findings. diff --git a/pkg/config/config.go b/pkg/config/config.go index 72a5aa656..f23b36f29 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -242,6 +242,23 @@ type AgentDefaults struct { TaskReminderInterval int `json:"task_reminder_interval" env:"PICOCLAW_AGENTS_DEFAULTS_TASK_REMINDER_INTERVAL"` Orchestration bool `json:"orchestration,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_ORCHESTRATION"` Routing *RoutingConfig `json:"routing,omitempty"` + OCR *OCRConfig `json:"ocr,omitempty"` +} + +// OCRConfig configures the external OCR command for PDF text extraction. +type OCRConfig struct { + Command string `json:"command"` // path to OCR binary (e.g. "/path/to/.venv/bin/yomitoku") + Args []string `json:"args,omitempty"` // static arguments (e.g. ["-f", "md", "--lite", ...]) + Env map[string]string `json:"env,omitempty"` // extra environment variables (e.g. {"HF_HOME": "/tmp/hf-home"}) + Timeout int `json:"timeout,omitempty"` // timeout in seconds (default: 600) +} + +// GetOCRTimeout returns the configured timeout or default (600s = 10min). +func (c *OCRConfig) GetOCRTimeout() int { + if c != nil && c.Timeout > 0 { + return c.Timeout + } + return 600 } const DefaultMaxMediaSize = 20 * 1024 * 1024 // 20 MB diff --git a/pkg/mediacache/pdf.go b/pkg/mediacache/pdf.go new file mode 100644 index 000000000..bb5e4acf3 --- /dev/null +++ b/pkg/mediacache/pdf.go @@ -0,0 +1,52 @@ +package mediacache + +import ( + "os" + "regexp" + "strconv" +) + +// pdfPageCountRe matches /Type /Pages ... /Count N in PDF cross-reference. +// This covers the vast majority of well-formed PDFs. +var pdfPageCountRe = regexp.MustCompile(`/Type\s*/Pages\b[^>]*/Count\s+(\d+)`) + +// PDFPageCount extracts the total page count from a PDF file by parsing +// the /Type /Pages dictionary. Returns 0 if the count cannot be determined +// (encrypted, malformed, or unusual structure). This is a best-effort +// extraction that avoids heavy PDF library dependencies. +func PDFPageCount(path string) int { + data, err := os.ReadFile(path) + if err != nil { + return 0 + } + // Search from the end of the file where the root Pages dict typically lives. + // Limit search to last 64KB for performance on large files. + searchStart := 0 + if len(data) > 64*1024 { + searchStart = len(data) - 64*1024 + } + matches := pdfPageCountRe.FindAllSubmatch(data[searchStart:], -1) + if len(matches) == 0 { + // Fallback: search entire file + matches = pdfPageCountRe.FindAllSubmatch(data, -1) + } + if len(matches) == 0 { + return 0 + } + // Use the largest /Count found (root Pages object has the total). + var maxCount int + for _, m := range matches { + if n, err := strconv.Atoi(string(m[1])); err == nil && n > maxCount { + maxCount = n + } + } + return maxCount +} + +// FormatPageCount returns pages as a string, or "?" if unknown. +func FormatPageCount(pages int) string { + if pages > 0 { + return strconv.Itoa(pages) + } + return "?" +} diff --git a/pkg/mediacache/pdf_test.go b/pkg/mediacache/pdf_test.go new file mode 100644 index 000000000..e0ab6b3be --- /dev/null +++ b/pkg/mediacache/pdf_test.go @@ -0,0 +1,81 @@ +package mediacache + +import ( + "os" + "path/filepath" + "testing" +) + +// minimalPDF is a valid PDF with 3 pages. +// This is the smallest possible multi-page PDF structure. +const minimalPDF = `%PDF-1.4 +1 0 obj <> endobj +2 0 obj <> endobj +3 0 obj <> endobj +4 0 obj <> endobj +5 0 obj <> endobj +xref +0 6 +trailer <> +startxref +0 +%%EOF` + +func TestPDFPageCount_ValidPDF(t *testing.T) { + path := writeTempFile(t, "test.pdf", minimalPDF) + count := PDFPageCount(path) + if count != 3 { + t.Errorf("PDFPageCount = %d, want 3", count) + } +} + +func TestPDFPageCount_NonExistent(t *testing.T) { + count := PDFPageCount("/nonexistent/file.pdf") + if count != 0 { + t.Errorf("PDFPageCount = %d, want 0 for missing file", count) + } +} + +func TestPDFPageCount_NotPDF(t *testing.T) { + path := writeTempFile(t, "test.txt", "hello world") + count := PDFPageCount(path) + if count != 0 { + t.Errorf("PDFPageCount = %d, want 0 for non-PDF", count) + } +} + +func TestPDFPageCount_SinglePage(t *testing.T) { + pdf := `%PDF-1.4 +1 0 obj <> endobj +2 0 obj <> endobj +3 0 obj <> endobj +xref +0 4 +trailer <> +startxref +0 +%%EOF` + path := writeTempFile(t, "single.pdf", pdf) + count := PDFPageCount(path) + if count != 1 { + t.Errorf("PDFPageCount = %d, want 1", count) + } +} + +func TestFormatPageCount(t *testing.T) { + if s := FormatPageCount(0); s != "?" { + t.Errorf("FormatPageCount(0) = %q, want %q", s, "?") + } + if s := FormatPageCount(18); s != "18" { + t.Errorf("FormatPageCount(18) = %q, want %q", s, "18") + } +} + +func writeTempFile(t *testing.T, name, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write temp file: %v", err) + } + return path +} From 4c73ae213b8088f5cc4869b38385d00dccfcd4a6 Mon Sep 17 00:00:00 2001 From: dj-oyu <68707227+dj-oyu@users.noreply.github.com> Date: Thu, 19 Mar 2026 22:42:25 +0900 Subject: [PATCH 2/8] feat: log image descriptions at INFO level for debugging Show the generated (or cached) image description in logs so users can verify what the vision model produced and diagnose issues. Co-Authored-By: Claude Opus 4.6 (1M context) --- pkg/agent/loop_media.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/agent/loop_media.go b/pkg/agent/loop_media.go index 3086b44cb..792919e42 100644 --- a/pkg/agent/loop_media.go +++ b/pkg/agent/loop_media.go @@ -262,7 +262,10 @@ func (al *AgentLoop) describeImage( hash := mediacache.HashData([]byte(dataURL)) if al.mediaCache != nil { if cached, ok := al.mediaCache.Get(hash, mediacache.TypeImageDesc); ok { - logger.DebugCF("agent", "Image description cache hit", map[string]any{"hash": hash}) + logger.InfoCF("agent", "Image description (cached)", map[string]any{ + "hash": hash, + "description": cached, + }) return cached } } @@ -317,6 +320,11 @@ func (al *AgentLoop) describeImage( desc := strings.TrimSpace(resp.Content) + logger.InfoCF("agent", "Image described", map[string]any{ + "hash": hash, + "description": desc, + }) + // Store in cache if al.mediaCache != nil { if cErr := al.mediaCache.Put(hash, mediacache.TypeImageDesc, desc); cErr != nil { From f101a977d38cf127099ea6b40176f89896015bd6 Mon Sep 17 00:00:00 2001 From: dj-oyu <68707227+dj-oyu@users.noreply.github.com> Date: Thu, 19 Mar 2026 22:56:24 +0900 Subject: [PATCH 3/8] feat: add Media tab to Research page with media cache viewer Research page now has Research/Media tab toggle. The Media tab shows all cached media processing results (image descriptions and PDF OCR) from media_cache.db with type filtering and expandable detail views. Backend: - mediacache.List(): query all entries with optional type filter - /api/media-cache: list endpoint with ?type= filter - /api/media-cache/{hash}: full content endpoint (reads MD for PDFs) Frontend: - media-cache-page.tsx: card list with type filter (All/Images/PDF) - Expandable entries: image descriptions inline, PDF with full content - research.tsx: tab toggle between Research and Media sub-pages Co-Authored-By: Claude Opus 4.6 (1M context) --- pkg/mediacache/cache.go | 41 ++++ web/backend/api/media_cache.go | 128 ++++++++++ web/backend/api/router.go | 3 + web/frontend/bun.lock | 34 ++- web/frontend/src/api/media-cache.ts | 40 +++ .../components/research/media-cache-page.tsx | 227 ++++++++++++++++++ .../src/components/research/research-page.tsx | 5 +- web/frontend/src/routes/research.tsx | 64 ++++- 8 files changed, 523 insertions(+), 19 deletions(-) create mode 100644 web/backend/api/media_cache.go create mode 100644 web/frontend/src/api/media-cache.ts create mode 100644 web/frontend/src/components/research/media-cache-page.tsx diff --git a/pkg/mediacache/cache.go b/pkg/mediacache/cache.go index d3f81ef1c..b49f06fcd 100644 --- a/pkg/mediacache/cache.go +++ b/pkg/mediacache/cache.go @@ -130,6 +130,47 @@ func (c *Cache) PutEntry(hash, entryType string, entry Entry) error { return err } +// ListEntry represents a full row from the media_cache table. +type ListEntry struct { + Hash string + Type string + Result string + FilePath string + Pages int + CreatedAt string + AccessedAt string +} + +// List returns all cache entries, optionally filtered by type. +// Pass empty string to list all types. Ordered by accessed_at desc. +func (c *Cache) List(entryType string) ([]ListEntry, error) { + var rows *sql.Rows + var err error + if entryType != "" { + rows, err = c.db.Query( + `SELECT hash, type, result, file_path, pages, created_at, accessed_at + FROM media_cache WHERE type = ? ORDER BY accessed_at DESC`, entryType) + } else { + rows, err = c.db.Query( + `SELECT hash, type, result, file_path, pages, created_at, accessed_at + FROM media_cache ORDER BY accessed_at DESC`) + } + if err != nil { + return nil, err + } + defer rows.Close() + + var entries []ListEntry + for rows.Next() { + var e ListEntry + if err := rows.Scan(&e.Hash, &e.Type, &e.Result, &e.FilePath, &e.Pages, &e.CreatedAt, &e.AccessedAt); err != nil { + return entries, err + } + entries = append(entries, e) + } + return entries, rows.Err() +} + // Prune removes entries not accessed within the given duration. // Returns the number of entries removed. func (c *Cache) Prune(ttl time.Duration) (int64, error) { diff --git a/web/backend/api/media_cache.go b/web/backend/api/media_cache.go new file mode 100644 index 000000000..7d9785db1 --- /dev/null +++ b/web/backend/api/media_cache.go @@ -0,0 +1,128 @@ +package api + +import ( + "encoding/json" + "net/http" + "os" + "path/filepath" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/mediacache" +) + +func (h *Handler) registerMediaCacheRoutes(mux *http.ServeMux) { + mux.HandleFunc("/api/media-cache", h.handleMediaCache) + mux.HandleFunc("/api/media-cache/", h.handleMediaCacheContent) +} + +type mediaCacheEntryJSON struct { + Hash string `json:"hash"` + Type string `json:"type"` + Result string `json:"result"` + FilePath string `json:"file_path,omitempty"` + Pages int `json:"pages,omitempty"` + CreatedAt string `json:"created_at"` + AccessedAt string `json:"accessed_at"` +} + +func (h *Handler) openMediaCache() (*mediacache.Cache, error) { + cfg, err := config.LoadConfig(h.configPath) + if err != nil { + return nil, err + } + ws := cfg.WorkspacePath() + return mediacache.Open(filepath.Join(ws, "media_cache.db")) +} + +// handleMediaCache lists all media cache entries. +func (h *Handler) handleMediaCache(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed) + return + } + + mc, err := h.openMediaCache() + if err != nil { + http.Error(w, `{"error":"media cache not available"}`, http.StatusServiceUnavailable) + return + } + defer mc.Close() + + typeFilter := r.URL.Query().Get("type") + entries, err := mc.List(typeFilter) + if err != nil { + http.Error(w, `{"error":"failed to list cache entries"}`, http.StatusInternalServerError) + return + } + + result := make([]mediaCacheEntryJSON, 0, len(entries)) + for _, e := range entries { + result = append(result, mediaCacheEntryJSON{ + Hash: e.Hash, + Type: e.Type, + Result: e.Result, + FilePath: e.FilePath, + Pages: e.Pages, + CreatedAt: e.CreatedAt, + AccessedAt: e.AccessedAt, + }) + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(result) +} + +// handleMediaCacheContent serves the full file content for a PDF OCR entry. +// GET /api/media-cache/{hash} +func (h *Handler) handleMediaCacheContent(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed) + return + } + + hash := filepath.Base(r.URL.Path) + if hash == "" || hash == "media-cache" { + http.Error(w, `{"error":"hash required"}`, http.StatusBadRequest) + return + } + + mc, err := h.openMediaCache() + if err != nil { + http.Error(w, `{"error":"media cache not available"}`, http.StatusServiceUnavailable) + return + } + defer mc.Close() + + entry, ok := mc.GetEntry(hash, mediacache.TypePDFOCR) + if !ok { + // Try image_desc + result, ok := mc.Get(hash, mediacache.TypeImageDesc) + if !ok { + http.Error(w, `{"error":"not found"}`, http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]string{ + "hash": hash, + "type": mediacache.TypeImageDesc, + "content": result, + }) + return + } + + // Read the full markdown file + content, err := os.ReadFile(entry.FilePath) + if err != nil { + http.Error(w, `{"error":"file not found"}`, http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]any{ + "hash": hash, + "type": mediacache.TypePDFOCR, + "content": string(content), + "file_path": entry.FilePath, + "pages": entry.Pages, + }) +} diff --git a/web/backend/api/router.go b/web/backend/api/router.go index d55269ed8..8de954f1a 100644 --- a/web/backend/api/router.go +++ b/web/backend/api/router.go @@ -72,6 +72,9 @@ func (h *Handler) RegisterRoutes(mux *http.ServeMux) { // Research tasks (proxy to gateway) h.registerResearchRoutes(mux) + + // Media cache (image descriptions, PDF OCR) + h.registerMediaCacheRoutes(mux) } // Shutdown gracefully shuts down the handler, stopping the gateway if it was started by this handler. diff --git a/web/frontend/bun.lock b/web/frontend/bun.lock index dea77e783..29d95e6ae 100644 --- a/web/frontend/bun.lock +++ b/web/frontend/bun.lock @@ -9,18 +9,18 @@ "@tabler/icons-react": "^3.38.0", "@tailwindcss/vite": "^4.2.1", "@tanstack/react-query": "^5.90.21", - "@tanstack/react-router": "^1.163.3", + "@tanstack/react-router": "^1.167.0", "@tanstack/react-router-devtools": "^1.163.3", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", - "dayjs": "^1.11.19", + "dayjs": "^1.11.20", "i18next": "^25.8.14", "i18next-browser-languagedetector": "^8.2.1", - "jotai": "^2.18.0", + "jotai": "^2.18.1", "radix-ui": "^1.4.3", "react": "^19.2.0", "react-dom": "^19.2.0", - "react-i18next": "^16.5.4", + "react-i18next": "^16.5.8", "react-markdown": "^10.1.0", "react-textarea-autosize": "^8.5.9", "remark-gfm": "^4.0.1", @@ -32,7 +32,7 @@ "wrap-ansi": "^10.0.0", }, "devDependencies": { - "@eslint/js": "^9.39.1", + "@eslint/js": "^9.39.3", "@tailwindcss/typography": "^0.5.19", "@tanstack/router-plugin": "^1.164.0", "@trivago/prettier-plugin-sort-imports": "^6.0.2", @@ -40,8 +40,8 @@ "@types/react": "^19.2.7", "@types/react-dom": "^19.2.3", "@typescript-eslint/eslint-plugin": "^8.56.1", - "@vitejs/plugin-react": "^5.1.1", - "eslint": "^9.39.1", + "@vitejs/plugin-react": "^5.2.0", + "eslint": "^9.39.3", "eslint-config-prettier": "^10.1.8", "eslint-plugin-react-hooks": "^7.0.1", "eslint-plugin-react-refresh": "^0.4.24", @@ -467,13 +467,13 @@ "@tailwindcss/vite": ["@tailwindcss/vite@4.2.1", "", { "dependencies": { "@tailwindcss/node": "4.2.1", "@tailwindcss/oxide": "4.2.1", "tailwindcss": "4.2.1" }, "peerDependencies": { "vite": "7.3.1" } }, "sha512-TBf2sJjYeb28jD2U/OhwdW0bbOsxkWPwQ7SrqGf9sVcoYwZj7rkXljroBO9wKBut9XnmQLXanuDUeqQK0lGg/w=="], - "@tanstack/history": ["@tanstack/history@1.161.4", "", {}, "sha512-Kp/WSt411ZWYvgXy6uiv5RmhHrz9cAml05AQPrtdAp7eUqvIDbMGPnML25OKbzR3RJ1q4wgENxDTvlGPa9+Mww=="], + "@tanstack/history": ["@tanstack/history@1.161.6", "", {}, "sha512-NaOGLRrddszbQj9upGat6HG/4TKvXLvu+osAIgfxPYA+eIvYKv8GKDJOrY2D3/U9MRnKfMWD7bU4jeD4xmqyIg=="], "@tanstack/query-core": ["@tanstack/query-core@5.90.20", "", {}, "sha512-OMD2HLpNouXEfZJWcKeVKUgQ5n+n3A2JFmBaScpNDUqSrQSjiveC7dKMe53uJUg1nDG16ttFPz2xfilz6i2uVg=="], "@tanstack/react-query": ["@tanstack/react-query@5.90.21", "", { "dependencies": { "@tanstack/query-core": "5.90.20" }, "peerDependencies": { "react": "19.2.4" } }, "sha512-0Lu6y5t+tvlTJMTO7oh5NSpJfpg/5D41LlThfepTixPYkJ0sE2Jj0m0f6yYqujBwIXlId87e234+MxG3D3g7kg=="], - "@tanstack/react-router": ["@tanstack/react-router@1.163.3", "", { "dependencies": { "@tanstack/history": "1.161.4", "@tanstack/react-store": "0.9.1", "@tanstack/router-core": "1.163.3", "isbot": "5.1.35", "tiny-invariant": "1.3.3", "tiny-warning": "1.0.3" }, "peerDependencies": { "react": "19.2.4", "react-dom": "19.2.4" } }, "sha512-hheBbFVb+PbxtrWp8iy6+TTRTbhx3Pn6hKo8Tv/sWlG89ZMcD1xpQWzx8ukHN9K8YWbh5rdzt4kv6u8X4kB28Q=="], + "@tanstack/react-router": ["@tanstack/react-router@1.167.5", "", { "dependencies": { "@tanstack/history": "1.161.6", "@tanstack/react-store": "^0.9.1", "@tanstack/router-core": "1.167.5", "isbot": "^5.1.22", "tiny-invariant": "^1.3.3", "tiny-warning": "^1.0.3" }, "peerDependencies": { "react": ">=18.0.0 || >=19.0.0", "react-dom": ">=18.0.0 || >=19.0.0" } }, "sha512-s1nP6l/7BYZfSwhoNbB7/rUmZ07q/AvkmhBoiDQl3tgy5dpb9Q1qjtIapYdvCOrao1aA/QCaWqxcbGc2Ct1bvQ=="], "@tanstack/react-router-devtools": ["@tanstack/react-router-devtools@1.163.3", "", { "dependencies": { "@tanstack/router-devtools-core": "1.163.3" }, "optionalDependencies": { "@tanstack/router-core": "1.163.3" }, "peerDependencies": { "@tanstack/react-router": "1.163.3", "react": "19.2.4", "react-dom": "19.2.4" } }, "sha512-42VMkV/2Z8ro7xzblPBRNZIEmCNXMzm2jD68G52p2qhjXm38wGpg46qneAESN9FtTQeVWk5aSXs47/jt7lkzmw=="], @@ -553,7 +553,7 @@ "@ungap/structured-clone": ["@ungap/structured-clone@1.3.0", "", {}, "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g=="], - "@vitejs/plugin-react": ["@vitejs/plugin-react@5.1.4", "", { "dependencies": { "@babel/core": "7.29.0", "@babel/plugin-transform-react-jsx-self": "7.27.1", "@babel/plugin-transform-react-jsx-source": "7.27.1", "@rolldown/pluginutils": "1.0.0-rc.3", "@types/babel__core": "7.20.5", "react-refresh": "0.18.0" }, "peerDependencies": { "vite": "7.3.1" } }, "sha512-VIcFLdRi/VYRU8OL/puL7QXMYafHmqOnwTZY50U1JPlCNj30PxCMx65c494b1K9be9hX83KVt0+gTEwTWLqToA=="], + "@vitejs/plugin-react": ["@vitejs/plugin-react@5.2.0", "", { "dependencies": { "@babel/core": "^7.29.0", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", "@rolldown/pluginutils": "1.0.0-rc.3", "@types/babel__core": "^7.20.5", "react-refresh": "^0.18.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-YmKkfhOAi3wsB1PhJq5Scj3GXMn3WvtQ/JC0xoopuHoXSdmtdStOpFrYaT1kie2YgFBcIe64ROzMYRjCrYOdYw=="], "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "3.0.2", "negotiator": "1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], @@ -673,7 +673,7 @@ "data-uri-to-buffer": ["data-uri-to-buffer@4.0.1", "", {}, "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A=="], - "dayjs": ["dayjs@1.11.19", "", {}, "sha512-t5EcLVS6QPBNqM2z8fakk/NKel+Xzshgt8FFKAn+qwlD1pzZWxh0nVCrvFK7ZDb6XucZeF9z8C7CBWTRIVApAw=="], + "dayjs": ["dayjs@1.11.20", "", {}, "sha512-YbwwqR/uYpeoP4pu043q+LTDLFBLApUP6VxRihdfNTqu4ubqMlGDLd6ErXhEgsyvY0K6nCs7nggYumAN+9uEuQ=="], "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], @@ -955,7 +955,7 @@ "jose": ["jose@6.1.3", "", {}, "sha512-0TpaTfihd4QMNwrz/ob2Bp7X04yuxJkjRGi4aKmOqwhov54i6u79oCv7T+C7lo70MKH6BesI3vscD1yb/yzKXQ=="], - "jotai": ["jotai@2.18.0", "", { "optionalDependencies": { "@babel/core": "7.29.0", "@babel/template": "7.28.6", "@types/react": "19.2.14", "react": "19.2.4" } }, "sha512-XI38kGWAvtxAZ+cwHcTgJsd+kJOJGf3OfL4XYaXWZMZ7IIY8e53abpIHvtVn1eAgJ5dlgwlGFnP4psrZ/vZbtA=="], + "jotai": ["jotai@2.18.1", "", { "peerDependencies": { "@babel/core": ">=7.0.0", "@babel/template": ">=7.0.0", "@types/react": ">=17.0.0", "react": ">=17.0.0" }, "optionalPeers": ["@babel/core", "@babel/template", "@types/react", "react"] }, "sha512-e0NOzK+yRFwHo7DOp0DS0Ycq74KMEAObDWFGmfEL28PD9nLqBTt3/Ug7jf9ca72x0gC9LQZG9zH+0ISICmy3iA=="], "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], @@ -1249,7 +1249,7 @@ "react-dom": ["react-dom@19.2.4", "", { "dependencies": { "scheduler": "0.27.0" }, "peerDependencies": { "react": "19.2.4" } }, "sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ=="], - "react-i18next": ["react-i18next@16.5.4", "", { "dependencies": { "@babel/runtime": "7.28.6", "html-parse-stringify": "3.0.1", "use-sync-external-store": "1.6.0" }, "optionalDependencies": { "react-dom": "19.2.4", "typescript": "5.9.3" }, "peerDependencies": { "i18next": "25.8.14", "react": "19.2.4" } }, "sha512-6yj+dcfMncEC21QPhOTsW8mOSO+pzFmT6uvU7XXdvM/Cp38zJkmTeMeKmTrmCMD5ToT79FmiE/mRWiYWcJYW4g=="], + "react-i18next": ["react-i18next@16.5.8", "", { "dependencies": { "@babel/runtime": "^7.28.4", "html-parse-stringify": "^3.0.1", "use-sync-external-store": "^1.6.0" }, "peerDependencies": { "i18next": ">= 25.6.2", "react": ">= 16.8.0", "typescript": "^5" }, "optionalPeers": ["typescript"] }, "sha512-2ABeHHlakxVY+LSirD+OiERxFL6+zip0PaHo979bgwzeHg27Sqc82xxXWIrSFmfWX0ZkrvXMHwhsi/NGUf5VQg=="], "react-markdown": ["react-markdown@10.1.0", "", { "dependencies": { "@types/hast": "3.0.4", "@types/mdast": "4.0.4", "devlop": "1.1.0", "hast-util-to-jsx-runtime": "2.3.6", "html-url-attributes": "3.0.1", "mdast-util-to-hast": "13.2.1", "remark-parse": "11.0.0", "remark-rehype": "11.1.2", "unified": "11.0.5", "unist-util-visit": "5.1.0", "vfile": "6.0.3" }, "peerDependencies": { "@types/react": "19.2.14", "react": "19.2.4" } }, "sha512-qKxVopLT/TyA6BX3Ue5NwabOsAzm0Q7kAPwq6L+wWDwisYs7R8vZ0nRXqq6rkueboxpkjvLGU9fWifiX/ZZFxQ=="], @@ -1529,6 +1529,12 @@ "@modelcontextprotocol/sdk/ajv": ["ajv@8.18.0", "", { "dependencies": { "fast-deep-equal": "3.1.3", "fast-uri": "3.1.0", "json-schema-traverse": "1.0.0", "require-from-string": "2.0.2" } }, "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A=="], + "@tanstack/react-router/@tanstack/router-core": ["@tanstack/router-core@1.167.5", "", { "dependencies": { "@tanstack/history": "1.161.6", "@tanstack/store": "^0.9.1", "cookie-es": "^2.0.0", "seroval": "^1.4.2", "seroval-plugins": "^1.4.2", "tiny-invariant": "^1.3.3", "tiny-warning": "^1.0.3" }, "bin": { "intent": "bin/intent.js" } }, "sha512-8fRgJ0zNJf77R4grCaJQ5Imatjyc4YT5v8rlsPkYYYeUlcFNLbuFRhLlAMdND9gRUMznpnbRDXngpTPgx2K7HQ=="], + + "@tanstack/router-core/@tanstack/history": ["@tanstack/history@1.161.4", "", {}, "sha512-Kp/WSt411ZWYvgXy6uiv5RmhHrz9cAml05AQPrtdAp7eUqvIDbMGPnML25OKbzR3RJ1q4wgENxDTvlGPa9+Mww=="], + + "@tanstack/router-plugin/@tanstack/react-router": ["@tanstack/react-router@1.163.3", "", { "dependencies": { "@tanstack/history": "1.161.4", "@tanstack/react-store": "0.9.1", "@tanstack/router-core": "1.163.3", "isbot": "5.1.35", "tiny-invariant": "1.3.3", "tiny-warning": "1.0.3" }, "peerDependencies": { "react": "19.2.4", "react-dom": "19.2.4" } }, "sha512-hheBbFVb+PbxtrWp8iy6+TTRTbhx3Pn6hKo8Tv/sWlG89ZMcD1xpQWzx8ukHN9K8YWbh5rdzt4kv6u8X4kB28Q=="], + "@ts-morph/common/minimatch": ["minimatch@10.2.4", "", { "dependencies": { "brace-expansion": "5.0.4" } }, "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg=="], "@typescript-eslint/typescript-estree/minimatch": ["minimatch@10.2.4", "", { "dependencies": { "brace-expansion": "5.0.4" } }, "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg=="], @@ -1619,6 +1625,8 @@ "@modelcontextprotocol/sdk/ajv/json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="], + "@tanstack/router-plugin/@tanstack/react-router/@tanstack/history": ["@tanstack/history@1.161.4", "", {}, "sha512-Kp/WSt411ZWYvgXy6uiv5RmhHrz9cAml05AQPrtdAp7eUqvIDbMGPnML25OKbzR3RJ1q4wgENxDTvlGPa9+Mww=="], + "@ts-morph/common/minimatch/brace-expansion": ["brace-expansion@5.0.4", "", { "dependencies": { "balanced-match": "4.0.4" } }, "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg=="], "@typescript-eslint/typescript-estree/minimatch/brace-expansion": ["brace-expansion@5.0.4", "", { "dependencies": { "balanced-match": "4.0.4" } }, "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg=="], diff --git a/web/frontend/src/api/media-cache.ts b/web/frontend/src/api/media-cache.ts new file mode 100644 index 000000000..74d0a3fb9 --- /dev/null +++ b/web/frontend/src/api/media-cache.ts @@ -0,0 +1,40 @@ +export interface MediaCacheEntry { + hash: string + type: "image_desc" | "pdf_ocr" + result: string + file_path?: string + pages?: number + created_at: string + accessed_at: string +} + +export interface MediaCacheContent { + hash: string + type: string + content: string + file_path?: string + pages?: number +} + +async function request(path: string): Promise { + const res = await fetch(path) + if (!res.ok) { + throw new Error(`API error: ${res.status}`) + } + return res.json() as Promise +} + +export async function getMediaCacheEntries( + type?: string, +): Promise { + const params = type ? `?type=${encodeURIComponent(type)}` : "" + return request(`/api/media-cache${params}`) +} + +export async function getMediaCacheContent( + hash: string, +): Promise { + return request( + `/api/media-cache/${encodeURIComponent(hash)}`, + ) +} diff --git a/web/frontend/src/components/research/media-cache-page.tsx b/web/frontend/src/components/research/media-cache-page.tsx new file mode 100644 index 000000000..1f434dc65 --- /dev/null +++ b/web/frontend/src/components/research/media-cache-page.tsx @@ -0,0 +1,227 @@ +import { + IconFileText, + IconPhoto, +} from "@tabler/icons-react" +import { useQuery } from "@tanstack/react-query" +import * as React from "react" + +import { + type MediaCacheContent, + type MediaCacheEntry, + getMediaCacheContent, + getMediaCacheEntries, +} from "@/api/media-cache" +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card" +import { Button } from "@/components/ui/button" +import { cn } from "@/lib/utils" + +export function MediaCachePage() { + const [typeFilter, setTypeFilter] = React.useState("") + const [expandedHash, setExpandedHash] = React.useState(null) + + const { data: entries, isLoading, error } = useQuery({ + queryKey: ["media-cache", typeFilter], + queryFn: () => getMediaCacheEntries(typeFilter || undefined), + refetchInterval: 30000, + }) + + return ( +
+
+ {/* Type filter */} +
+ setTypeFilter("")} + > + All + + setTypeFilter("image_desc")} + > + + Images + + setTypeFilter("pdf_ocr")} + > + + PDF + +
+ + {isLoading ? ( +
Loading...
+ ) : error ? ( +
+ Failed to load media cache. +
+ ) : !entries?.length ? ( + + + No cached media yet. Send an image or PDF to get started. + + + ) : ( +
+ {entries.map((entry) => ( + + setExpandedHash( + expandedHash === entry.hash ? null : entry.hash, + ) + } + /> + ))} +
+ )} +
+
+ ) +} + +function FilterButton({ + active, + onClick, + children, +}: { + active: boolean + onClick: () => void + children: React.ReactNode +}) { + return ( + + ) +} + +function MediaEntry({ + entry, + expanded, + onToggle, +}: { + entry: MediaCacheEntry + expanded: boolean + onToggle: () => void +}) { + const isImage = entry.type === "image_desc" + const Icon = isImage ? IconPhoto : IconFileText + const typeLabel = isImage ? "Image" : "PDF" + const typeColor = isImage + ? "text-blue-600 bg-blue-50" + : "text-orange-600 bg-orange-50" + + const accessed = new Date(entry.accessed_at) + const timeStr = accessed.toLocaleString() + + return ( + + +
+
+ + + {entry.hash} + + + {entry.result} + +
+
+ + {typeLabel} + {entry.pages ? ` (${entry.pages}p)` : ""} + + {timeStr} +
+
+ + {expanded && ( + + + + )} + + ) +} + +function ExpandedContent({ entry }: { entry: MediaCacheEntry }) { + const isPDF = entry.type === "pdf_ocr" + + const { data, isLoading } = useQuery({ + queryKey: ["media-cache-content", entry.hash], + queryFn: () => getMediaCacheContent(entry.hash), + enabled: isPDF, // only fetch full content for PDFs + }) + + if (!isPDF) { + // Image description: show full result inline + return ( +
+
+ Description +
+
+ {entry.result} +
+
+ ) + } + + // PDF OCR: show preview + full content on demand + return ( +
+
+
Preview
+
+ {entry.result} +
+
+ {entry.file_path && ( +
+ + {entry.file_path} +
+ )} + {isLoading ? ( +
+ Loading full content... +
+ ) : data?.content ? ( +
+
+ Full OCR Content +
+
+ {data.content} +
+
+ ) : null} +
+ ) +} diff --git a/web/frontend/src/components/research/research-page.tsx b/web/frontend/src/components/research/research-page.tsx index b23d31545..6456b615e 100644 --- a/web/frontend/src/components/research/research-page.tsx +++ b/web/frontend/src/components/research/research-page.tsx @@ -17,7 +17,6 @@ import { createResearchTask, getResearchTasks, } from "@/api/research" -import { PageHeader } from "@/components/page-header" import { Button } from "@/components/ui/button" import { Card, @@ -83,7 +82,7 @@ export function ResearchPage() { return (
- +
diff --git a/web/frontend/src/routes/research.tsx b/web/frontend/src/routes/research.tsx index 8405b21be..e848473f5 100644 --- a/web/frontend/src/routes/research.tsx +++ b/web/frontend/src/routes/research.tsx @@ -1,10 +1,19 @@ +import { + IconDatabase, + IconFileSearch, +} from "@tabler/icons-react" import { Outlet, createFileRoute, useRouterState, } from "@tanstack/react-router" +import * as React from "react" +import { MediaCachePage } from "@/components/research/media-cache-page" import { ResearchPage } from "@/components/research/research-page" +import { PageHeader } from "@/components/page-header" +import { Button } from "@/components/ui/button" +import { cn } from "@/lib/utils" export const Route = createFileRoute("/research")({ component: ResearchRouteLayout, @@ -14,10 +23,59 @@ function ResearchRouteLayout() { const pathname = useRouterState({ select: (state) => state.location.pathname, }) + const [tab, setTab] = React.useState<"research" | "media">("research") - if (pathname === "/research") { - return + // If on a detail sub-route, show Outlet + if (pathname !== "/research") { + return } - return + return ( +
+ +
+ setTab("research")} + > + + Research + + setTab("media")} + > + + Media + +
+ + + {tab === "research" ? : } +
+ ) +} + +function TabButton({ + active, + onClick, + children, +}: { + active: boolean + onClick: () => void + children: React.ReactNode +}) { + return ( + + ) } From 51f08e7f50d328da9df78f44e936662222ffef52 Mon Sep 17 00:00:00 2001 From: dj-oyu <68707227+dj-oyu@users.noreply.github.com> Date: Thu, 19 Mar 2026 23:04:16 +0900 Subject: [PATCH 4/8] fix: resolve lint issues (govet shadow, golines, gci formatting) Co-Authored-By: Claude Opus 4.6 (1M context) --- pkg/agent/loop_media.go | 8 ++++---- pkg/config/config.go | 8 ++++---- pkg/mediacache/cache.go | 5 ++++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/pkg/agent/loop_media.go b/pkg/agent/loop_media.go index 792919e42..28f512a17 100644 --- a/pkg/agent/loop_media.go +++ b/pkg/agent/loop_media.go @@ -544,8 +544,8 @@ func (al *AgentLoop) ocrPDF( "cmd": ocrCfg.Command, }) - if err := cmd.Start(); err != nil { - logger.WarnCF("agent", "Failed to start OCR command", map[string]any{"error": err.Error()}) + if startErr := cmd.Start(); startErr != nil { + logger.WarnCF("agent", "Failed to start OCR command", map[string]any{"error": startErr.Error()}) return fmt.Sprintf("[file:%s]", pdfPath) } @@ -560,10 +560,10 @@ func (al *AgentLoop) ocrPDF( } } - if err := cmd.Wait(); err != nil { + if waitErr := cmd.Wait(); waitErr != nil { logger.WarnCF("agent", "OCR command failed", map[string]any{ "path": pdfPath, - "error": err.Error(), + "error": waitErr.Error(), }) return fmt.Sprintf("[file:%s]", pdfPath) } diff --git a/pkg/config/config.go b/pkg/config/config.go index f23b36f29..776e472b7 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -247,10 +247,10 @@ type AgentDefaults struct { // OCRConfig configures the external OCR command for PDF text extraction. type OCRConfig struct { - Command string `json:"command"` // path to OCR binary (e.g. "/path/to/.venv/bin/yomitoku") - Args []string `json:"args,omitempty"` // static arguments (e.g. ["-f", "md", "--lite", ...]) - Env map[string]string `json:"env,omitempty"` // extra environment variables (e.g. {"HF_HOME": "/tmp/hf-home"}) - Timeout int `json:"timeout,omitempty"` // timeout in seconds (default: 600) + Command string `json:"command"` + Args []string `json:"args,omitempty"` + Env map[string]string `json:"env,omitempty"` + Timeout int `json:"timeout,omitempty"` } // GetOCRTimeout returns the configured timeout or default (600s = 10min). diff --git a/pkg/mediacache/cache.go b/pkg/mediacache/cache.go index b49f06fcd..91439aedb 100644 --- a/pkg/mediacache/cache.go +++ b/pkg/mediacache/cache.go @@ -163,7 +163,10 @@ func (c *Cache) List(entryType string) ([]ListEntry, error) { var entries []ListEntry for rows.Next() { var e ListEntry - if err := rows.Scan(&e.Hash, &e.Type, &e.Result, &e.FilePath, &e.Pages, &e.CreatedAt, &e.AccessedAt); err != nil { + if err := rows.Scan( + &e.Hash, &e.Type, &e.Result, &e.FilePath, + &e.Pages, &e.CreatedAt, &e.AccessedAt, + ); err != nil { return entries, err } entries = append(entries, e) From b4b6c110cc6cf1b58def753746deb8a6fa920568 Mon Sep 17 00:00:00 2001 From: dj-oyu <68707227+dj-oyu@users.noreply.github.com> Date: Thu, 19 Mar 2026 23:09:53 +0900 Subject: [PATCH 5/8] feat: clean up yomitoku page images (_pN.jpg) after OCR yomitoku always generates per-page JPEG files that cannot be suppressed via CLI options. Remove these after successful OCR while preserving the markdown output and figures/ directory (referenced by markdown). Co-Authored-By: Claude Opus 4.6 (1M context) --- pkg/agent/loop_media.go | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/pkg/agent/loop_media.go b/pkg/agent/loop_media.go index 28f512a17..6474df3d0 100644 --- a/pkg/agent/loop_media.go +++ b/pkg/agent/loop_media.go @@ -568,6 +568,11 @@ func (al *AgentLoop) ocrPDF( return fmt.Sprintf("[file:%s]", pdfPath) } + // Clean up page images (_pN.jpg) generated by yomitoku. + // These are always created and cannot be suppressed via CLI options. + // Keep: .md files, figures/ directory (referenced by markdown output). + cleanupOCRPageImages(outputDir, pdfPath) + // Find the output markdown file mdPath := findOCROutput(outputDir, pdfPath) if mdPath == "" { @@ -669,3 +674,39 @@ func ocrEnvSlice(env map[string]string) []string { } return result } + +// cleanupOCRPageImages removes _pN.jpg files generated by yomitoku. +// These per-page images are always created by the CLI and cannot be suppressed. +// Only top-level _pN.jpg files matching the PDF basename are removed; +// the figures/ subdirectory and .md files are preserved. +func cleanupOCRPageImages(outputDir, pdfPath string) { + base := strings.TrimSuffix(filepath.Base(pdfPath), filepath.Ext(pdfPath)) + entries, err := os.ReadDir(outputDir) + if err != nil { + return + } + + removed := 0 + for _, e := range entries { + if e.IsDir() { + continue + } + name := e.Name() + // Match pattern: __pN.jpg + if !strings.HasSuffix(strings.ToLower(name), ".jpg") { + continue + } + if !strings.Contains(name, base+"_p") { + continue + } + if rmErr := os.Remove(filepath.Join(outputDir, name)); rmErr == nil { + removed++ + } + } + if removed > 0 { + logger.DebugCF("agent", "Cleaned up OCR page images", map[string]any{ + "dir": outputDir, + "removed": removed, + }) + } +} From dad372eb30e241904be4bcd3b45fee1889e299e1 Mon Sep 17 00:00:00 2001 From: dj-oyu <68707227+dj-oyu@users.noreply.github.com> Date: Thu, 19 Mar 2026 23:15:05 +0900 Subject: [PATCH 6/8] feat: figure mode toggle via message keywords + PDF hint message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When sharing a PDF, include keywords like "figures" or "図版" in the message to enable --figure --figure_letter for in-figure text extraction. Cache keys include the figure mode so both variants are stored separately. A one-time hint message is sent to chat when PDF OCR starts, explaining the figure keyword option. Keywords: figure, figures, with images, 図版, 図付き, 画像付き, 図も Co-Authored-By: Claude Opus 4.6 (1M context) --- pkg/agent/loop_media.go | 66 ++++++++++++++++++++++++++++++------ pkg/agent/loop_media_test.go | 23 +++++++++++-- 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/pkg/agent/loop_media.go b/pkg/agent/loop_media.go index 6474df3d0..c2139b6c0 100644 --- a/pkg/agent/loop_media.go +++ b/pkg/agent/loop_media.go @@ -419,6 +419,27 @@ func injectImageDescriptions(content string, descriptions []string) string { // in the media cache for PDF OCR results. const maxPreviewRunes = 500 +// figureKeywords triggers --figure --figure_letter when found in the message. +var figureKeywords = []string{ + "figure", "figures", "with images", + "図版", "図付き", "画像付き", "図も", +} + +// wantFigures returns true if the message content contains a figure keyword. +func wantFigures(content string) bool { + lower := strings.ToLower(content) + for _, kw := range figureKeywords { + if strings.Contains(lower, kw) { + return true + } + } + return false +} + +// pdfHintMessage is sent once when a PDF is first processed to explain options. +const pdfHintMessage = "PDF OCR in progress. " + + "Tip: include \"figures\" or \"図版\" in your message to extract images and in-figure text." + // processPDFsInMessages finds [file:/path.pdf] tags in messages and replaces // them with [document: preview... (full: /path/to.md, N pages)] tags after // running OCR. A braille spinner with page progress is shown during processing. @@ -433,7 +454,10 @@ func (al *AgentLoop) processPDFsInMessages( if !strings.Contains(m.Content, "[file:") { continue } - result[i].Content = al.replacePDFTags(ctx, m.Content, ocrCfg, channel, chatID) + withFigures := wantFigures(m.Content) + result[i].Content = al.replacePDFTags( + ctx, m.Content, ocrCfg, channel, chatID, withFigures, + ) } return result @@ -445,7 +469,7 @@ const pdfTagPrefix = "[file:" // replacePDFTags finds [file:*.pdf] tags and replaces them with OCR results. func (al *AgentLoop) replacePDFTags( ctx context.Context, content string, ocrCfg *config.OCRConfig, - channel, chatID string, + channel, chatID string, withFigures bool, ) string { var out strings.Builder rest := content @@ -469,7 +493,7 @@ func (al *AgentLoop) replacePDFTags( out.WriteString(rest[:idx]) if strings.HasSuffix(strings.ToLower(path), ".pdf") { - out.WriteString(al.ocrPDF(ctx, path, ocrCfg, channel, chatID)) + out.WriteString(al.ocrPDF(ctx, path, ocrCfg, channel, chatID, withFigures)) } else { out.WriteString(tag) } @@ -482,17 +506,23 @@ func (al *AgentLoop) replacePDFTags( // ocrPDF runs OCR on a PDF file and returns a document tag with preview. // Uses the media cache to avoid redundant OCR runs. +// When withFigures is true, --figure and --figure_letter flags are added. func (al *AgentLoop) ocrPDF( ctx context.Context, pdfPath string, ocrCfg *config.OCRConfig, - channel, chatID string, + channel, chatID string, withFigures bool, ) string { - // Hash the file content for cache lookup + // Hash the file content for cache lookup. + // Include figure mode in the hash so both variants are cached separately. pdfData, err := os.ReadFile(pdfPath) if err != nil { logger.WarnCF("agent", "Failed to read PDF", map[string]any{"path": pdfPath, "error": err.Error()}) return fmt.Sprintf("[file:%s]", pdfPath) } - hash := mediacache.HashData(pdfData) + hashInput := pdfData + if withFigures { + hashInput = append(hashInput, []byte(":figures")...) + } + hash := mediacache.HashData(hashInput) // Check cache if al.mediaCache != nil { @@ -506,9 +536,22 @@ func (al *AgentLoop) ocrPDF( totalPages := mediacache.PDFPageCount(pdfPath) totalStr := mediacache.FormatPageCount(totalPages) - // Start progress indicator + // Send hint message and start progress indicator + if al.bus != nil && channel != "" && chatID != "" { + _ = al.bus.PublishOutbound(ctx, bus.OutboundMessage{ + Channel: channel, + ChatID: chatID, + Content: pdfHintMessage, + SkipPlaceholder: true, + }) + } + + modeLabel := "Processing PDF" + if withFigures { + modeLabel = "Processing PDF (with figures)" + } indicator := al.processingIndicator(ctx, channel, chatID, - fmt.Sprintf("Processing PDF (0/%s)...", totalStr)) + fmt.Sprintf("%s (0/%s)...", modeLabel, totalStr)) defer indicator.Stop() // Determine output directory for OCR results @@ -520,8 +563,11 @@ func (al *AgentLoop) ocrPDF( cmdCtx, cmdCancel := context.WithTimeout(ctx, timeout) defer cmdCancel() - args := make([]string, 0, len(ocrCfg.Args)+4) + args := make([]string, 0, len(ocrCfg.Args)+6) args = append(args, ocrCfg.Args...) + if withFigures { + args = append(args, "--figure", "--figure_letter") + } args = append(args, pdfPath, "-o", outputDir) cmd := exec.CommandContext(cmdCtx, ocrCfg.Command, args...) @@ -556,7 +602,7 @@ func (al *AgentLoop) ocrPDF( line := scanner.Text() if strings.Contains(line, "TextDetector __call__") { page++ - indicator.UpdateLabel(fmt.Sprintf("Processing PDF (%d/%s)...", page, totalStr)) + indicator.UpdateLabel(fmt.Sprintf("%s (%d/%s)...", modeLabel, page, totalStr)) } } diff --git a/pkg/agent/loop_media_test.go b/pkg/agent/loop_media_test.go index e21e89b3b..4b2456659 100644 --- a/pkg/agent/loop_media_test.go +++ b/pkg/agent/loop_media_test.go @@ -151,7 +151,7 @@ func TestFormatDocumentTag_UnknownPages(t *testing.T) { func TestReplacePDFTags_NoPDF(t *testing.T) { al := &AgentLoop{} content := "Check this out [file:/path/to/audio.mp3]" - result := al.replacePDFTags(t.Context(), content, &config.OCRConfig{Command: "echo"}, "", "") + result := al.replacePDFTags(t.Context(), content, &config.OCRConfig{Command: "echo"}, "", "", false) if result != content { t.Errorf("non-PDF should be unchanged, got %q", result) } @@ -160,7 +160,7 @@ func TestReplacePDFTags_NoPDF(t *testing.T) { func TestReplacePDFTags_NoTags(t *testing.T) { al := &AgentLoop{} content := "Hello world" - result := al.replacePDFTags(t.Context(), content, &config.OCRConfig{Command: "echo"}, "", "") + result := al.replacePDFTags(t.Context(), content, &config.OCRConfig{Command: "echo"}, "", "", false) if result != content { t.Errorf("no tags should be unchanged, got %q", result) } @@ -177,3 +177,22 @@ func TestProcessPDFs_NilOCR(t *testing.T) { t.Errorf("content should be unchanged when OCR config is nil") } } + +func TestWantFigures(t *testing.T) { + tests := []struct { + content string + want bool + }{ + {"check this pdf", false}, + {"extract with figures please", true}, + {"Figures included", true}, + {"figure mode", true}, + {"with images", true}, + {"", false}, + } + for _, tt := range tests { + if got := wantFigures(tt.content); got != tt.want { + t.Errorf("wantFigures(%q) = %v, want %v", tt.content, got, tt.want) + } + } +} From 13962755798cb208af7e3d3a451f8de04a65beec Mon Sep 17 00:00:00 2001 From: dj-oyu <68707227+dj-oyu@users.noreply.github.com> Date: Fri, 20 Mar 2026 02:20:19 +0900 Subject: [PATCH 7/8] fix: suppress gosmopolitan lint for intentional CJK keywords Co-Authored-By: Claude Opus 4.6 (1M context) --- pkg/agent/loop_media.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/agent/loop_media.go b/pkg/agent/loop_media.go index c2139b6c0..2a8f8e472 100644 --- a/pkg/agent/loop_media.go +++ b/pkg/agent/loop_media.go @@ -420,6 +420,8 @@ func injectImageDescriptions(content string, descriptions []string) string { const maxPreviewRunes = 500 // figureKeywords triggers --figure --figure_letter when found in the message. +// +//nolint:gosmopolitan // intentional CJK keywords for Japanese users var figureKeywords = []string{ "figure", "figures", "with images", "図版", "図付き", "画像付き", "図も", @@ -436,9 +438,9 @@ func wantFigures(content string) bool { return false } -// pdfHintMessage is sent once when a PDF is first processed to explain options. +//nolint:gosmopolitan // intentional CJK in user-facing hint const pdfHintMessage = "PDF OCR in progress. " + - "Tip: include \"figures\" or \"図版\" in your message to extract images and in-figure text." + "Tip: include \"figures\" or \"\u56f3\u7248\" in your message to extract images and in-figure text." // processPDFsInMessages finds [file:/path.pdf] tags in messages and replaces // them with [document: preview... (full: /path/to.md, N pages)] tags after From c0a860426b6e766f3536c84c525d8e1228c5457a Mon Sep 17 00:00:00 2001 From: dj-oyu <68707227+dj-oyu@users.noreply.github.com> Date: Fri, 20 Mar 2026 02:23:50 +0900 Subject: [PATCH 8/8] fix: remove unused nolint directive (unicode escape already avoids detection) Co-Authored-By: Claude Opus 4.6 (1M context) --- pkg/agent/loop_media.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/agent/loop_media.go b/pkg/agent/loop_media.go index 2a8f8e472..9ab243bcc 100644 --- a/pkg/agent/loop_media.go +++ b/pkg/agent/loop_media.go @@ -438,7 +438,6 @@ func wantFigures(content string) bool { return false } -//nolint:gosmopolitan // intentional CJK in user-facing hint const pdfHintMessage = "PDF OCR in progress. " + "Tip: include \"figures\" or \"\u56f3\u7248\" in your message to extract images and in-figure text."