fix: isolate OCR output per PDF + add cache deletion UI

Root cause: yomitoku's -o flag received a shared directory (.ocr_cache/),
causing multiple PDFs' outputs to mix. findOCROutput's fallback would
return a previous PDF's result file.

Changes:
- Output directory is now .ocr_cache/<hash>/ (1 PDF = 1 subdirectory)
- Hash matches cache key (FNV-1a 64bit) for reverse lookup
- findOCROutput candidates include yomitoku's <parentdir>_<base>.md pattern
- savePdftotextResult outputs to same per-hash directory
- Add Delete/DeleteAll to mediacache.Cache
- Add DELETE endpoints to web backend and miniapp APIs
- Add delete buttons to Mini App cache page (per-entry + clear all)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
dj-oyu 2026-03-20 15:52:56 +09:00
parent a9455c3727
commit f293f54dca
13 changed files with 377 additions and 70 deletions

View file

@ -3,6 +3,8 @@ package agent
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/sipeed/picoclaw/pkg/mediacache"
@ -292,3 +294,42 @@ func (al *AgentLoop) ListMediaCache(entryType string) []mediacache.ListEntry {
}
return entries
}
// DeleteMediaCache deletes all cache entries for the given hash and cleans up files.
func (al *AgentLoop) DeleteMediaCache(hash string) error {
if al.mediaCache == nil {
return nil
}
for _, t := range []string{mediacache.TypePDFOCR, mediacache.TypePDFText, mediacache.TypeImageDesc} {
entry, err := al.mediaCache.Delete(hash, t)
if err != nil {
continue
}
if entry.FilePath != "" {
dir := filepath.Dir(entry.FilePath)
if filepath.Base(dir) == hash {
os.RemoveAll(dir)
} else {
os.Remove(entry.FilePath)
}
}
}
return nil
}
// DeleteAllMediaCache deletes all cache entries and cleans up the OCR output directory.
func (al *AgentLoop) DeleteAllMediaCache() (int64, error) {
if al.mediaCache == nil {
return 0, nil
}
n, err := al.mediaCache.DeleteAll()
if err != nil {
return 0, err
}
// Clean up .ocr_cache directory
registry := al.GetRegistry()
if agent := registry.GetDefaultAgent(); agent != nil {
os.RemoveAll(filepath.Join(agent.Workspace, ".ocr_cache"))
}
return n, nil
}

View file

@ -580,11 +580,15 @@ func (al *AgentLoop) ocrPDF(
}
}
// Per-PDF output directory: .ocr_cache/<hash>/
// Using the cache hash ensures isolation between PDFs and enables reverse lookup.
outputDir := filepath.Join(al.ocrOutputDir(), hash)
// Fast path: try pdftotext for PDFs with a text layer (skip if figures requested).
// pdftotext is orders of magnitude faster than OCR.
if !withFigures {
if text, pages, ok := tryPdftotextExtract(ctx, pdfPath); ok {
return al.savePdftotextResult(pdfPath, text, pages, hash)
return al.savePdftotextResult(pdfPath, text, pages, hash, outputDir)
}
}
@ -618,8 +622,7 @@ func (al *AgentLoop) ocrPDF(
fmt.Sprintf("%s (0/%s)...", modeLabel, totalStr))
defer indicator.Stop()
// Determine output directory for OCR results
outputDir := al.ocrOutputDir()
// Create the per-PDF output directory
os.MkdirAll(outputDir, 0o755)
// Build command
@ -757,12 +760,15 @@ func (al *AgentLoop) ocrOutputDir() string {
}
// findOCROutput locates the markdown file generated by yomitoku.
// yomitoku names output as <basename>.md or <basename>_combined.md in the output dir.
// yomitoku names output as <parent_dirname>_<basename>.md in the output dir,
// where parent_dirname is the basename of the input PDF's parent directory.
func findOCROutput(outputDir, pdfPath string) string {
base := strings.TrimSuffix(filepath.Base(pdfPath), filepath.Ext(pdfPath))
parentDir := filepath.Base(filepath.Dir(pdfPath))
// Try common yomitoku output patterns
// Try yomitoku output patterns
candidates := []string{
filepath.Join(outputDir, parentDir+"_"+base+".md"),
filepath.Join(outputDir, base+".md"),
filepath.Join(outputDir, base+"_combined.md"),
}

View file

@ -107,16 +107,14 @@ func pdfinfoPageCount(ctx context.Context, pdfPath string) int {
return 0
}
// savePdftotextResult writes extracted text to a .md file in the output
// directory and caches it. Returns the document tag string.
// savePdftotextResult writes extracted text to a .md file in the per-PDF
// output directory and caches it. Returns the document tag string.
func (al *AgentLoop) savePdftotextResult(
pdfPath, text string, pages int, hash string,
pdfPath, text string, pages int, hash, outputDir string,
) string {
outputDir := al.ocrOutputDir()
os.MkdirAll(outputDir, 0o755)
base := strings.TrimSuffix(filepath.Base(pdfPath), filepath.Ext(pdfPath))
mdPath := filepath.Join(outputDir, base+"_text.md")
mdPath := filepath.Join(outputDir, "document.md")
if err := os.WriteFile(mdPath, []byte(text), 0o644); err != nil {
logger.WarnCF("agent", "Failed to write pdftotext output", map[string]any{

View file

@ -908,6 +908,14 @@ func (p *agentLoopDataProvider) ListMediaCache(entryType string) []miniapp.Media
return entries
}
func (p *agentLoopDataProvider) DeleteMediaCache(hash string) error {
return p.loop.DeleteMediaCache(hash)
}
func (p *agentLoopDataProvider) DeleteAllMediaCache() (int64, error) {
return p.loop.DeleteAllMediaCache()
}
func (p *agentLoopDataProvider) GetGitRepos() []miniapp.GitRepoSummary {
if time.Since(p.gitReposCacheAt) < gitCacheTTL {
return p.gitReposCache

View file

@ -183,6 +183,39 @@ func (c *Cache) Prune(ttl time.Duration) (int64, error) {
return res.RowsAffected()
}
// Delete removes a single cache entry by hash and type.
// Returns the entry before deletion so callers can clean up associated files.
func (c *Cache) Delete(hash, entryType string) (Entry, error) {
var entry Entry
var filePath sql.NullString
row := c.db.QueryRow(
`SELECT result, file_path, pages FROM media_cache WHERE hash = ? AND type = ?`,
hash, entryType,
)
if err := row.Scan(&entry.Result, &filePath, &entry.Pages); err != nil {
if err == sql.ErrNoRows {
return Entry{}, nil
}
return Entry{}, err
}
entry.FilePath = filePath.String
_, err := c.db.Exec(
`DELETE FROM media_cache WHERE hash = ? AND type = ?`,
hash, entryType,
)
return entry, err
}
// DeleteAll removes all cache entries. Returns the number of entries removed.
func (c *Cache) DeleteAll() (int64, error) {
res, err := c.db.Exec(`DELETE FROM media_cache`)
if err != nil {
return 0, err
}
return res.RowsAffected()
}
func (c *Cache) touchAccessed(hash, entryType string) {
_, _ = c.db.Exec(
`UPDATE media_cache SET accessed_at = ? WHERE hash = ? AND type = ?`,

View file

@ -166,6 +166,60 @@ func TestCache_GetEntry_Miss(t *testing.T) {
}
}
func TestCache_Delete(t *testing.T) {
c := openTestCache(t)
hash := HashData([]byte("delete-me"))
_ = c.PutEntry(hash, TypePDFOCR, Entry{Result: "preview", FilePath: "/tmp/test.md", Pages: 3})
entry, err := c.Delete(hash, TypePDFOCR)
if err != nil {
t.Fatalf("Delete: %v", err)
}
if entry.FilePath != "/tmp/test.md" {
t.Errorf("returned FilePath = %q", entry.FilePath)
}
if entry.Pages != 3 {
t.Errorf("returned Pages = %d", entry.Pages)
}
if _, ok := c.GetEntry(hash, TypePDFOCR); ok {
t.Error("entry should be deleted")
}
}
func TestCache_Delete_NotFound(t *testing.T) {
c := openTestCache(t)
entry, err := c.Delete("nonexistent", TypePDFOCR)
if err != nil {
t.Fatalf("Delete: %v", err)
}
if entry.FilePath != "" {
t.Error("expected empty entry for not-found")
}
}
func TestCache_DeleteAll(t *testing.T) {
c := openTestCache(t)
_ = c.Put(HashData([]byte("a")), TypeImageDesc, "desc1")
_ = c.Put(HashData([]byte("b")), TypeImageDesc, "desc2")
_ = c.PutEntry(HashData([]byte("c")), TypePDFOCR, Entry{Result: "pdf"})
n, err := c.DeleteAll()
if err != nil {
t.Fatalf("DeleteAll: %v", err)
}
if n != 3 {
t.Errorf("deleted %d, want 3", n)
}
entries, _ := c.List("")
if len(entries) != 0 {
t.Errorf("list should be empty, got %d", len(entries))
}
}
func TestCache_SimpleGetIgnoresFilePath(t *testing.T) {
// Simple Get/Put should still work with the new schema
c := openTestCache(t)

View file

@ -286,12 +286,42 @@ func writeWorktreeAPIError(w http.ResponseWriter, err error) bool {
// apiDevConsole receives console output from dev preview iframes.
// apiCache returns a list of media cache entries.
// apiCache dispatches GET (list) and DELETE (clear all) for media cache.
func (h *Handler) apiCache(w http.ResponseWriter, r *http.Request) {
entryType := r.URL.Query().Get("type")
entries := h.provider.ListMediaCache(entryType)
if entries == nil {
entries = []MediaCacheEntry{}
switch r.Method {
case http.MethodGet:
entryType := r.URL.Query().Get("type")
entries := h.provider.ListMediaCache(entryType)
if entries == nil {
entries = []MediaCacheEntry{}
}
writeJSON(w, entries)
case http.MethodDelete:
n, err := h.provider.DeleteAllMediaCache()
if err != nil {
http.Error(w, `{"error":"failed to delete cache"}`, http.StatusInternalServerError)
return
}
writeJSON(w, map[string]any{"deleted": n})
default:
http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed)
}
writeJSON(w, entries)
}
// apiCacheEntry handles DELETE for a single cache entry.
func (h *Handler) apiCacheEntry(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodDelete {
http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed)
return
}
hash := r.URL.Path[len("/miniapp/api/cache/"):]
if hash == "" {
http.Error(w, `{"error":"hash required"}`, http.StatusBadRequest)
return
}
if err := h.provider.DeleteMediaCache(hash); err != nil {
http.Error(w, `{"error":"failed to delete entry"}`, http.StatusInternalServerError)
return
}
writeJSON(w, map[string]string{"status": "ok"})
}

View file

@ -112,6 +112,7 @@ func (h *Handler) RegisterRoutes(mux *http.ServeMux) {
mux.HandleFunc("/miniapp/dev/console", h.apiDevConsole)
mux.HandleFunc("/miniapp/dev/", h.serveDevProxy)
h.handleProtectedFunc(mux, "/miniapp/api/cache", h.apiCache)
h.handleProtectedFunc(mux, "/miniapp/api/cache/", h.apiCacheEntry)
h.handleProtectedFunc(mux, "/miniapp/api/research", h.apiResearch)
h.handleProtectedFunc(mux, "/miniapp/api/research/focus", h.apiResearchFocus)
h.handleProtectedFunc(mux, "/miniapp/api/research/", h.apiResearchDetail)

View file

@ -215,6 +215,10 @@ func (m *mockDataProvider) ListMediaCache(entryType string) []MediaCacheEntry {
return nil
}
func (m *mockDataProvider) DeleteMediaCache(hash string) error { return nil }
func (m *mockDataProvider) DeleteAllMediaCache() (int64, error) { return 0, nil }
type mockSender struct{}
func (m *mockSender) SendCommand(senderID, chatID, command string) {}
@ -571,6 +575,10 @@ func (m *mutatingDataProvider) ListMediaCache(entryType string) []MediaCacheEntr
return nil
}
func (m *mutatingDataProvider) DeleteMediaCache(hash string) error { return nil }
func (m *mutatingDataProvider) DeleteAllMediaCache() (int64, error) { return 0, nil }
// ── Dev proxy tests ──
func TestDevProxy_RegisterAndActivate(t *testing.T) {

View file

@ -125,7 +125,7 @@ type MediaCacheEntry struct {
AccessedAt string `json:"accessed_at"`
}
// DataProvider is the read-only interface to agent state for the Mini App API.
// DataProvider is the interface to agent state for the Mini App API.
type DataProvider interface {
ListSkills() []skills.SkillInfo
GetPlanInfo() PlanInfo
@ -137,6 +137,8 @@ type DataProvider interface {
GetContextInfo() ContextInfo
GetSystemPrompt() string
ListMediaCache(entryType string) []MediaCacheEntry
DeleteMediaCache(hash string) error
DeleteAllMediaCache() (int64, error)
}
// CommandSender injects a command into the message bus on behalf of a user.

View file

@ -34,13 +34,19 @@ func (h *Handler) openMediaCache() (*mediacache.Cache, error) {
return mediacache.Open(filepath.Join(ws, "media_cache.db"))
}
// handleMediaCache lists all media cache entries.
// handleMediaCache dispatches GET (list) and DELETE (clear all) for media cache.
func (h *Handler) handleMediaCache(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
switch r.Method {
case http.MethodGet:
h.listMediaCache(w, r)
case http.MethodDelete:
h.deleteAllMediaCache(w, r)
default:
http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed)
return
}
}
func (h *Handler) listMediaCache(w http.ResponseWriter, r *http.Request) {
mc, err := h.openMediaCache()
if err != nil {
http.Error(w, `{"error":"media cache not available"}`, http.StatusServiceUnavailable)
@ -72,20 +78,51 @@ func (h *Handler) handleMediaCache(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(result)
}
// handleMediaCacheContent serves the full file content for a PDF OCR entry.
// GET /api/media-cache/{hash}
func (h *Handler) handleMediaCacheContent(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed)
func (h *Handler) deleteAllMediaCache(w http.ResponseWriter, _ *http.Request) {
mc, err := h.openMediaCache()
if err != nil {
http.Error(w, `{"error":"media cache not available"}`, http.StatusServiceUnavailable)
return
}
defer mc.Close()
removed, err := mc.DeleteAll()
if err != nil {
http.Error(w, `{"error":"failed to delete cache"}`, http.StatusInternalServerError)
return
}
// Clean up .ocr_cache directory
cfg, _ := config.LoadConfig(h.configPath)
if cfg != nil {
ocrDir := filepath.Join(cfg.WorkspacePath(), ".ocr_cache")
os.RemoveAll(ocrDir)
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]any{"deleted": removed})
}
// handleMediaCacheContent dispatches GET (content) and DELETE (single entry).
// /api/media-cache/{hash}
func (h *Handler) handleMediaCacheContent(w http.ResponseWriter, r *http.Request) {
hash := filepath.Base(r.URL.Path)
if hash == "" || hash == "media-cache" {
http.Error(w, `{"error":"hash required"}`, http.StatusBadRequest)
return
}
switch r.Method {
case http.MethodGet:
h.getMediaCacheContent(w, hash)
case http.MethodDelete:
h.deleteMediaCacheEntry(w, hash)
default:
http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed)
}
}
func (h *Handler) getMediaCacheContent(w http.ResponseWriter, hash string) {
mc, err := h.openMediaCache()
if err != nil {
http.Error(w, `{"error":"media cache not available"}`, http.StatusServiceUnavailable)
@ -95,7 +132,9 @@ func (h *Handler) handleMediaCacheContent(w http.ResponseWriter, r *http.Request
entry, ok := mc.GetEntry(hash, mediacache.TypePDFOCR)
if !ok {
// Try image_desc
entry, ok = mc.GetEntry(hash, mediacache.TypePDFText)
}
if !ok {
result, ok := mc.Get(hash, mediacache.TypeImageDesc)
if !ok {
http.Error(w, `{"error":"not found"}`, http.StatusNotFound)
@ -110,7 +149,6 @@ func (h *Handler) handleMediaCacheContent(w http.ResponseWriter, r *http.Request
return
}
// Read the full markdown file
content, err := os.ReadFile(entry.FilePath)
if err != nil {
http.Error(w, `{"error":"file not found"}`, http.StatusNotFound)
@ -126,3 +164,32 @@ func (h *Handler) handleMediaCacheContent(w http.ResponseWriter, r *http.Request
"pages": entry.Pages,
})
}
func (h *Handler) deleteMediaCacheEntry(w http.ResponseWriter, hash string) {
mc, err := h.openMediaCache()
if err != nil {
http.Error(w, `{"error":"media cache not available"}`, http.StatusServiceUnavailable)
return
}
defer mc.Close()
// Delete all types for this hash, clean up files
for _, t := range []string{mediacache.TypePDFOCR, mediacache.TypePDFText, mediacache.TypeImageDesc} {
entry, err := mc.Delete(hash, t)
if err != nil {
continue
}
if entry.FilePath != "" {
// Remove the per-hash subdirectory if it exists
dir := filepath.Dir(entry.FilePath)
if filepath.Base(dir) == hash {
os.RemoveAll(dir)
} else {
os.Remove(entry.FilePath)
}
}
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
}

View file

@ -38,3 +38,16 @@ export async function getMediaCacheContent(
`/api/media-cache/${encodeURIComponent(hash)}`,
)
}
export async function deleteMediaCacheEntry(hash: string): Promise<void> {
const res = await fetch(`/api/media-cache/${encodeURIComponent(hash)}`, {
method: "DELETE",
})
if (!res.ok) throw new Error(`API error: ${res.status}`)
}
export async function deleteAllMediaCache(): Promise<{ deleted: number }> {
const res = await fetch("/api/media-cache", { method: "DELETE" })
if (!res.ok) throw new Error(`API error: ${res.status}`)
return res.json() as Promise<{ deleted: number }>
}

View file

@ -1,13 +1,16 @@
import {
IconFileText,
IconPhoto,
IconTrash,
} from "@tabler/icons-react"
import { useQuery } from "@tanstack/react-query"
import { useQuery, useQueryClient } from "@tanstack/react-query"
import * as React from "react"
import {
type MediaCacheContent,
type MediaCacheEntry,
deleteAllMediaCache,
deleteMediaCacheEntry,
getMediaCacheContent,
getMediaCacheEntries,
} from "@/api/media-cache"
@ -24,6 +27,7 @@ import { cn } from "@/lib/utils"
export function MediaCachePage() {
const [typeFilter, setTypeFilter] = React.useState<string>("")
const [expandedHash, setExpandedHash] = React.useState<string | null>(null)
const queryClient = useQueryClient()
const { data: entries, isLoading, error } = useQuery({
queryKey: ["media-cache", typeFilter],
@ -31,11 +35,24 @@ export function MediaCachePage() {
refetchInterval: 30000,
})
const handleDeleteAll = async () => {
if (!confirm("Delete all cached media?")) return
await deleteAllMediaCache()
setExpandedHash(null)
queryClient.invalidateQueries({ queryKey: ["media-cache"] })
}
const handleDeleteEntry = async (hash: string) => {
await deleteMediaCacheEntry(hash)
if (expandedHash === hash) setExpandedHash(null)
queryClient.invalidateQueries({ queryKey: ["media-cache"] })
}
return (
<div className="flex-1 overflow-auto px-6 py-3">
<div className="w-full max-w-6xl space-y-4">
{/* Type filter */}
<div className="flex gap-2">
{/* Type filter + clear all */}
<div className="flex items-center gap-2">
<FilterButton
active={typeFilter === ""}
onClick={() => setTypeFilter("")}
@ -56,6 +73,17 @@ export function MediaCachePage() {
<IconFileText className="size-3.5" />
PDF
</FilterButton>
{entries && entries.length > 0 && (
<Button
variant="ghost"
size="sm"
className="text-destructive ml-auto gap-1"
onClick={handleDeleteAll}
>
<IconTrash className="size-3.5" />
Clear All
</Button>
)}
</div>
{isLoading ? (
@ -82,6 +110,7 @@ export function MediaCachePage() {
expandedHash === entry.hash ? null : entry.hash,
)
}
onDelete={() => handleDeleteEntry(entry.hash)}
/>
))}
</div>
@ -116,10 +145,12 @@ function MediaEntry({
entry,
expanded,
onToggle,
onDelete,
}: {
entry: MediaCacheEntry
expanded: boolean
onToggle: () => void
onDelete: () => void
}) {
const isImage = entry.type === "image_desc"
const Icon = isImage ? IconPhoto : IconFileText
@ -163,14 +194,20 @@ function MediaEntry({
</CardHeader>
{expanded && (
<CardContent className="border-t pt-3">
<ExpandedContent entry={entry} />
<ExpandedContent entry={entry} onDelete={onDelete} />
</CardContent>
)}
</Card>
)
}
function ExpandedContent({ entry }: { entry: MediaCacheEntry }) {
function ExpandedContent({
entry,
onDelete,
}: {
entry: MediaCacheEntry
onDelete: () => void
}) {
const isPDF = entry.type === "pdf_ocr"
const { data, isLoading } = useQuery({
@ -179,49 +216,58 @@ function ExpandedContent({ entry }: { entry: MediaCacheEntry }) {
enabled: isPDF, // only fetch full content for PDFs
})
if (!isPDF) {
// Image description: show full result inline
return (
<div className="space-y-2">
<div className="text-muted-foreground text-xs font-medium">
Description
</div>
<div className="bg-muted rounded-md p-3 text-sm whitespace-pre-wrap">
{entry.result}
</div>
</div>
)
}
// PDF OCR: show preview + full content on demand
return (
<div className="space-y-3">
<div className="space-y-1">
<div className="text-muted-foreground text-xs font-medium">Preview</div>
<div className="bg-muted rounded-md p-3 text-sm whitespace-pre-wrap">
{entry.result}
</div>
</div>
{entry.file_path && (
<div className="text-muted-foreground flex items-center gap-1 text-xs">
<IconFileText className="size-3" />
<span className="font-mono">{entry.file_path}</span>
</div>
)}
{isLoading ? (
<div className="text-muted-foreground py-2 text-sm">
Loading full content...
</div>
) : data?.content ? (
<div className="space-y-1">
{!isPDF ? (
<div className="space-y-2">
<div className="text-muted-foreground text-xs font-medium">
Full OCR Content
Description
</div>
<div className="bg-muted max-h-96 overflow-auto rounded-md p-3 text-sm whitespace-pre-wrap">
{data.content}
<div className="bg-muted rounded-md p-3 text-sm whitespace-pre-wrap">
{entry.result}
</div>
</div>
) : null}
) : (
<>
<div className="space-y-1">
<div className="text-muted-foreground text-xs font-medium">Preview</div>
<div className="bg-muted rounded-md p-3 text-sm whitespace-pre-wrap">
{entry.result}
</div>
</div>
{entry.file_path && (
<div className="text-muted-foreground flex items-center gap-1 text-xs">
<IconFileText className="size-3" />
<span className="font-mono">{entry.file_path}</span>
</div>
)}
{isLoading ? (
<div className="text-muted-foreground py-2 text-sm">
Loading full content...
</div>
) : data?.content ? (
<div className="space-y-1">
<div className="text-muted-foreground text-xs font-medium">
Full OCR Content
</div>
<div className="bg-muted max-h-96 overflow-auto rounded-md p-3 text-sm whitespace-pre-wrap">
{data.content}
</div>
</div>
) : null}
</>
)}
<div className="flex justify-end">
<Button
variant="ghost"
size="sm"
className="text-destructive gap-1"
onClick={onDelete}
>
<IconTrash className="size-3.5" />
Delete
</Button>
</div>
</div>
)
}