diff --git a/pkg/agent/loop_info.go b/pkg/agent/loop_info.go index 6a1b8c7f2..60db4dc66 100644 --- a/pkg/agent/loop_info.go +++ b/pkg/agent/loop_info.go @@ -3,6 +3,8 @@ package agent import ( "encoding/json" "fmt" + "os" + "path/filepath" "strings" "github.com/sipeed/picoclaw/pkg/mediacache" @@ -292,3 +294,42 @@ func (al *AgentLoop) ListMediaCache(entryType string) []mediacache.ListEntry { } return entries } + +// DeleteMediaCache deletes all cache entries for the given hash and cleans up files. +func (al *AgentLoop) DeleteMediaCache(hash string) error { + if al.mediaCache == nil { + return nil + } + for _, t := range []string{mediacache.TypePDFOCR, mediacache.TypePDFText, mediacache.TypeImageDesc} { + entry, err := al.mediaCache.Delete(hash, t) + if err != nil { + continue + } + if entry.FilePath != "" { + dir := filepath.Dir(entry.FilePath) + if filepath.Base(dir) == hash { + os.RemoveAll(dir) + } else { + os.Remove(entry.FilePath) + } + } + } + return nil +} + +// DeleteAllMediaCache deletes all cache entries and cleans up the OCR output directory. +func (al *AgentLoop) DeleteAllMediaCache() (int64, error) { + if al.mediaCache == nil { + return 0, nil + } + n, err := al.mediaCache.DeleteAll() + if err != nil { + return 0, err + } + // Clean up .ocr_cache directory + registry := al.GetRegistry() + if agent := registry.GetDefaultAgent(); agent != nil { + os.RemoveAll(filepath.Join(agent.Workspace, ".ocr_cache")) + } + return n, nil +} diff --git a/pkg/agent/loop_media.go b/pkg/agent/loop_media.go index 6b7c7d7af..c8aa32d3d 100644 --- a/pkg/agent/loop_media.go +++ b/pkg/agent/loop_media.go @@ -580,11 +580,15 @@ func (al *AgentLoop) ocrPDF( } } + // Per-PDF output directory: .ocr_cache// + // Using the cache hash ensures isolation between PDFs and enables reverse lookup. + outputDir := filepath.Join(al.ocrOutputDir(), hash) + // Fast path: try pdftotext for PDFs with a text layer (skip if figures requested). // pdftotext is orders of magnitude faster than OCR. if !withFigures { if text, pages, ok := tryPdftotextExtract(ctx, pdfPath); ok { - return al.savePdftotextResult(pdfPath, text, pages, hash) + return al.savePdftotextResult(pdfPath, text, pages, hash, outputDir) } } @@ -618,8 +622,7 @@ func (al *AgentLoop) ocrPDF( fmt.Sprintf("%s (0/%s)...", modeLabel, totalStr)) defer indicator.Stop() - // Determine output directory for OCR results - outputDir := al.ocrOutputDir() + // Create the per-PDF output directory os.MkdirAll(outputDir, 0o755) // Build command @@ -757,12 +760,15 @@ func (al *AgentLoop) ocrOutputDir() string { } // findOCROutput locates the markdown file generated by yomitoku. -// yomitoku names output as .md or _combined.md in the output dir. +// yomitoku names output as _.md in the output dir, +// where parent_dirname is the basename of the input PDF's parent directory. func findOCROutput(outputDir, pdfPath string) string { base := strings.TrimSuffix(filepath.Base(pdfPath), filepath.Ext(pdfPath)) + parentDir := filepath.Base(filepath.Dir(pdfPath)) - // Try common yomitoku output patterns + // Try yomitoku output patterns candidates := []string{ + filepath.Join(outputDir, parentDir+"_"+base+".md"), filepath.Join(outputDir, base+".md"), filepath.Join(outputDir, base+"_combined.md"), } diff --git a/pkg/agent/loop_media_pypdf.go b/pkg/agent/loop_media_pypdf.go index 589da2db3..50401670f 100644 --- a/pkg/agent/loop_media_pypdf.go +++ b/pkg/agent/loop_media_pypdf.go @@ -107,16 +107,14 @@ func pdfinfoPageCount(ctx context.Context, pdfPath string) int { return 0 } -// savePdftotextResult writes extracted text to a .md file in the output -// directory and caches it. Returns the document tag string. +// savePdftotextResult writes extracted text to a .md file in the per-PDF +// output directory and caches it. Returns the document tag string. func (al *AgentLoop) savePdftotextResult( - pdfPath, text string, pages int, hash string, + pdfPath, text string, pages int, hash, outputDir string, ) string { - outputDir := al.ocrOutputDir() os.MkdirAll(outputDir, 0o755) - base := strings.TrimSuffix(filepath.Base(pdfPath), filepath.Ext(pdfPath)) - mdPath := filepath.Join(outputDir, base+"_text.md") + mdPath := filepath.Join(outputDir, "document.md") if err := os.WriteFile(mdPath, []byte(text), 0o644); err != nil { logger.WarnCF("agent", "Failed to write pdftotext output", map[string]any{ diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go index 15160e8df..dc600f6bb 100644 --- a/pkg/gateway/gateway.go +++ b/pkg/gateway/gateway.go @@ -908,6 +908,14 @@ func (p *agentLoopDataProvider) ListMediaCache(entryType string) []miniapp.Media return entries } +func (p *agentLoopDataProvider) DeleteMediaCache(hash string) error { + return p.loop.DeleteMediaCache(hash) +} + +func (p *agentLoopDataProvider) DeleteAllMediaCache() (int64, error) { + return p.loop.DeleteAllMediaCache() +} + func (p *agentLoopDataProvider) GetGitRepos() []miniapp.GitRepoSummary { if time.Since(p.gitReposCacheAt) < gitCacheTTL { return p.gitReposCache diff --git a/pkg/mediacache/cache.go b/pkg/mediacache/cache.go index 05309e848..1e376b6c0 100644 --- a/pkg/mediacache/cache.go +++ b/pkg/mediacache/cache.go @@ -183,6 +183,39 @@ func (c *Cache) Prune(ttl time.Duration) (int64, error) { return res.RowsAffected() } +// Delete removes a single cache entry by hash and type. +// Returns the entry before deletion so callers can clean up associated files. +func (c *Cache) Delete(hash, entryType string) (Entry, error) { + var entry Entry + var filePath sql.NullString + row := c.db.QueryRow( + `SELECT result, file_path, pages FROM media_cache WHERE hash = ? AND type = ?`, + hash, entryType, + ) + if err := row.Scan(&entry.Result, &filePath, &entry.Pages); err != nil { + if err == sql.ErrNoRows { + return Entry{}, nil + } + return Entry{}, err + } + entry.FilePath = filePath.String + + _, err := c.db.Exec( + `DELETE FROM media_cache WHERE hash = ? AND type = ?`, + hash, entryType, + ) + return entry, err +} + +// DeleteAll removes all cache entries. Returns the number of entries removed. +func (c *Cache) DeleteAll() (int64, error) { + res, err := c.db.Exec(`DELETE FROM media_cache`) + if err != nil { + return 0, err + } + return res.RowsAffected() +} + func (c *Cache) touchAccessed(hash, entryType string) { _, _ = c.db.Exec( `UPDATE media_cache SET accessed_at = ? WHERE hash = ? AND type = ?`, diff --git a/pkg/mediacache/cache_test.go b/pkg/mediacache/cache_test.go index 51f8b6d78..5009ed5fd 100644 --- a/pkg/mediacache/cache_test.go +++ b/pkg/mediacache/cache_test.go @@ -166,6 +166,60 @@ func TestCache_GetEntry_Miss(t *testing.T) { } } +func TestCache_Delete(t *testing.T) { + c := openTestCache(t) + hash := HashData([]byte("delete-me")) + + _ = c.PutEntry(hash, TypePDFOCR, Entry{Result: "preview", FilePath: "/tmp/test.md", Pages: 3}) + + entry, err := c.Delete(hash, TypePDFOCR) + if err != nil { + t.Fatalf("Delete: %v", err) + } + if entry.FilePath != "/tmp/test.md" { + t.Errorf("returned FilePath = %q", entry.FilePath) + } + if entry.Pages != 3 { + t.Errorf("returned Pages = %d", entry.Pages) + } + + if _, ok := c.GetEntry(hash, TypePDFOCR); ok { + t.Error("entry should be deleted") + } +} + +func TestCache_Delete_NotFound(t *testing.T) { + c := openTestCache(t) + entry, err := c.Delete("nonexistent", TypePDFOCR) + if err != nil { + t.Fatalf("Delete: %v", err) + } + if entry.FilePath != "" { + t.Error("expected empty entry for not-found") + } +} + +func TestCache_DeleteAll(t *testing.T) { + c := openTestCache(t) + + _ = c.Put(HashData([]byte("a")), TypeImageDesc, "desc1") + _ = c.Put(HashData([]byte("b")), TypeImageDesc, "desc2") + _ = c.PutEntry(HashData([]byte("c")), TypePDFOCR, Entry{Result: "pdf"}) + + n, err := c.DeleteAll() + if err != nil { + t.Fatalf("DeleteAll: %v", err) + } + if n != 3 { + t.Errorf("deleted %d, want 3", n) + } + + entries, _ := c.List("") + if len(entries) != 0 { + t.Errorf("list should be empty, got %d", len(entries)) + } +} + func TestCache_SimpleGetIgnoresFilePath(t *testing.T) { // Simple Get/Put should still work with the new schema c := openTestCache(t) diff --git a/pkg/miniapp/api.go b/pkg/miniapp/api.go index b0a170e76..cdabe3748 100644 --- a/pkg/miniapp/api.go +++ b/pkg/miniapp/api.go @@ -286,12 +286,42 @@ func writeWorktreeAPIError(w http.ResponseWriter, err error) bool { // apiDevConsole receives console output from dev preview iframes. -// apiCache returns a list of media cache entries. +// apiCache dispatches GET (list) and DELETE (clear all) for media cache. func (h *Handler) apiCache(w http.ResponseWriter, r *http.Request) { - entryType := r.URL.Query().Get("type") - entries := h.provider.ListMediaCache(entryType) - if entries == nil { - entries = []MediaCacheEntry{} + switch r.Method { + case http.MethodGet: + entryType := r.URL.Query().Get("type") + entries := h.provider.ListMediaCache(entryType) + if entries == nil { + entries = []MediaCacheEntry{} + } + writeJSON(w, entries) + case http.MethodDelete: + n, err := h.provider.DeleteAllMediaCache() + if err != nil { + http.Error(w, `{"error":"failed to delete cache"}`, http.StatusInternalServerError) + return + } + writeJSON(w, map[string]any{"deleted": n}) + default: + http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed) } - writeJSON(w, entries) +} + +// apiCacheEntry handles DELETE for a single cache entry. +func (h *Handler) apiCacheEntry(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed) + return + } + hash := r.URL.Path[len("/miniapp/api/cache/"):] + if hash == "" { + http.Error(w, `{"error":"hash required"}`, http.StatusBadRequest) + return + } + if err := h.provider.DeleteMediaCache(hash); err != nil { + http.Error(w, `{"error":"failed to delete entry"}`, http.StatusInternalServerError) + return + } + writeJSON(w, map[string]string{"status": "ok"}) } diff --git a/pkg/miniapp/miniapp.go b/pkg/miniapp/miniapp.go index 289292125..3d2a4c46d 100644 --- a/pkg/miniapp/miniapp.go +++ b/pkg/miniapp/miniapp.go @@ -112,6 +112,7 @@ func (h *Handler) RegisterRoutes(mux *http.ServeMux) { mux.HandleFunc("/miniapp/dev/console", h.apiDevConsole) mux.HandleFunc("/miniapp/dev/", h.serveDevProxy) h.handleProtectedFunc(mux, "/miniapp/api/cache", h.apiCache) + h.handleProtectedFunc(mux, "/miniapp/api/cache/", h.apiCacheEntry) h.handleProtectedFunc(mux, "/miniapp/api/research", h.apiResearch) h.handleProtectedFunc(mux, "/miniapp/api/research/focus", h.apiResearchFocus) h.handleProtectedFunc(mux, "/miniapp/api/research/", h.apiResearchDetail) diff --git a/pkg/miniapp/miniapp_test.go b/pkg/miniapp/miniapp_test.go index 3be2e4958..73a0517c3 100644 --- a/pkg/miniapp/miniapp_test.go +++ b/pkg/miniapp/miniapp_test.go @@ -215,6 +215,10 @@ func (m *mockDataProvider) ListMediaCache(entryType string) []MediaCacheEntry { return nil } +func (m *mockDataProvider) DeleteMediaCache(hash string) error { return nil } + +func (m *mockDataProvider) DeleteAllMediaCache() (int64, error) { return 0, nil } + type mockSender struct{} func (m *mockSender) SendCommand(senderID, chatID, command string) {} @@ -571,6 +575,10 @@ func (m *mutatingDataProvider) ListMediaCache(entryType string) []MediaCacheEntr return nil } +func (m *mutatingDataProvider) DeleteMediaCache(hash string) error { return nil } + +func (m *mutatingDataProvider) DeleteAllMediaCache() (int64, error) { return 0, nil } + // ── Dev proxy tests ── func TestDevProxy_RegisterAndActivate(t *testing.T) { diff --git a/pkg/miniapp/types.go b/pkg/miniapp/types.go index 874a56097..ac7a7b1b9 100644 --- a/pkg/miniapp/types.go +++ b/pkg/miniapp/types.go @@ -125,7 +125,7 @@ type MediaCacheEntry struct { AccessedAt string `json:"accessed_at"` } -// DataProvider is the read-only interface to agent state for the Mini App API. +// DataProvider is the interface to agent state for the Mini App API. type DataProvider interface { ListSkills() []skills.SkillInfo GetPlanInfo() PlanInfo @@ -137,6 +137,8 @@ type DataProvider interface { GetContextInfo() ContextInfo GetSystemPrompt() string ListMediaCache(entryType string) []MediaCacheEntry + DeleteMediaCache(hash string) error + DeleteAllMediaCache() (int64, error) } // CommandSender injects a command into the message bus on behalf of a user. diff --git a/web/backend/api/media_cache.go b/web/backend/api/media_cache.go index 7d9785db1..ebb113ebd 100644 --- a/web/backend/api/media_cache.go +++ b/web/backend/api/media_cache.go @@ -34,13 +34,19 @@ func (h *Handler) openMediaCache() (*mediacache.Cache, error) { return mediacache.Open(filepath.Join(ws, "media_cache.db")) } -// handleMediaCache lists all media cache entries. +// handleMediaCache dispatches GET (list) and DELETE (clear all) for media cache. func (h *Handler) handleMediaCache(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { + switch r.Method { + case http.MethodGet: + h.listMediaCache(w, r) + case http.MethodDelete: + h.deleteAllMediaCache(w, r) + default: http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed) - return } +} +func (h *Handler) listMediaCache(w http.ResponseWriter, r *http.Request) { mc, err := h.openMediaCache() if err != nil { http.Error(w, `{"error":"media cache not available"}`, http.StatusServiceUnavailable) @@ -72,20 +78,51 @@ func (h *Handler) handleMediaCache(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(result) } -// handleMediaCacheContent serves the full file content for a PDF OCR entry. -// GET /api/media-cache/{hash} -func (h *Handler) handleMediaCacheContent(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed) +func (h *Handler) deleteAllMediaCache(w http.ResponseWriter, _ *http.Request) { + mc, err := h.openMediaCache() + if err != nil { + http.Error(w, `{"error":"media cache not available"}`, http.StatusServiceUnavailable) + return + } + defer mc.Close() + + removed, err := mc.DeleteAll() + if err != nil { + http.Error(w, `{"error":"failed to delete cache"}`, http.StatusInternalServerError) return } + // Clean up .ocr_cache directory + cfg, _ := config.LoadConfig(h.configPath) + if cfg != nil { + ocrDir := filepath.Join(cfg.WorkspacePath(), ".ocr_cache") + os.RemoveAll(ocrDir) + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]any{"deleted": removed}) +} + +// handleMediaCacheContent dispatches GET (content) and DELETE (single entry). +// /api/media-cache/{hash} +func (h *Handler) handleMediaCacheContent(w http.ResponseWriter, r *http.Request) { hash := filepath.Base(r.URL.Path) if hash == "" || hash == "media-cache" { http.Error(w, `{"error":"hash required"}`, http.StatusBadRequest) return } + switch r.Method { + case http.MethodGet: + h.getMediaCacheContent(w, hash) + case http.MethodDelete: + h.deleteMediaCacheEntry(w, hash) + default: + http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed) + } +} + +func (h *Handler) getMediaCacheContent(w http.ResponseWriter, hash string) { mc, err := h.openMediaCache() if err != nil { http.Error(w, `{"error":"media cache not available"}`, http.StatusServiceUnavailable) @@ -95,7 +132,9 @@ func (h *Handler) handleMediaCacheContent(w http.ResponseWriter, r *http.Request entry, ok := mc.GetEntry(hash, mediacache.TypePDFOCR) if !ok { - // Try image_desc + entry, ok = mc.GetEntry(hash, mediacache.TypePDFText) + } + if !ok { result, ok := mc.Get(hash, mediacache.TypeImageDesc) if !ok { http.Error(w, `{"error":"not found"}`, http.StatusNotFound) @@ -110,7 +149,6 @@ func (h *Handler) handleMediaCacheContent(w http.ResponseWriter, r *http.Request return } - // Read the full markdown file content, err := os.ReadFile(entry.FilePath) if err != nil { http.Error(w, `{"error":"file not found"}`, http.StatusNotFound) @@ -126,3 +164,32 @@ func (h *Handler) handleMediaCacheContent(w http.ResponseWriter, r *http.Request "pages": entry.Pages, }) } + +func (h *Handler) deleteMediaCacheEntry(w http.ResponseWriter, hash string) { + mc, err := h.openMediaCache() + if err != nil { + http.Error(w, `{"error":"media cache not available"}`, http.StatusServiceUnavailable) + return + } + defer mc.Close() + + // Delete all types for this hash, clean up files + for _, t := range []string{mediacache.TypePDFOCR, mediacache.TypePDFText, mediacache.TypeImageDesc} { + entry, err := mc.Delete(hash, t) + if err != nil { + continue + } + if entry.FilePath != "" { + // Remove the per-hash subdirectory if it exists + dir := filepath.Dir(entry.FilePath) + if filepath.Base(dir) == hash { + os.RemoveAll(dir) + } else { + os.Remove(entry.FilePath) + } + } + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) +} diff --git a/web/frontend/src/api/media-cache.ts b/web/frontend/src/api/media-cache.ts index 74d0a3fb9..7d40400c6 100644 --- a/web/frontend/src/api/media-cache.ts +++ b/web/frontend/src/api/media-cache.ts @@ -38,3 +38,16 @@ export async function getMediaCacheContent( `/api/media-cache/${encodeURIComponent(hash)}`, ) } + +export async function deleteMediaCacheEntry(hash: string): Promise { + const res = await fetch(`/api/media-cache/${encodeURIComponent(hash)}`, { + method: "DELETE", + }) + if (!res.ok) throw new Error(`API error: ${res.status}`) +} + +export async function deleteAllMediaCache(): Promise<{ deleted: number }> { + const res = await fetch("/api/media-cache", { method: "DELETE" }) + if (!res.ok) throw new Error(`API error: ${res.status}`) + return res.json() as Promise<{ deleted: number }> +} diff --git a/web/frontend/src/components/research/media-cache-page.tsx b/web/frontend/src/components/research/media-cache-page.tsx index 1f434dc65..dac73ad8c 100644 --- a/web/frontend/src/components/research/media-cache-page.tsx +++ b/web/frontend/src/components/research/media-cache-page.tsx @@ -1,13 +1,16 @@ import { IconFileText, IconPhoto, + IconTrash, } from "@tabler/icons-react" -import { useQuery } from "@tanstack/react-query" +import { useQuery, useQueryClient } from "@tanstack/react-query" import * as React from "react" import { type MediaCacheContent, type MediaCacheEntry, + deleteAllMediaCache, + deleteMediaCacheEntry, getMediaCacheContent, getMediaCacheEntries, } from "@/api/media-cache" @@ -24,6 +27,7 @@ import { cn } from "@/lib/utils" export function MediaCachePage() { const [typeFilter, setTypeFilter] = React.useState("") const [expandedHash, setExpandedHash] = React.useState(null) + const queryClient = useQueryClient() const { data: entries, isLoading, error } = useQuery({ queryKey: ["media-cache", typeFilter], @@ -31,11 +35,24 @@ export function MediaCachePage() { refetchInterval: 30000, }) + const handleDeleteAll = async () => { + if (!confirm("Delete all cached media?")) return + await deleteAllMediaCache() + setExpandedHash(null) + queryClient.invalidateQueries({ queryKey: ["media-cache"] }) + } + + const handleDeleteEntry = async (hash: string) => { + await deleteMediaCacheEntry(hash) + if (expandedHash === hash) setExpandedHash(null) + queryClient.invalidateQueries({ queryKey: ["media-cache"] }) + } + return (
- {/* Type filter */} -
+ {/* Type filter + clear all */} +
setTypeFilter("")} @@ -56,6 +73,17 @@ export function MediaCachePage() { PDF + {entries && entries.length > 0 && ( + + )}
{isLoading ? ( @@ -82,6 +110,7 @@ export function MediaCachePage() { expandedHash === entry.hash ? null : entry.hash, ) } + onDelete={() => handleDeleteEntry(entry.hash)} /> ))}
@@ -116,10 +145,12 @@ function MediaEntry({ entry, expanded, onToggle, + onDelete, }: { entry: MediaCacheEntry expanded: boolean onToggle: () => void + onDelete: () => void }) { const isImage = entry.type === "image_desc" const Icon = isImage ? IconPhoto : IconFileText @@ -163,14 +194,20 @@ function MediaEntry({ {expanded && ( - + )} ) } -function ExpandedContent({ entry }: { entry: MediaCacheEntry }) { +function ExpandedContent({ + entry, + onDelete, +}: { + entry: MediaCacheEntry + onDelete: () => void +}) { const isPDF = entry.type === "pdf_ocr" const { data, isLoading } = useQuery({ @@ -179,49 +216,58 @@ function ExpandedContent({ entry }: { entry: MediaCacheEntry }) { enabled: isPDF, // only fetch full content for PDFs }) - if (!isPDF) { - // Image description: show full result inline - return ( -
-
- Description -
-
- {entry.result} -
-
- ) - } - - // PDF OCR: show preview + full content on demand return (
-
-
Preview
-
- {entry.result} -
-
- {entry.file_path && ( -
- - {entry.file_path} -
- )} - {isLoading ? ( -
- Loading full content... -
- ) : data?.content ? ( -
+ {!isPDF ? ( +
- Full OCR Content + Description
-
- {data.content} +
+ {entry.result}
- ) : null} + ) : ( + <> +
+
Preview
+
+ {entry.result} +
+
+ {entry.file_path && ( +
+ + {entry.file_path} +
+ )} + {isLoading ? ( +
+ Loading full content... +
+ ) : data?.content ? ( +
+
+ Full OCR Content +
+
+ {data.content} +
+
+ ) : null} + + )} +
+ +
) }