Merge pull request #64 from dj-oyu/feature/ocr-output-isolation
fix: isolate OCR output per PDF + cache deletion UI
This commit is contained in:
commit
f05fb6abb8
13 changed files with 486 additions and 71 deletions
91
docs/yomitoku_ocr.md
Normal file
91
docs/yomitoku_ocr.md
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
# yomitoku OCR Integration
|
||||
|
||||
picoclaw は [yomitoku](https://github.com/kotaro-kinoshita/yomitoku) CLI と連携し、スキャン PDF や画像ベースの PDF からテキストを抽出します。
|
||||
|
||||
## Setup
|
||||
|
||||
`config.json` の `ocr` セクションで設定:
|
||||
|
||||
```json
|
||||
{
|
||||
"ocr": {
|
||||
"command": "/path/to/yomitoku/.venv/bin/yomitoku",
|
||||
"args": [],
|
||||
"env": {},
|
||||
"timeout": 600,
|
||||
"reading_order": "auto"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
| フィールド | 説明 | デフォルト |
|
||||
|---|---|---|
|
||||
| `command` | yomitoku CLI のパス | (必須) |
|
||||
| `args` | 追加の CLI 引数 | `[]` |
|
||||
| `env` | 環境変数 (key-value) | `{}` |
|
||||
| `timeout` | OCR タイムアウト (秒) | `600` |
|
||||
| `reading_order` | デフォルトの読み順 | `"auto"` |
|
||||
|
||||
### reading_order の値
|
||||
|
||||
| 値 | 用途 |
|
||||
|---|---|
|
||||
| `auto` | yomitoku が自動判定 (デフォルト) |
|
||||
| `right2left` | 縦書き文書 |
|
||||
| `top2bottom` | 横書きの一般文書 |
|
||||
| `left2right` | 横並びの帳票系 |
|
||||
|
||||
## Processing Flow
|
||||
|
||||
PDF ファイルが送られると以下の順序で処理されます:
|
||||
|
||||
1. **Phase 1 (キーワード待ち)**: PDF のみ送信された場合、5秒間フォローアップメッセージを待つ
|
||||
2. **pdftotext fast path**: テキストレイヤーのある PDF は `pdftotext` で高速抽出 (figures 指定時はスキップ)
|
||||
3. **yomitoku OCR**: テキスト抽出に失敗した場合、yomitoku で OCR 実行
|
||||
4. **Phase 2 (バッファリング)**: OCR 中のメッセージをバッファし、完了後に LLM に渡す
|
||||
|
||||
## Chat Keywords
|
||||
|
||||
メッセージ中のキーワードで OCR オプションを制御できます:
|
||||
|
||||
### Figures (図版抽出)
|
||||
|
||||
`--figure --figure_letter` を追加:
|
||||
|
||||
- `figure`, `figures`, `with images`
|
||||
- `図版`, `図付き`, `画像付き`, `図も`
|
||||
|
||||
### Reading Order (読み順)
|
||||
|
||||
- `縦書き`, `たてがき`, `vertical` → `--reading_order right2left`
|
||||
- `横書き`, `よこがき`, `horizontal` → `--reading_order top2bottom`
|
||||
- `right2left`, `top2bottom`, `left2right` → そのまま指定
|
||||
|
||||
### Cancel (中断)
|
||||
|
||||
OCR 実行中に以下のキーワードで中断:
|
||||
|
||||
- `cancel`, `abort`, `stop`
|
||||
- `中止`, `キャンセル`, `やめ`
|
||||
|
||||
## Output Structure
|
||||
|
||||
OCR 結果は `.ocr_cache/<hash>/` に保存されます:
|
||||
|
||||
```
|
||||
.ocr_cache/
|
||||
c5a9f00fe7567ac0/ # FNV-1a 64bit hash (= cache key)
|
||||
document.md # OCR テキスト (yomitoku 出力をリネーム)
|
||||
figures/ # 抽出された図版 (--figure 時)
|
||||
b92a7e9171b9b9e5/
|
||||
document.md # pdftotext 結果
|
||||
```
|
||||
|
||||
- ハッシュはキャッシュキーと同一 → ファイルからキャッシュを逆引き可能
|
||||
- figures 有無・reading order が異なれば別ハッシュ (別ディレクトリ)
|
||||
|
||||
## Cache Management
|
||||
|
||||
- **自動 prune**: 7日間アクセスのないエントリを自動削除
|
||||
- **Mini App**: キャッシュ一覧 + 個別削除 / 全削除 UI (`/miniapp/api/cache`)
|
||||
- **API**: `DELETE /api/media-cache/{hash}` (個別) / `DELETE /api/media-cache` (全削除)
|
||||
|
|
@ -3,6 +3,8 @@ package agent
|
|||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/sipeed/picoclaw/pkg/mediacache"
|
||||
|
|
@ -292,3 +294,42 @@ func (al *AgentLoop) ListMediaCache(entryType string) []mediacache.ListEntry {
|
|||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
// DeleteMediaCache deletes all cache entries for the given hash and cleans up files.
|
||||
func (al *AgentLoop) DeleteMediaCache(hash string) error {
|
||||
if al.mediaCache == nil {
|
||||
return nil
|
||||
}
|
||||
for _, t := range []string{mediacache.TypePDFOCR, mediacache.TypePDFText, mediacache.TypeImageDesc} {
|
||||
entry, err := al.mediaCache.Delete(hash, t)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if entry.FilePath != "" {
|
||||
dir := filepath.Dir(entry.FilePath)
|
||||
if filepath.Base(dir) == hash {
|
||||
os.RemoveAll(dir)
|
||||
} else {
|
||||
os.Remove(entry.FilePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteAllMediaCache deletes all cache entries and cleans up the OCR output directory.
|
||||
func (al *AgentLoop) DeleteAllMediaCache() (int64, error) {
|
||||
if al.mediaCache == nil {
|
||||
return 0, nil
|
||||
}
|
||||
n, err := al.mediaCache.DeleteAll()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Clean up .ocr_cache directory
|
||||
registry := al.GetRegistry()
|
||||
if agent := registry.GetDefaultAgent(); agent != nil {
|
||||
os.RemoveAll(filepath.Join(agent.Workspace, ".ocr_cache"))
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -580,11 +580,15 @@ func (al *AgentLoop) ocrPDF(
|
|||
}
|
||||
}
|
||||
|
||||
// Per-PDF output directory: .ocr_cache/<hash>/
|
||||
// Using the cache hash ensures isolation between PDFs and enables reverse lookup.
|
||||
outputDir := filepath.Join(al.ocrOutputDir(), hash)
|
||||
|
||||
// Fast path: try pdftotext for PDFs with a text layer (skip if figures requested).
|
||||
// pdftotext is orders of magnitude faster than OCR.
|
||||
if !withFigures {
|
||||
if text, pages, ok := tryPdftotextExtract(ctx, pdfPath); ok {
|
||||
return al.savePdftotextResult(pdfPath, text, pages, hash)
|
||||
return al.savePdftotextResult(pdfPath, text, pages, hash, outputDir)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -618,8 +622,7 @@ func (al *AgentLoop) ocrPDF(
|
|||
fmt.Sprintf("%s (0/%s)...", modeLabel, totalStr))
|
||||
defer indicator.Stop()
|
||||
|
||||
// Determine output directory for OCR results
|
||||
outputDir := al.ocrOutputDir()
|
||||
// Create the per-PDF output directory
|
||||
os.MkdirAll(outputDir, 0o755)
|
||||
|
||||
// Build command
|
||||
|
|
@ -694,12 +697,18 @@ func (al *AgentLoop) ocrPDF(
|
|||
// Keep: .md files, figures/ directory (referenced by markdown output).
|
||||
cleanupOCRPageImages(outputDir, pdfPath)
|
||||
|
||||
// Find the output markdown file
|
||||
// Find the output markdown file and normalize its name to document.md
|
||||
mdPath := findOCROutput(outputDir, pdfPath)
|
||||
if mdPath == "" {
|
||||
logger.WarnCF("agent", "OCR output not found", map[string]any{"output_dir": outputDir})
|
||||
return fmt.Sprintf("[file:%s]", pdfPath)
|
||||
}
|
||||
canonical := filepath.Join(outputDir, "document.md")
|
||||
if mdPath != canonical {
|
||||
if renameErr := os.Rename(mdPath, canonical); renameErr == nil {
|
||||
mdPath = canonical
|
||||
}
|
||||
}
|
||||
|
||||
// Read preview from first part of the markdown
|
||||
mdData, err := os.ReadFile(mdPath)
|
||||
|
|
@ -757,12 +766,15 @@ func (al *AgentLoop) ocrOutputDir() string {
|
|||
}
|
||||
|
||||
// findOCROutput locates the markdown file generated by yomitoku.
|
||||
// yomitoku names output as <basename>.md or <basename>_combined.md in the output dir.
|
||||
// yomitoku names output as <parent_dirname>_<basename>.md in the output dir,
|
||||
// where parent_dirname is the basename of the input PDF's parent directory.
|
||||
func findOCROutput(outputDir, pdfPath string) string {
|
||||
base := strings.TrimSuffix(filepath.Base(pdfPath), filepath.Ext(pdfPath))
|
||||
parentDir := filepath.Base(filepath.Dir(pdfPath))
|
||||
|
||||
// Try common yomitoku output patterns
|
||||
// Try yomitoku output patterns
|
||||
candidates := []string{
|
||||
filepath.Join(outputDir, parentDir+"_"+base+".md"),
|
||||
filepath.Join(outputDir, base+".md"),
|
||||
filepath.Join(outputDir, base+"_combined.md"),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -107,16 +107,14 @@ func pdfinfoPageCount(ctx context.Context, pdfPath string) int {
|
|||
return 0
|
||||
}
|
||||
|
||||
// savePdftotextResult writes extracted text to a .md file in the output
|
||||
// directory and caches it. Returns the document tag string.
|
||||
// savePdftotextResult writes extracted text to a .md file in the per-PDF
|
||||
// output directory and caches it. Returns the document tag string.
|
||||
func (al *AgentLoop) savePdftotextResult(
|
||||
pdfPath, text string, pages int, hash string,
|
||||
pdfPath, text string, pages int, hash, outputDir string,
|
||||
) string {
|
||||
outputDir := al.ocrOutputDir()
|
||||
os.MkdirAll(outputDir, 0o755)
|
||||
|
||||
base := strings.TrimSuffix(filepath.Base(pdfPath), filepath.Ext(pdfPath))
|
||||
mdPath := filepath.Join(outputDir, base+"_text.md")
|
||||
mdPath := filepath.Join(outputDir, "document.md")
|
||||
|
||||
if err := os.WriteFile(mdPath, []byte(text), 0o644); err != nil {
|
||||
logger.WarnCF("agent", "Failed to write pdftotext output", map[string]any{
|
||||
|
|
|
|||
|
|
@ -324,6 +324,7 @@ func setupAndStartServices(
|
|||
cfg.Channels.Telegram.AllowFrom,
|
||||
cfg.WorkspacePath(),
|
||||
)
|
||||
handler.SetCacheMutator(dataProvider)
|
||||
agentLoop.OnStateChange = miniappNotifier.Notify
|
||||
if b := agentLoop.GetOrchBroadcaster(); b != nil {
|
||||
handler.SetOrchBroadcaster(b)
|
||||
|
|
@ -908,6 +909,14 @@ func (p *agentLoopDataProvider) ListMediaCache(entryType string) []miniapp.Media
|
|||
return entries
|
||||
}
|
||||
|
||||
func (p *agentLoopDataProvider) DeleteMediaCache(hash string) error {
|
||||
return p.loop.DeleteMediaCache(hash)
|
||||
}
|
||||
|
||||
func (p *agentLoopDataProvider) DeleteAllMediaCache() (int64, error) {
|
||||
return p.loop.DeleteAllMediaCache()
|
||||
}
|
||||
|
||||
func (p *agentLoopDataProvider) GetGitRepos() []miniapp.GitRepoSummary {
|
||||
if time.Since(p.gitReposCacheAt) < gitCacheTTL {
|
||||
return p.gitReposCache
|
||||
|
|
|
|||
|
|
@ -183,6 +183,39 @@ func (c *Cache) Prune(ttl time.Duration) (int64, error) {
|
|||
return res.RowsAffected()
|
||||
}
|
||||
|
||||
// Delete removes a single cache entry by hash and type.
|
||||
// Returns the entry before deletion so callers can clean up associated files.
|
||||
func (c *Cache) Delete(hash, entryType string) (Entry, error) {
|
||||
var entry Entry
|
||||
var filePath sql.NullString
|
||||
row := c.db.QueryRow(
|
||||
`SELECT result, file_path, pages FROM media_cache WHERE hash = ? AND type = ?`,
|
||||
hash, entryType,
|
||||
)
|
||||
if err := row.Scan(&entry.Result, &filePath, &entry.Pages); err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return Entry{}, nil
|
||||
}
|
||||
return Entry{}, err
|
||||
}
|
||||
entry.FilePath = filePath.String
|
||||
|
||||
_, err := c.db.Exec(
|
||||
`DELETE FROM media_cache WHERE hash = ? AND type = ?`,
|
||||
hash, entryType,
|
||||
)
|
||||
return entry, err
|
||||
}
|
||||
|
||||
// DeleteAll removes all cache entries. Returns the number of entries removed.
|
||||
func (c *Cache) DeleteAll() (int64, error) {
|
||||
res, err := c.db.Exec(`DELETE FROM media_cache`)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return res.RowsAffected()
|
||||
}
|
||||
|
||||
func (c *Cache) touchAccessed(hash, entryType string) {
|
||||
_, _ = c.db.Exec(
|
||||
`UPDATE media_cache SET accessed_at = ? WHERE hash = ? AND type = ?`,
|
||||
|
|
|
|||
|
|
@ -166,6 +166,60 @@ func TestCache_GetEntry_Miss(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestCache_Delete(t *testing.T) {
|
||||
c := openTestCache(t)
|
||||
hash := HashData([]byte("delete-me"))
|
||||
|
||||
_ = c.PutEntry(hash, TypePDFOCR, Entry{Result: "preview", FilePath: "/tmp/test.md", Pages: 3})
|
||||
|
||||
entry, err := c.Delete(hash, TypePDFOCR)
|
||||
if err != nil {
|
||||
t.Fatalf("Delete: %v", err)
|
||||
}
|
||||
if entry.FilePath != "/tmp/test.md" {
|
||||
t.Errorf("returned FilePath = %q", entry.FilePath)
|
||||
}
|
||||
if entry.Pages != 3 {
|
||||
t.Errorf("returned Pages = %d", entry.Pages)
|
||||
}
|
||||
|
||||
if _, ok := c.GetEntry(hash, TypePDFOCR); ok {
|
||||
t.Error("entry should be deleted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCache_Delete_NotFound(t *testing.T) {
|
||||
c := openTestCache(t)
|
||||
entry, err := c.Delete("nonexistent", TypePDFOCR)
|
||||
if err != nil {
|
||||
t.Fatalf("Delete: %v", err)
|
||||
}
|
||||
if entry.FilePath != "" {
|
||||
t.Error("expected empty entry for not-found")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCache_DeleteAll(t *testing.T) {
|
||||
c := openTestCache(t)
|
||||
|
||||
_ = c.Put(HashData([]byte("a")), TypeImageDesc, "desc1")
|
||||
_ = c.Put(HashData([]byte("b")), TypeImageDesc, "desc2")
|
||||
_ = c.PutEntry(HashData([]byte("c")), TypePDFOCR, Entry{Result: "pdf"})
|
||||
|
||||
n, err := c.DeleteAll()
|
||||
if err != nil {
|
||||
t.Fatalf("DeleteAll: %v", err)
|
||||
}
|
||||
if n != 3 {
|
||||
t.Errorf("deleted %d, want 3", n)
|
||||
}
|
||||
|
||||
entries, _ := c.List("")
|
||||
if len(entries) != 0 {
|
||||
t.Errorf("list should be empty, got %d", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCache_SimpleGetIgnoresFilePath(t *testing.T) {
|
||||
// Simple Get/Put should still work with the new schema
|
||||
c := openTestCache(t)
|
||||
|
|
|
|||
|
|
@ -286,12 +286,50 @@ func writeWorktreeAPIError(w http.ResponseWriter, err error) bool {
|
|||
|
||||
// apiDevConsole receives console output from dev preview iframes.
|
||||
|
||||
// apiCache returns a list of media cache entries.
|
||||
// apiCache dispatches GET (list) and DELETE (clear all) for media cache.
|
||||
func (h *Handler) apiCache(w http.ResponseWriter, r *http.Request) {
|
||||
entryType := r.URL.Query().Get("type")
|
||||
entries := h.provider.ListMediaCache(entryType)
|
||||
if entries == nil {
|
||||
entries = []MediaCacheEntry{}
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
entryType := r.URL.Query().Get("type")
|
||||
entries := h.provider.ListMediaCache(entryType)
|
||||
if entries == nil {
|
||||
entries = []MediaCacheEntry{}
|
||||
}
|
||||
writeJSON(w, entries)
|
||||
case http.MethodDelete:
|
||||
if h.cacheMutator == nil {
|
||||
http.Error(w, `{"error":"not supported"}`, http.StatusNotImplemented)
|
||||
return
|
||||
}
|
||||
n, err := h.cacheMutator.DeleteAllMediaCache()
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"failed to delete cache"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, map[string]any{"deleted": n})
|
||||
default:
|
||||
http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed)
|
||||
}
|
||||
writeJSON(w, entries)
|
||||
}
|
||||
|
||||
// apiCacheEntry handles DELETE for a single cache entry.
|
||||
func (h *Handler) apiCacheEntry(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodDelete {
|
||||
http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
if h.cacheMutator == nil {
|
||||
http.Error(w, `{"error":"not supported"}`, http.StatusNotImplemented)
|
||||
return
|
||||
}
|
||||
hash := r.URL.Path[len("/miniapp/api/cache/"):]
|
||||
if hash == "" {
|
||||
http.Error(w, `{"error":"hash required"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.cacheMutator.DeleteMediaCache(hash); err != nil {
|
||||
http.Error(w, `{"error":"failed to delete entry"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, map[string]string{"status": "ok"})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ func mustMiniappStaticFS() fs.FS {
|
|||
// Handler serves the Mini App HTML and API endpoints.
|
||||
type Handler struct {
|
||||
provider DataProvider
|
||||
cacheMutator CacheMutator
|
||||
sender CommandSender
|
||||
botToken string
|
||||
notifier *StateNotifier
|
||||
|
|
@ -84,6 +85,11 @@ func (h *Handler) SetOrchBroadcaster(b *orch.Broadcaster) {
|
|||
h.orchBroadcaster = b
|
||||
}
|
||||
|
||||
// SetCacheMutator enables cache mutation operations (delete entry/clear all).
|
||||
func (h *Handler) SetCacheMutator(m CacheMutator) {
|
||||
h.cacheMutator = m
|
||||
}
|
||||
|
||||
func (h *Handler) handleProtectedFunc(mux *http.ServeMux, pattern string, handler http.HandlerFunc) {
|
||||
mux.HandleFunc(pattern, h.requireAuth(handler))
|
||||
}
|
||||
|
|
@ -112,6 +118,7 @@ func (h *Handler) RegisterRoutes(mux *http.ServeMux) {
|
|||
mux.HandleFunc("/miniapp/dev/console", h.apiDevConsole)
|
||||
mux.HandleFunc("/miniapp/dev/", h.serveDevProxy)
|
||||
h.handleProtectedFunc(mux, "/miniapp/api/cache", h.apiCache)
|
||||
h.handleProtectedFunc(mux, "/miniapp/api/cache/", h.apiCacheEntry)
|
||||
h.handleProtectedFunc(mux, "/miniapp/api/research", h.apiResearch)
|
||||
h.handleProtectedFunc(mux, "/miniapp/api/research/focus", h.apiResearchFocus)
|
||||
h.handleProtectedFunc(mux, "/miniapp/api/research/", h.apiResearchDetail)
|
||||
|
|
|
|||
|
|
@ -125,7 +125,7 @@ type MediaCacheEntry struct {
|
|||
AccessedAt string `json:"accessed_at"`
|
||||
}
|
||||
|
||||
// DataProvider is the read-only interface to agent state for the Mini App API.
|
||||
// DataProvider is the interface to agent state for the Mini App API.
|
||||
type DataProvider interface {
|
||||
ListSkills() []skills.SkillInfo
|
||||
GetPlanInfo() PlanInfo
|
||||
|
|
@ -139,6 +139,12 @@ type DataProvider interface {
|
|||
ListMediaCache(entryType string) []MediaCacheEntry
|
||||
}
|
||||
|
||||
// CacheMutator extends DataProvider with cache mutation operations.
|
||||
type CacheMutator interface {
|
||||
DeleteMediaCache(hash string) error
|
||||
DeleteAllMediaCache() (int64, error)
|
||||
}
|
||||
|
||||
// CommandSender injects a command into the message bus on behalf of a user.
|
||||
type CommandSender interface {
|
||||
SendCommand(senderID, chatID, command string)
|
||||
|
|
|
|||
|
|
@ -34,13 +34,19 @@ func (h *Handler) openMediaCache() (*mediacache.Cache, error) {
|
|||
return mediacache.Open(filepath.Join(ws, "media_cache.db"))
|
||||
}
|
||||
|
||||
// handleMediaCache lists all media cache entries.
|
||||
// handleMediaCache dispatches GET (list) and DELETE (clear all) for media cache.
|
||||
func (h *Handler) handleMediaCache(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
h.listMediaCache(w, r)
|
||||
case http.MethodDelete:
|
||||
h.deleteAllMediaCache(w, r)
|
||||
default:
|
||||
http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Handler) listMediaCache(w http.ResponseWriter, r *http.Request) {
|
||||
mc, err := h.openMediaCache()
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"media cache not available"}`, http.StatusServiceUnavailable)
|
||||
|
|
@ -72,20 +78,51 @@ func (h *Handler) handleMediaCache(w http.ResponseWriter, r *http.Request) {
|
|||
json.NewEncoder(w).Encode(result)
|
||||
}
|
||||
|
||||
// handleMediaCacheContent serves the full file content for a PDF OCR entry.
|
||||
// GET /api/media-cache/{hash}
|
||||
func (h *Handler) handleMediaCacheContent(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed)
|
||||
func (h *Handler) deleteAllMediaCache(w http.ResponseWriter, _ *http.Request) {
|
||||
mc, err := h.openMediaCache()
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"media cache not available"}`, http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
defer mc.Close()
|
||||
|
||||
removed, err := mc.DeleteAll()
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"failed to delete cache"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Clean up .ocr_cache directory
|
||||
cfg, _ := config.LoadConfig(h.configPath)
|
||||
if cfg != nil {
|
||||
ocrDir := filepath.Join(cfg.WorkspacePath(), ".ocr_cache")
|
||||
os.RemoveAll(ocrDir)
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]any{"deleted": removed})
|
||||
}
|
||||
|
||||
// handleMediaCacheContent dispatches GET (content) and DELETE (single entry).
|
||||
// /api/media-cache/{hash}
|
||||
func (h *Handler) handleMediaCacheContent(w http.ResponseWriter, r *http.Request) {
|
||||
hash := filepath.Base(r.URL.Path)
|
||||
if hash == "" || hash == "media-cache" {
|
||||
http.Error(w, `{"error":"hash required"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
h.getMediaCacheContent(w, hash)
|
||||
case http.MethodDelete:
|
||||
h.deleteMediaCacheEntry(w, hash)
|
||||
default:
|
||||
http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Handler) getMediaCacheContent(w http.ResponseWriter, hash string) {
|
||||
mc, err := h.openMediaCache()
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"media cache not available"}`, http.StatusServiceUnavailable)
|
||||
|
|
@ -95,7 +132,9 @@ func (h *Handler) handleMediaCacheContent(w http.ResponseWriter, r *http.Request
|
|||
|
||||
entry, ok := mc.GetEntry(hash, mediacache.TypePDFOCR)
|
||||
if !ok {
|
||||
// Try image_desc
|
||||
entry, ok = mc.GetEntry(hash, mediacache.TypePDFText)
|
||||
}
|
||||
if !ok {
|
||||
result, ok := mc.Get(hash, mediacache.TypeImageDesc)
|
||||
if !ok {
|
||||
http.Error(w, `{"error":"not found"}`, http.StatusNotFound)
|
||||
|
|
@ -110,7 +149,6 @@ func (h *Handler) handleMediaCacheContent(w http.ResponseWriter, r *http.Request
|
|||
return
|
||||
}
|
||||
|
||||
// Read the full markdown file
|
||||
content, err := os.ReadFile(entry.FilePath)
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"file not found"}`, http.StatusNotFound)
|
||||
|
|
@ -126,3 +164,32 @@ func (h *Handler) handleMediaCacheContent(w http.ResponseWriter, r *http.Request
|
|||
"pages": entry.Pages,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *Handler) deleteMediaCacheEntry(w http.ResponseWriter, hash string) {
|
||||
mc, err := h.openMediaCache()
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"media cache not available"}`, http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
defer mc.Close()
|
||||
|
||||
// Delete all types for this hash, clean up files
|
||||
for _, t := range []string{mediacache.TypePDFOCR, mediacache.TypePDFText, mediacache.TypeImageDesc} {
|
||||
entry, err := mc.Delete(hash, t)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if entry.FilePath != "" {
|
||||
// Remove the per-hash subdirectory if it exists
|
||||
dir := filepath.Dir(entry.FilePath)
|
||||
if filepath.Base(dir) == hash {
|
||||
os.RemoveAll(dir)
|
||||
} else {
|
||||
os.Remove(entry.FilePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,3 +38,16 @@ export async function getMediaCacheContent(
|
|||
`/api/media-cache/${encodeURIComponent(hash)}`,
|
||||
)
|
||||
}
|
||||
|
||||
export async function deleteMediaCacheEntry(hash: string): Promise<void> {
|
||||
const res = await fetch(`/api/media-cache/${encodeURIComponent(hash)}`, {
|
||||
method: "DELETE",
|
||||
})
|
||||
if (!res.ok) throw new Error(`API error: ${res.status}`)
|
||||
}
|
||||
|
||||
export async function deleteAllMediaCache(): Promise<{ deleted: number }> {
|
||||
const res = await fetch("/api/media-cache", { method: "DELETE" })
|
||||
if (!res.ok) throw new Error(`API error: ${res.status}`)
|
||||
return res.json() as Promise<{ deleted: number }>
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,13 +1,16 @@
|
|||
import {
|
||||
IconFileText,
|
||||
IconPhoto,
|
||||
IconTrash,
|
||||
} from "@tabler/icons-react"
|
||||
import { useQuery } from "@tanstack/react-query"
|
||||
import { useQuery, useQueryClient } from "@tanstack/react-query"
|
||||
import * as React from "react"
|
||||
|
||||
import {
|
||||
type MediaCacheContent,
|
||||
type MediaCacheEntry,
|
||||
deleteAllMediaCache,
|
||||
deleteMediaCacheEntry,
|
||||
getMediaCacheContent,
|
||||
getMediaCacheEntries,
|
||||
} from "@/api/media-cache"
|
||||
|
|
@ -24,6 +27,7 @@ import { cn } from "@/lib/utils"
|
|||
export function MediaCachePage() {
|
||||
const [typeFilter, setTypeFilter] = React.useState<string>("")
|
||||
const [expandedHash, setExpandedHash] = React.useState<string | null>(null)
|
||||
const queryClient = useQueryClient()
|
||||
|
||||
const { data: entries, isLoading, error } = useQuery({
|
||||
queryKey: ["media-cache", typeFilter],
|
||||
|
|
@ -31,11 +35,24 @@ export function MediaCachePage() {
|
|||
refetchInterval: 30000,
|
||||
})
|
||||
|
||||
const handleDeleteAll = async () => {
|
||||
if (!confirm("Delete all cached media?")) return
|
||||
await deleteAllMediaCache()
|
||||
setExpandedHash(null)
|
||||
queryClient.invalidateQueries({ queryKey: ["media-cache"] })
|
||||
}
|
||||
|
||||
const handleDeleteEntry = async (hash: string) => {
|
||||
await deleteMediaCacheEntry(hash)
|
||||
if (expandedHash === hash) setExpandedHash(null)
|
||||
queryClient.invalidateQueries({ queryKey: ["media-cache"] })
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex-1 overflow-auto px-6 py-3">
|
||||
<div className="w-full max-w-6xl space-y-4">
|
||||
{/* Type filter */}
|
||||
<div className="flex gap-2">
|
||||
{/* Type filter + clear all */}
|
||||
<div className="flex items-center gap-2">
|
||||
<FilterButton
|
||||
active={typeFilter === ""}
|
||||
onClick={() => setTypeFilter("")}
|
||||
|
|
@ -56,6 +73,17 @@ export function MediaCachePage() {
|
|||
<IconFileText className="size-3.5" />
|
||||
PDF
|
||||
</FilterButton>
|
||||
{entries && entries.length > 0 && (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className="text-destructive ml-auto gap-1"
|
||||
onClick={handleDeleteAll}
|
||||
>
|
||||
<IconTrash className="size-3.5" />
|
||||
Clear All
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{isLoading ? (
|
||||
|
|
@ -82,6 +110,7 @@ export function MediaCachePage() {
|
|||
expandedHash === entry.hash ? null : entry.hash,
|
||||
)
|
||||
}
|
||||
onDelete={() => handleDeleteEntry(entry.hash)}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
|
|
@ -116,10 +145,12 @@ function MediaEntry({
|
|||
entry,
|
||||
expanded,
|
||||
onToggle,
|
||||
onDelete,
|
||||
}: {
|
||||
entry: MediaCacheEntry
|
||||
expanded: boolean
|
||||
onToggle: () => void
|
||||
onDelete: () => void
|
||||
}) {
|
||||
const isImage = entry.type === "image_desc"
|
||||
const Icon = isImage ? IconPhoto : IconFileText
|
||||
|
|
@ -163,14 +194,20 @@ function MediaEntry({
|
|||
</CardHeader>
|
||||
{expanded && (
|
||||
<CardContent className="border-t pt-3">
|
||||
<ExpandedContent entry={entry} />
|
||||
<ExpandedContent entry={entry} onDelete={onDelete} />
|
||||
</CardContent>
|
||||
)}
|
||||
</Card>
|
||||
)
|
||||
}
|
||||
|
||||
function ExpandedContent({ entry }: { entry: MediaCacheEntry }) {
|
||||
function ExpandedContent({
|
||||
entry,
|
||||
onDelete,
|
||||
}: {
|
||||
entry: MediaCacheEntry
|
||||
onDelete: () => void
|
||||
}) {
|
||||
const isPDF = entry.type === "pdf_ocr"
|
||||
|
||||
const { data, isLoading } = useQuery({
|
||||
|
|
@ -179,49 +216,58 @@ function ExpandedContent({ entry }: { entry: MediaCacheEntry }) {
|
|||
enabled: isPDF, // only fetch full content for PDFs
|
||||
})
|
||||
|
||||
if (!isPDF) {
|
||||
// Image description: show full result inline
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
<div className="text-muted-foreground text-xs font-medium">
|
||||
Description
|
||||
</div>
|
||||
<div className="bg-muted rounded-md p-3 text-sm whitespace-pre-wrap">
|
||||
{entry.result}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// PDF OCR: show preview + full content on demand
|
||||
return (
|
||||
<div className="space-y-3">
|
||||
<div className="space-y-1">
|
||||
<div className="text-muted-foreground text-xs font-medium">Preview</div>
|
||||
<div className="bg-muted rounded-md p-3 text-sm whitespace-pre-wrap">
|
||||
{entry.result}
|
||||
</div>
|
||||
</div>
|
||||
{entry.file_path && (
|
||||
<div className="text-muted-foreground flex items-center gap-1 text-xs">
|
||||
<IconFileText className="size-3" />
|
||||
<span className="font-mono">{entry.file_path}</span>
|
||||
</div>
|
||||
)}
|
||||
{isLoading ? (
|
||||
<div className="text-muted-foreground py-2 text-sm">
|
||||
Loading full content...
|
||||
</div>
|
||||
) : data?.content ? (
|
||||
<div className="space-y-1">
|
||||
{!isPDF ? (
|
||||
<div className="space-y-2">
|
||||
<div className="text-muted-foreground text-xs font-medium">
|
||||
Full OCR Content
|
||||
Description
|
||||
</div>
|
||||
<div className="bg-muted max-h-96 overflow-auto rounded-md p-3 text-sm whitespace-pre-wrap">
|
||||
{data.content}
|
||||
<div className="bg-muted rounded-md p-3 text-sm whitespace-pre-wrap">
|
||||
{entry.result}
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
) : (
|
||||
<>
|
||||
<div className="space-y-1">
|
||||
<div className="text-muted-foreground text-xs font-medium">Preview</div>
|
||||
<div className="bg-muted rounded-md p-3 text-sm whitespace-pre-wrap">
|
||||
{entry.result}
|
||||
</div>
|
||||
</div>
|
||||
{entry.file_path && (
|
||||
<div className="text-muted-foreground flex items-center gap-1 text-xs">
|
||||
<IconFileText className="size-3" />
|
||||
<span className="font-mono">{entry.file_path}</span>
|
||||
</div>
|
||||
)}
|
||||
{isLoading ? (
|
||||
<div className="text-muted-foreground py-2 text-sm">
|
||||
Loading full content...
|
||||
</div>
|
||||
) : data?.content ? (
|
||||
<div className="space-y-1">
|
||||
<div className="text-muted-foreground text-xs font-medium">
|
||||
Full OCR Content
|
||||
</div>
|
||||
<div className="bg-muted max-h-96 overflow-auto rounded-md p-3 text-sm whitespace-pre-wrap">
|
||||
{data.content}
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
</>
|
||||
)}
|
||||
<div className="flex justify-end">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className="text-destructive gap-1"
|
||||
onClick={onDelete}
|
||||
>
|
||||
<IconTrash className="size-3.5" />
|
||||
Delete
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue