From 2dc2703d6516218f4f01ac32d13929949497e97b Mon Sep 17 00:00:00 2001 From: dj-oyu <68707227+dj-oyu@users.noreply.github.com> Date: Fri, 20 Mar 2026 05:25:02 +0900 Subject: [PATCH] feat: pdftotext fast path for text-extractable PDFs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skip OCR for PDFs that have a text layer by trying pdftotext (poppler-utils) first. Falls back to OCR if: - pdftotext is not installed - Extracted text is too short (<100 chars) or sparse (image-based PDF) - User requested figures mode ("図版" keyword) Also uses pdfinfo for page count instead of regex parsing, with fallback to the existing regex method when pdfinfo is unavailable. Adds TypePDFText cache type so text-extracted results are cached separately from OCR results. Requires poppler-utils on the deployment host (pdftotext + pdfinfo). No new test dependencies — gracefully degrades when tools are absent. Co-Authored-By: Claude Opus 4.6 (1M context) --- go.mod | 14 ++-- go.sum | 20 ++--- pkg/agent/loop_media.go | 21 ++++- pkg/agent/loop_media_pypdf.go | 141 ++++++++++++++++++++++++++++++++++ pkg/mediacache/cache.go | 1 + 5 files changed, 177 insertions(+), 20 deletions(-) create mode 100644 pkg/agent/loop_media_pypdf.go diff --git a/go.mod b/go.mod index ac288e56d..520cb2b1c 100644 --- a/go.mod +++ b/go.mod @@ -22,13 +22,14 @@ require ( github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1 github.com/openai/openai-go/v3 v3.22.0 github.com/rivo/tview v0.42.0 + github.com/rs/zerolog v1.34.0 github.com/slack-go/slack v0.17.3 github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.11.1 github.com/tencent-connect/botgo v0.2.1 go.mau.fi/whatsmeow v0.0.0-20260219150138-7ae702b1eed4 golang.org/x/oauth2 v0.36.0 - golang.org/x/term v0.40.0 + golang.org/x/term v0.41.0 golang.org/x/time v0.14.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 @@ -54,7 +55,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/rs/zerolog v1.34.0 // indirect github.com/segmentio/asm v1.1.3 // indirect github.com/segmentio/encoding v0.5.3 // indirect github.com/spf13/pflag v1.0.10 // indirect @@ -62,7 +62,7 @@ require ( go.mau.fi/libsignal v0.2.1 // indirect go.mau.fi/util v0.9.6 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/text v0.34.0 // indirect + golang.org/x/text v0.35.0 // indirect modernc.org/libc v1.67.6 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect @@ -92,8 +92,8 @@ require ( github.com/valyala/fastjson v1.6.10 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect golang.org/x/arch v0.24.0 // indirect - golang.org/x/crypto v0.48.0 - golang.org/x/net v0.51.0 // indirect - golang.org/x/sync v0.19.0 // indirect - golang.org/x/sys v0.41.0 // indirect + golang.org/x/crypto v0.49.0 + golang.org/x/net v0.51.0 + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.42.0 // indirect ) diff --git a/go.sum b/go.sum index f0e3fc132..25d941bc7 100644 --- a/go.sum +++ b/go.sum @@ -250,8 +250,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= -golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -284,8 +284,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -307,15 +307,15 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= -golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= +golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -323,8 +323,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/pkg/agent/loop_media.go b/pkg/agent/loop_media.go index 6b0a211f4..12750c2d7 100644 --- a/pkg/agent/loop_media.go +++ b/pkg/agent/loop_media.go @@ -525,16 +525,31 @@ func (al *AgentLoop) ocrPDF( } hash := mediacache.HashData(hashInput) - // Check cache + // Check cache (both text extraction and OCR) if al.mediaCache != nil { + if entry, ok := al.mediaCache.GetEntry(hash, mediacache.TypePDFText); ok { + logger.DebugCF("agent", "PDF text cache hit", map[string]any{"hash": hash}) + return formatDocumentTag(entry.Result, entry.FilePath, entry.Pages) + } if entry, ok := al.mediaCache.GetEntry(hash, mediacache.TypePDFOCR); ok { logger.DebugCF("agent", "PDF OCR cache hit", map[string]any{"hash": hash}) return formatDocumentTag(entry.Result, entry.FilePath, entry.Pages) } } - // Get page count for progress display - totalPages := mediacache.PDFPageCount(pdfPath) + // Fast path: try pdftotext for PDFs with a text layer (skip if figures requested). + // pdftotext is orders of magnitude faster than OCR. + if !withFigures { + if text, pages, ok := tryPdftotextExtract(ctx, pdfPath); ok { + return al.savePdftotextResult(pdfPath, text, pages, hash) + } + } + + // Get page count for progress display (use pdfinfo, fall back to regex) + totalPages := pdfinfoPageCount(ctx, pdfPath) + if totalPages == 0 { + totalPages = mediacache.PDFPageCount(pdfPath) + } totalStr := mediacache.FormatPageCount(totalPages) // Send hint message (only if not already sent by Phase 1 waitForPDFFollowUp) diff --git a/pkg/agent/loop_media_pypdf.go b/pkg/agent/loop_media_pypdf.go new file mode 100644 index 000000000..589da2db3 --- /dev/null +++ b/pkg/agent/loop_media_pypdf.go @@ -0,0 +1,141 @@ +package agent + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + "unicode" + + "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/mediacache" +) + +// minTextDensity is the minimum ratio of non-whitespace characters to total +// runes for extracted text to be considered "real" (not garbage/metadata). +const minTextDensity = 0.3 + +// minTextLength is the minimum total rune count of extracted text for the +// fast path to succeed. Very short texts likely mean the PDF is image-based. +const minTextLength = 100 + +// pdftotextTimeout is the maximum time for pdftotext text extraction. +const pdftotextTimeout = 30 * time.Second + +// tryPdftotextExtract attempts to extract text from a PDF using pdftotext +// (poppler-utils). Returns the extracted text, page count, and true if +// successful. Returns "", 0, false if pdftotext is unavailable, PDF has +// no text layer, or the extracted text is too short/garbage. +func tryPdftotextExtract(ctx context.Context, pdfPath string) (text string, pages int, ok bool) { + cmdCtx, cancel := context.WithTimeout(ctx, pdftotextTimeout) + defer cancel() + + // Extract text to stdout. -layout preserves reading order. + cmd := exec.CommandContext(cmdCtx, "pdftotext", "-layout", pdfPath, "-") + out, err := cmd.Output() + if err != nil { + logger.DebugCF("agent", "pdftotext extraction failed", map[string]any{ + "path": pdfPath, + "error": err.Error(), + }) + return "", 0, false + } + + extracted := strings.TrimSpace(string(out)) + + // Get page count via pdfinfo + pages = pdfinfoPageCount(ctx, pdfPath) + + // Check quality: too short or too sparse → probably image-based PDF + runes := []rune(extracted) + if len(runes) < minTextLength { + logger.DebugCF("agent", "pdftotext: text too short, falling back to OCR", map[string]any{ + "path": pdfPath, + "runes": len(runes), + }) + return "", pages, false + } + + nonSpace := 0 + for _, r := range runes { + if !unicode.IsSpace(r) { + nonSpace++ + } + } + density := float64(nonSpace) / float64(len(runes)) + if density < minTextDensity { + logger.DebugCF("agent", "pdftotext: low text density, falling back to OCR", map[string]any{ + "path": pdfPath, + "density": fmt.Sprintf("%.2f", density), + }) + return "", pages, false + } + + logger.InfoCF("agent", "pdftotext: text extraction successful", map[string]any{ + "path": pdfPath, + "pages": pages, + "runes": len(runes), + }) + + return extracted, pages, true +} + +// pdfinfoPageCount runs `pdfinfo` to get the page count of a PDF. +// Returns 0 if pdfinfo is unavailable or fails. +func pdfinfoPageCount(ctx context.Context, pdfPath string) int { + cmdCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + cmd := exec.CommandContext(cmdCtx, "pdfinfo", pdfPath) + out, err := cmd.Output() + if err != nil { + return 0 + } + + for _, line := range strings.Split(string(out), "\n") { + if strings.HasPrefix(line, "Pages:") { + field := strings.TrimSpace(strings.TrimPrefix(line, "Pages:")) + if n, err := strconv.Atoi(field); err == nil { + return n + } + } + } + return 0 +} + +// savePdftotextResult writes extracted text to a .md file in the output +// directory and caches it. Returns the document tag string. +func (al *AgentLoop) savePdftotextResult( + pdfPath, text string, pages int, hash string, +) string { + outputDir := al.ocrOutputDir() + os.MkdirAll(outputDir, 0o755) + + base := strings.TrimSuffix(filepath.Base(pdfPath), filepath.Ext(pdfPath)) + mdPath := filepath.Join(outputDir, base+"_text.md") + + if err := os.WriteFile(mdPath, []byte(text), 0o644); err != nil { + logger.WarnCF("agent", "Failed to write pdftotext output", map[string]any{ + "path": mdPath, + "error": err.Error(), + }) + return fmt.Sprintf("[file:%s]", pdfPath) + } + + preview := extractPreview(text, maxPreviewRunes) + + // Cache the result + if al.mediaCache != nil { + _ = al.mediaCache.PutEntry(hash, mediacache.TypePDFText, mediacache.Entry{ + Result: preview, + FilePath: mdPath, + Pages: pages, + }) + } + + return formatDocumentTag(preview, mdPath, pages) +} diff --git a/pkg/mediacache/cache.go b/pkg/mediacache/cache.go index 91439aedb..ecbfb3bb8 100644 --- a/pkg/mediacache/cache.go +++ b/pkg/mediacache/cache.go @@ -27,6 +27,7 @@ CREATE INDEX IF NOT EXISTS idx_media_cache_accessed ON media_cache(accessed_at); const ( TypeImageDesc = "image_desc" TypePDFOCR = "pdf_ocr" + TypePDFText = "pdf_text" // pypdf text extraction (no OCR) ) // Cache provides SQLite-backed caching for media processing results.