feat: pdftotext fast path for text-extractable PDFs
Skip OCR for PDFs that have a text layer by trying pdftotext (poppler-utils)
first. Falls back to OCR if:
- pdftotext is not installed
- Extracted text is too short (<100 chars) or sparse (image-based PDF)
- User requested figures mode ("図版" keyword)
Also uses pdfinfo for page count instead of regex parsing, with fallback
to the existing regex method when pdfinfo is unavailable.
Adds TypePDFText cache type so text-extracted results are cached separately
from OCR results.
Requires poppler-utils on the deployment host (pdftotext + pdfinfo).
No new test dependencies — gracefully degrades when tools are absent.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
dj-oyu
parent
76d9611b4b
commit
2dc2703d65
5 changed files with 177 additions and 20 deletions
14
go.mod
14
go.mod
|
|
@ -22,13 +22,14 @@ require (
|
|||
github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1
|
||||
github.com/openai/openai-go/v3 v3.22.0
|
||||
github.com/rivo/tview v0.42.0
|
||||
github.com/rs/zerolog v1.34.0
|
||||
github.com/slack-go/slack v0.17.3
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/tencent-connect/botgo v0.2.1
|
||||
go.mau.fi/whatsmeow v0.0.0-20260219150138-7ae702b1eed4
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/term v0.40.0
|
||||
golang.org/x/term v0.41.0
|
||||
golang.org/x/time v0.14.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
|
|
@ -54,7 +55,6 @@ require (
|
|||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/rs/zerolog v1.34.0 // indirect
|
||||
github.com/segmentio/asm v1.1.3 // indirect
|
||||
github.com/segmentio/encoding v0.5.3 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
|
|
@ -62,7 +62,7 @@ require (
|
|||
go.mau.fi/libsignal v0.2.1 // indirect
|
||||
go.mau.fi/util v0.9.6 // indirect
|
||||
golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
modernc.org/libc v1.67.6 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
|
|
@ -92,8 +92,8 @@ require (
|
|||
github.com/valyala/fastjson v1.6.10 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
golang.org/x/arch v0.24.0 // indirect
|
||||
golang.org/x/crypto v0.48.0
|
||||
golang.org/x/net v0.51.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/crypto v0.49.0
|
||||
golang.org/x/net v0.51.0
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
)
|
||||
|
|
|
|||
20
go.sum
20
go.sum
|
|
@ -250,8 +250,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
|
|||
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
|
||||
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
|
||||
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
|
||||
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
|
||||
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
|
||||
golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o=
|
||||
golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
|
|
@ -284,8 +284,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
|
|||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
|
|
@ -307,15 +307,15 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
|||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
|
||||
golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
|
||||
golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
|
||||
golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
|
||||
golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
|
|
@ -323,8 +323,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
|||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
|
||||
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
|
|
|
|||
|
|
@ -525,16 +525,31 @@ func (al *AgentLoop) ocrPDF(
|
|||
}
|
||||
hash := mediacache.HashData(hashInput)
|
||||
|
||||
// Check cache
|
||||
// Check cache (both text extraction and OCR)
|
||||
if al.mediaCache != nil {
|
||||
if entry, ok := al.mediaCache.GetEntry(hash, mediacache.TypePDFText); ok {
|
||||
logger.DebugCF("agent", "PDF text cache hit", map[string]any{"hash": hash})
|
||||
return formatDocumentTag(entry.Result, entry.FilePath, entry.Pages)
|
||||
}
|
||||
if entry, ok := al.mediaCache.GetEntry(hash, mediacache.TypePDFOCR); ok {
|
||||
logger.DebugCF("agent", "PDF OCR cache hit", map[string]any{"hash": hash})
|
||||
return formatDocumentTag(entry.Result, entry.FilePath, entry.Pages)
|
||||
}
|
||||
}
|
||||
|
||||
// Get page count for progress display
|
||||
totalPages := mediacache.PDFPageCount(pdfPath)
|
||||
// Fast path: try pdftotext for PDFs with a text layer (skip if figures requested).
|
||||
// pdftotext is orders of magnitude faster than OCR.
|
||||
if !withFigures {
|
||||
if text, pages, ok := tryPdftotextExtract(ctx, pdfPath); ok {
|
||||
return al.savePdftotextResult(pdfPath, text, pages, hash)
|
||||
}
|
||||
}
|
||||
|
||||
// Get page count for progress display (use pdfinfo, fall back to regex)
|
||||
totalPages := pdfinfoPageCount(ctx, pdfPath)
|
||||
if totalPages == 0 {
|
||||
totalPages = mediacache.PDFPageCount(pdfPath)
|
||||
}
|
||||
totalStr := mediacache.FormatPageCount(totalPages)
|
||||
|
||||
// Send hint message (only if not already sent by Phase 1 waitForPDFFollowUp)
|
||||
|
|
|
|||
141
pkg/agent/loop_media_pypdf.go
Normal file
141
pkg/agent/loop_media_pypdf.go
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/sipeed/picoclaw/pkg/logger"
|
||||
"github.com/sipeed/picoclaw/pkg/mediacache"
|
||||
)
|
||||
|
||||
// minTextDensity is the minimum ratio of non-whitespace characters to total
|
||||
// runes for extracted text to be considered "real" (not garbage/metadata).
|
||||
const minTextDensity = 0.3
|
||||
|
||||
// minTextLength is the minimum total rune count of extracted text for the
|
||||
// fast path to succeed. Very short texts likely mean the PDF is image-based.
|
||||
const minTextLength = 100
|
||||
|
||||
// pdftotextTimeout is the maximum time for pdftotext text extraction.
|
||||
const pdftotextTimeout = 30 * time.Second
|
||||
|
||||
// tryPdftotextExtract attempts to extract text from a PDF using pdftotext
|
||||
// (poppler-utils). Returns the extracted text, page count, and true if
|
||||
// successful. Returns "", 0, false if pdftotext is unavailable, PDF has
|
||||
// no text layer, or the extracted text is too short/garbage.
|
||||
func tryPdftotextExtract(ctx context.Context, pdfPath string) (text string, pages int, ok bool) {
|
||||
cmdCtx, cancel := context.WithTimeout(ctx, pdftotextTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Extract text to stdout. -layout preserves reading order.
|
||||
cmd := exec.CommandContext(cmdCtx, "pdftotext", "-layout", pdfPath, "-")
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
logger.DebugCF("agent", "pdftotext extraction failed", map[string]any{
|
||||
"path": pdfPath,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return "", 0, false
|
||||
}
|
||||
|
||||
extracted := strings.TrimSpace(string(out))
|
||||
|
||||
// Get page count via pdfinfo
|
||||
pages = pdfinfoPageCount(ctx, pdfPath)
|
||||
|
||||
// Check quality: too short or too sparse → probably image-based PDF
|
||||
runes := []rune(extracted)
|
||||
if len(runes) < minTextLength {
|
||||
logger.DebugCF("agent", "pdftotext: text too short, falling back to OCR", map[string]any{
|
||||
"path": pdfPath,
|
||||
"runes": len(runes),
|
||||
})
|
||||
return "", pages, false
|
||||
}
|
||||
|
||||
nonSpace := 0
|
||||
for _, r := range runes {
|
||||
if !unicode.IsSpace(r) {
|
||||
nonSpace++
|
||||
}
|
||||
}
|
||||
density := float64(nonSpace) / float64(len(runes))
|
||||
if density < minTextDensity {
|
||||
logger.DebugCF("agent", "pdftotext: low text density, falling back to OCR", map[string]any{
|
||||
"path": pdfPath,
|
||||
"density": fmt.Sprintf("%.2f", density),
|
||||
})
|
||||
return "", pages, false
|
||||
}
|
||||
|
||||
logger.InfoCF("agent", "pdftotext: text extraction successful", map[string]any{
|
||||
"path": pdfPath,
|
||||
"pages": pages,
|
||||
"runes": len(runes),
|
||||
})
|
||||
|
||||
return extracted, pages, true
|
||||
}
|
||||
|
||||
// pdfinfoPageCount runs `pdfinfo` to get the page count of a PDF.
|
||||
// Returns 0 if pdfinfo is unavailable or fails.
|
||||
func pdfinfoPageCount(ctx context.Context, pdfPath string) int {
|
||||
cmdCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(cmdCtx, "pdfinfo", pdfPath)
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
if strings.HasPrefix(line, "Pages:") {
|
||||
field := strings.TrimSpace(strings.TrimPrefix(line, "Pages:"))
|
||||
if n, err := strconv.Atoi(field); err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// savePdftotextResult writes extracted text to a .md file in the output
|
||||
// directory and caches it. Returns the document tag string.
|
||||
func (al *AgentLoop) savePdftotextResult(
|
||||
pdfPath, text string, pages int, hash string,
|
||||
) string {
|
||||
outputDir := al.ocrOutputDir()
|
||||
os.MkdirAll(outputDir, 0o755)
|
||||
|
||||
base := strings.TrimSuffix(filepath.Base(pdfPath), filepath.Ext(pdfPath))
|
||||
mdPath := filepath.Join(outputDir, base+"_text.md")
|
||||
|
||||
if err := os.WriteFile(mdPath, []byte(text), 0o644); err != nil {
|
||||
logger.WarnCF("agent", "Failed to write pdftotext output", map[string]any{
|
||||
"path": mdPath,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return fmt.Sprintf("[file:%s]", pdfPath)
|
||||
}
|
||||
|
||||
preview := extractPreview(text, maxPreviewRunes)
|
||||
|
||||
// Cache the result
|
||||
if al.mediaCache != nil {
|
||||
_ = al.mediaCache.PutEntry(hash, mediacache.TypePDFText, mediacache.Entry{
|
||||
Result: preview,
|
||||
FilePath: mdPath,
|
||||
Pages: pages,
|
||||
})
|
||||
}
|
||||
|
||||
return formatDocumentTag(preview, mdPath, pages)
|
||||
}
|
||||
|
|
@ -27,6 +27,7 @@ CREATE INDEX IF NOT EXISTS idx_media_cache_accessed ON media_cache(accessed_at);
|
|||
const (
|
||||
TypeImageDesc = "image_desc"
|
||||
TypePDFOCR = "pdf_ocr"
|
||||
TypePDFText = "pdf_text" // pypdf text extraction (no OCR)
|
||||
)
|
||||
|
||||
// Cache provides SQLite-backed caching for media processing results.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue