Merge pull request #58 from dj-oyu/feature/pdftotext-fast-path
feat: pdftotext fast path for text-extractable PDFs
This commit is contained in:
dj-oyu
GitHub
commit
85c9b6902a
5 changed files with 177 additions and 20 deletions
14
go.mod
14
go.mod
|
|
@ -22,13 +22,14 @@ require (
|
|||
github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1
|
||||
github.com/openai/openai-go/v3 v3.22.0
|
||||
github.com/rivo/tview v0.42.0
|
||||
github.com/rs/zerolog v1.34.0
|
||||
github.com/slack-go/slack v0.17.3
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/tencent-connect/botgo v0.2.1
|
||||
go.mau.fi/whatsmeow v0.0.0-20260219150138-7ae702b1eed4
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/term v0.40.0
|
||||
golang.org/x/term v0.41.0
|
||||
golang.org/x/time v0.14.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
|
|
@ -54,7 +55,6 @@ require (
|
|||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/rs/zerolog v1.34.0 // indirect
|
||||
github.com/segmentio/asm v1.1.3 // indirect
|
||||
github.com/segmentio/encoding v0.5.3 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
|
|
@ -62,7 +62,7 @@ require (
|
|||
go.mau.fi/libsignal v0.2.1 // indirect
|
||||
go.mau.fi/util v0.9.6 // indirect
|
||||
golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
modernc.org/libc v1.67.6 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
|
|
@ -92,8 +92,8 @@ require (
|
|||
github.com/valyala/fastjson v1.6.10 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
golang.org/x/arch v0.24.0 // indirect
|
||||
golang.org/x/crypto v0.48.0
|
||||
golang.org/x/net v0.51.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/crypto v0.49.0
|
||||
golang.org/x/net v0.51.0
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
)
|
||||
|
|
|
|||
20
go.sum
20
go.sum
|
|
@ -250,8 +250,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
|
|||
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
|
||||
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
|
||||
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
|
||||
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
|
||||
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
|
||||
golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o=
|
||||
golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
|
|
@ -284,8 +284,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
|
|||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
|
|
@ -307,15 +307,15 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
|||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
|
||||
golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
|
||||
golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
|
||||
golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
|
||||
golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
|
|
@ -323,8 +323,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
|||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
|
||||
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
|
|
|
|||
|
|
@ -525,16 +525,31 @@ func (al *AgentLoop) ocrPDF(
|
|||
}
|
||||
hash := mediacache.HashData(hashInput)
|
||||
|
||||
// Check cache
|
||||
// Check cache (both text extraction and OCR)
|
||||
if al.mediaCache != nil {
|
||||
if entry, ok := al.mediaCache.GetEntry(hash, mediacache.TypePDFText); ok {
|
||||
logger.DebugCF("agent", "PDF text cache hit", map[string]any{"hash": hash})
|
||||
return formatDocumentTag(entry.Result, entry.FilePath, entry.Pages)
|
||||
}
|
||||
if entry, ok := al.mediaCache.GetEntry(hash, mediacache.TypePDFOCR); ok {
|
||||
logger.DebugCF("agent", "PDF OCR cache hit", map[string]any{"hash": hash})
|
||||
return formatDocumentTag(entry.Result, entry.FilePath, entry.Pages)
|
||||
}
|
||||
}
|
||||
|
||||
// Get page count for progress display
|
||||
totalPages := mediacache.PDFPageCount(pdfPath)
|
||||
// Fast path: try pdftotext for PDFs with a text layer (skip if figures requested).
|
||||
// pdftotext is orders of magnitude faster than OCR.
|
||||
if !withFigures {
|
||||
if text, pages, ok := tryPdftotextExtract(ctx, pdfPath); ok {
|
||||
return al.savePdftotextResult(pdfPath, text, pages, hash)
|
||||
}
|
||||
}
|
||||
|
||||
// Get page count for progress display (use pdfinfo, fall back to regex)
|
||||
totalPages := pdfinfoPageCount(ctx, pdfPath)
|
||||
if totalPages == 0 {
|
||||
totalPages = mediacache.PDFPageCount(pdfPath)
|
||||
}
|
||||
totalStr := mediacache.FormatPageCount(totalPages)
|
||||
|
||||
// Send hint message (only if not already sent by Phase 1 waitForPDFFollowUp)
|
||||
|
|
|
|||
141
pkg/agent/loop_media_pypdf.go
Normal file
141
pkg/agent/loop_media_pypdf.go
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/sipeed/picoclaw/pkg/logger"
|
||||
"github.com/sipeed/picoclaw/pkg/mediacache"
|
||||
)
|
||||
|
||||
// minTextDensity is the minimum ratio of non-whitespace characters to total
|
||||
// runes for extracted text to be considered "real" (not garbage/metadata).
|
||||
const minTextDensity = 0.3
|
||||
|
||||
// minTextLength is the minimum total rune count of extracted text for the
|
||||
// fast path to succeed. Very short texts likely mean the PDF is image-based.
|
||||
const minTextLength = 100
|
||||
|
||||
// pdftotextTimeout is the maximum time for pdftotext text extraction.
|
||||
const pdftotextTimeout = 30 * time.Second
|
||||
|
||||
// tryPdftotextExtract attempts to extract text from a PDF using pdftotext
|
||||
// (poppler-utils). Returns the extracted text, page count, and true if
|
||||
// successful. Returns "", 0, false if pdftotext is unavailable, PDF has
|
||||
// no text layer, or the extracted text is too short/garbage.
|
||||
func tryPdftotextExtract(ctx context.Context, pdfPath string) (text string, pages int, ok bool) {
|
||||
cmdCtx, cancel := context.WithTimeout(ctx, pdftotextTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Extract text to stdout. -layout preserves reading order.
|
||||
cmd := exec.CommandContext(cmdCtx, "pdftotext", "-layout", pdfPath, "-")
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
logger.DebugCF("agent", "pdftotext extraction failed", map[string]any{
|
||||
"path": pdfPath,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return "", 0, false
|
||||
}
|
||||
|
||||
extracted := strings.TrimSpace(string(out))
|
||||
|
||||
// Get page count via pdfinfo
|
||||
pages = pdfinfoPageCount(ctx, pdfPath)
|
||||
|
||||
// Check quality: too short or too sparse → probably image-based PDF
|
||||
runes := []rune(extracted)
|
||||
if len(runes) < minTextLength {
|
||||
logger.DebugCF("agent", "pdftotext: text too short, falling back to OCR", map[string]any{
|
||||
"path": pdfPath,
|
||||
"runes": len(runes),
|
||||
})
|
||||
return "", pages, false
|
||||
}
|
||||
|
||||
nonSpace := 0
|
||||
for _, r := range runes {
|
||||
if !unicode.IsSpace(r) {
|
||||
nonSpace++
|
||||
}
|
||||
}
|
||||
density := float64(nonSpace) / float64(len(runes))
|
||||
if density < minTextDensity {
|
||||
logger.DebugCF("agent", "pdftotext: low text density, falling back to OCR", map[string]any{
|
||||
"path": pdfPath,
|
||||
"density": fmt.Sprintf("%.2f", density),
|
||||
})
|
||||
return "", pages, false
|
||||
}
|
||||
|
||||
logger.InfoCF("agent", "pdftotext: text extraction successful", map[string]any{
|
||||
"path": pdfPath,
|
||||
"pages": pages,
|
||||
"runes": len(runes),
|
||||
})
|
||||
|
||||
return extracted, pages, true
|
||||
}
|
||||
|
||||
// pdfinfoPageCount runs `pdfinfo` to get the page count of a PDF.
|
||||
// Returns 0 if pdfinfo is unavailable or fails.
|
||||
func pdfinfoPageCount(ctx context.Context, pdfPath string) int {
|
||||
cmdCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(cmdCtx, "pdfinfo", pdfPath)
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
if strings.HasPrefix(line, "Pages:") {
|
||||
field := strings.TrimSpace(strings.TrimPrefix(line, "Pages:"))
|
||||
if n, err := strconv.Atoi(field); err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// savePdftotextResult writes extracted text to a .md file in the output
|
||||
// directory and caches it. Returns the document tag string.
|
||||
func (al *AgentLoop) savePdftotextResult(
|
||||
pdfPath, text string, pages int, hash string,
|
||||
) string {
|
||||
outputDir := al.ocrOutputDir()
|
||||
os.MkdirAll(outputDir, 0o755)
|
||||
|
||||
base := strings.TrimSuffix(filepath.Base(pdfPath), filepath.Ext(pdfPath))
|
||||
mdPath := filepath.Join(outputDir, base+"_text.md")
|
||||
|
||||
if err := os.WriteFile(mdPath, []byte(text), 0o644); err != nil {
|
||||
logger.WarnCF("agent", "Failed to write pdftotext output", map[string]any{
|
||||
"path": mdPath,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return fmt.Sprintf("[file:%s]", pdfPath)
|
||||
}
|
||||
|
||||
preview := extractPreview(text, maxPreviewRunes)
|
||||
|
||||
// Cache the result
|
||||
if al.mediaCache != nil {
|
||||
_ = al.mediaCache.PutEntry(hash, mediacache.TypePDFText, mediacache.Entry{
|
||||
Result: preview,
|
||||
FilePath: mdPath,
|
||||
Pages: pages,
|
||||
})
|
||||
}
|
||||
|
||||
return formatDocumentTag(preview, mdPath, pages)
|
||||
}
|
||||
|
|
@ -27,6 +27,7 @@ CREATE INDEX IF NOT EXISTS idx_media_cache_accessed ON media_cache(accessed_at);
|
|||
const (
|
||||
TypeImageDesc = "image_desc"
|
||||
TypePDFOCR = "pdf_ocr"
|
||||
TypePDFText = "pdf_text" // pypdf text extraction (no OCR)
|
||||
)
|
||||
|
||||
// Cache provides SQLite-backed caching for media processing results.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue