diff --git a/go.mod b/go.mod index ac288e56d..520cb2b1c 100644 --- a/go.mod +++ b/go.mod @@ -22,13 +22,14 @@ require ( github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1 github.com/openai/openai-go/v3 v3.22.0 github.com/rivo/tview v0.42.0 + github.com/rs/zerolog v1.34.0 github.com/slack-go/slack v0.17.3 github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.11.1 github.com/tencent-connect/botgo v0.2.1 go.mau.fi/whatsmeow v0.0.0-20260219150138-7ae702b1eed4 golang.org/x/oauth2 v0.36.0 - golang.org/x/term v0.40.0 + golang.org/x/term v0.41.0 golang.org/x/time v0.14.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 @@ -54,7 +55,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/rs/zerolog v1.34.0 // indirect github.com/segmentio/asm v1.1.3 // indirect github.com/segmentio/encoding v0.5.3 // indirect github.com/spf13/pflag v1.0.10 // indirect @@ -62,7 +62,7 @@ require ( go.mau.fi/libsignal v0.2.1 // indirect go.mau.fi/util v0.9.6 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/text v0.34.0 // indirect + golang.org/x/text v0.35.0 // indirect modernc.org/libc v1.67.6 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect @@ -92,8 +92,8 @@ require ( github.com/valyala/fastjson v1.6.10 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect golang.org/x/arch v0.24.0 // indirect - golang.org/x/crypto v0.48.0 - golang.org/x/net v0.51.0 // indirect - golang.org/x/sync v0.19.0 // indirect - golang.org/x/sys v0.41.0 // indirect + golang.org/x/crypto v0.49.0 + golang.org/x/net v0.51.0 + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.42.0 // indirect ) diff --git a/go.sum b/go.sum index f0e3fc132..25d941bc7 100644 --- a/go.sum +++ b/go.sum @@ -250,8 +250,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= -golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -284,8 +284,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -307,15 +307,15 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= -golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= +golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -323,8 +323,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/pkg/agent/loop_media.go b/pkg/agent/loop_media.go index 6b0a211f4..12750c2d7 100644 --- a/pkg/agent/loop_media.go +++ b/pkg/agent/loop_media.go @@ -525,16 +525,31 @@ func (al *AgentLoop) ocrPDF( } hash := mediacache.HashData(hashInput) - // Check cache + // Check cache (both text extraction and OCR) if al.mediaCache != nil { + if entry, ok := al.mediaCache.GetEntry(hash, mediacache.TypePDFText); ok { + logger.DebugCF("agent", "PDF text cache hit", map[string]any{"hash": hash}) + return formatDocumentTag(entry.Result, entry.FilePath, entry.Pages) + } if entry, ok := al.mediaCache.GetEntry(hash, mediacache.TypePDFOCR); ok { logger.DebugCF("agent", "PDF OCR cache hit", map[string]any{"hash": hash}) return formatDocumentTag(entry.Result, entry.FilePath, entry.Pages) } } - // Get page count for progress display - totalPages := mediacache.PDFPageCount(pdfPath) + // Fast path: try pdftotext for PDFs with a text layer (skip if figures requested). + // pdftotext is orders of magnitude faster than OCR. + if !withFigures { + if text, pages, ok := tryPdftotextExtract(ctx, pdfPath); ok { + return al.savePdftotextResult(pdfPath, text, pages, hash) + } + } + + // Get page count for progress display (use pdfinfo, fall back to regex) + totalPages := pdfinfoPageCount(ctx, pdfPath) + if totalPages == 0 { + totalPages = mediacache.PDFPageCount(pdfPath) + } totalStr := mediacache.FormatPageCount(totalPages) // Send hint message (only if not already sent by Phase 1 waitForPDFFollowUp) diff --git a/pkg/agent/loop_media_pypdf.go b/pkg/agent/loop_media_pypdf.go new file mode 100644 index 000000000..589da2db3 --- /dev/null +++ b/pkg/agent/loop_media_pypdf.go @@ -0,0 +1,141 @@ +package agent + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + "unicode" + + "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/mediacache" +) + +// minTextDensity is the minimum ratio of non-whitespace characters to total +// runes for extracted text to be considered "real" (not garbage/metadata). +const minTextDensity = 0.3 + +// minTextLength is the minimum total rune count of extracted text for the +// fast path to succeed. Very short texts likely mean the PDF is image-based. +const minTextLength = 100 + +// pdftotextTimeout is the maximum time for pdftotext text extraction. +const pdftotextTimeout = 30 * time.Second + +// tryPdftotextExtract attempts to extract text from a PDF using pdftotext +// (poppler-utils). Returns the extracted text, page count, and true if +// successful. Returns "", 0, false if pdftotext is unavailable, PDF has +// no text layer, or the extracted text is too short/garbage. +func tryPdftotextExtract(ctx context.Context, pdfPath string) (text string, pages int, ok bool) { + cmdCtx, cancel := context.WithTimeout(ctx, pdftotextTimeout) + defer cancel() + + // Extract text to stdout. -layout preserves reading order. + cmd := exec.CommandContext(cmdCtx, "pdftotext", "-layout", pdfPath, "-") + out, err := cmd.Output() + if err != nil { + logger.DebugCF("agent", "pdftotext extraction failed", map[string]any{ + "path": pdfPath, + "error": err.Error(), + }) + return "", 0, false + } + + extracted := strings.TrimSpace(string(out)) + + // Get page count via pdfinfo + pages = pdfinfoPageCount(ctx, pdfPath) + + // Check quality: too short or too sparse → probably image-based PDF + runes := []rune(extracted) + if len(runes) < minTextLength { + logger.DebugCF("agent", "pdftotext: text too short, falling back to OCR", map[string]any{ + "path": pdfPath, + "runes": len(runes), + }) + return "", pages, false + } + + nonSpace := 0 + for _, r := range runes { + if !unicode.IsSpace(r) { + nonSpace++ + } + } + density := float64(nonSpace) / float64(len(runes)) + if density < minTextDensity { + logger.DebugCF("agent", "pdftotext: low text density, falling back to OCR", map[string]any{ + "path": pdfPath, + "density": fmt.Sprintf("%.2f", density), + }) + return "", pages, false + } + + logger.InfoCF("agent", "pdftotext: text extraction successful", map[string]any{ + "path": pdfPath, + "pages": pages, + "runes": len(runes), + }) + + return extracted, pages, true +} + +// pdfinfoPageCount runs `pdfinfo` to get the page count of a PDF. +// Returns 0 if pdfinfo is unavailable or fails. +func pdfinfoPageCount(ctx context.Context, pdfPath string) int { + cmdCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + cmd := exec.CommandContext(cmdCtx, "pdfinfo", pdfPath) + out, err := cmd.Output() + if err != nil { + return 0 + } + + for _, line := range strings.Split(string(out), "\n") { + if strings.HasPrefix(line, "Pages:") { + field := strings.TrimSpace(strings.TrimPrefix(line, "Pages:")) + if n, err := strconv.Atoi(field); err == nil { + return n + } + } + } + return 0 +} + +// savePdftotextResult writes extracted text to a .md file in the output +// directory and caches it. Returns the document tag string. +func (al *AgentLoop) savePdftotextResult( + pdfPath, text string, pages int, hash string, +) string { + outputDir := al.ocrOutputDir() + os.MkdirAll(outputDir, 0o755) + + base := strings.TrimSuffix(filepath.Base(pdfPath), filepath.Ext(pdfPath)) + mdPath := filepath.Join(outputDir, base+"_text.md") + + if err := os.WriteFile(mdPath, []byte(text), 0o644); err != nil { + logger.WarnCF("agent", "Failed to write pdftotext output", map[string]any{ + "path": mdPath, + "error": err.Error(), + }) + return fmt.Sprintf("[file:%s]", pdfPath) + } + + preview := extractPreview(text, maxPreviewRunes) + + // Cache the result + if al.mediaCache != nil { + _ = al.mediaCache.PutEntry(hash, mediacache.TypePDFText, mediacache.Entry{ + Result: preview, + FilePath: mdPath, + Pages: pages, + }) + } + + return formatDocumentTag(preview, mdPath, pages) +} diff --git a/pkg/mediacache/cache.go b/pkg/mediacache/cache.go index 91439aedb..ecbfb3bb8 100644 --- a/pkg/mediacache/cache.go +++ b/pkg/mediacache/cache.go @@ -27,6 +27,7 @@ CREATE INDEX IF NOT EXISTS idx_media_cache_accessed ON media_cache(accessed_at); const ( TypeImageDesc = "image_desc" TypePDFOCR = "pdf_ocr" + TypePDFText = "pdf_text" // pypdf text extraction (no OCR) ) // Cache provides SQLite-backed caching for media processing results.