From dad372eb30e241904be4bcd3b45fee1889e299e1 Mon Sep 17 00:00:00 2001 From: dj-oyu <68707227+dj-oyu@users.noreply.github.com> Date: Thu, 19 Mar 2026 23:15:05 +0900 Subject: [PATCH] feat: figure mode toggle via message keywords + PDF hint message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When sharing a PDF, include keywords like "figures" or "図版" in the message to enable --figure --figure_letter for in-figure text extraction. Cache keys include the figure mode so both variants are stored separately. A one-time hint message is sent to chat when PDF OCR starts, explaining the figure keyword option. Keywords: figure, figures, with images, 図版, 図付き, 画像付き, 図も Co-Authored-By: Claude Opus 4.6 (1M context) --- pkg/agent/loop_media.go | 66 ++++++++++++++++++++++++++++++------ pkg/agent/loop_media_test.go | 23 +++++++++++-- 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/pkg/agent/loop_media.go b/pkg/agent/loop_media.go index 6474df3d0..c2139b6c0 100644 --- a/pkg/agent/loop_media.go +++ b/pkg/agent/loop_media.go @@ -419,6 +419,27 @@ func injectImageDescriptions(content string, descriptions []string) string { // in the media cache for PDF OCR results. const maxPreviewRunes = 500 +// figureKeywords triggers --figure --figure_letter when found in the message. +var figureKeywords = []string{ + "figure", "figures", "with images", + "図版", "図付き", "画像付き", "図も", +} + +// wantFigures returns true if the message content contains a figure keyword. +func wantFigures(content string) bool { + lower := strings.ToLower(content) + for _, kw := range figureKeywords { + if strings.Contains(lower, kw) { + return true + } + } + return false +} + +// pdfHintMessage is sent once when a PDF is first processed to explain options. +const pdfHintMessage = "PDF OCR in progress. " + + "Tip: include \"figures\" or \"図版\" in your message to extract images and in-figure text." + // processPDFsInMessages finds [file:/path.pdf] tags in messages and replaces // them with [document: preview... (full: /path/to.md, N pages)] tags after // running OCR. A braille spinner with page progress is shown during processing. @@ -433,7 +454,10 @@ func (al *AgentLoop) processPDFsInMessages( if !strings.Contains(m.Content, "[file:") { continue } - result[i].Content = al.replacePDFTags(ctx, m.Content, ocrCfg, channel, chatID) + withFigures := wantFigures(m.Content) + result[i].Content = al.replacePDFTags( + ctx, m.Content, ocrCfg, channel, chatID, withFigures, + ) } return result @@ -445,7 +469,7 @@ const pdfTagPrefix = "[file:" // replacePDFTags finds [file:*.pdf] tags and replaces them with OCR results. func (al *AgentLoop) replacePDFTags( ctx context.Context, content string, ocrCfg *config.OCRConfig, - channel, chatID string, + channel, chatID string, withFigures bool, ) string { var out strings.Builder rest := content @@ -469,7 +493,7 @@ func (al *AgentLoop) replacePDFTags( out.WriteString(rest[:idx]) if strings.HasSuffix(strings.ToLower(path), ".pdf") { - out.WriteString(al.ocrPDF(ctx, path, ocrCfg, channel, chatID)) + out.WriteString(al.ocrPDF(ctx, path, ocrCfg, channel, chatID, withFigures)) } else { out.WriteString(tag) } @@ -482,17 +506,23 @@ func (al *AgentLoop) replacePDFTags( // ocrPDF runs OCR on a PDF file and returns a document tag with preview. // Uses the media cache to avoid redundant OCR runs. +// When withFigures is true, --figure and --figure_letter flags are added. func (al *AgentLoop) ocrPDF( ctx context.Context, pdfPath string, ocrCfg *config.OCRConfig, - channel, chatID string, + channel, chatID string, withFigures bool, ) string { - // Hash the file content for cache lookup + // Hash the file content for cache lookup. + // Include figure mode in the hash so both variants are cached separately. pdfData, err := os.ReadFile(pdfPath) if err != nil { logger.WarnCF("agent", "Failed to read PDF", map[string]any{"path": pdfPath, "error": err.Error()}) return fmt.Sprintf("[file:%s]", pdfPath) } - hash := mediacache.HashData(pdfData) + hashInput := pdfData + if withFigures { + hashInput = append(hashInput, []byte(":figures")...) + } + hash := mediacache.HashData(hashInput) // Check cache if al.mediaCache != nil { @@ -506,9 +536,22 @@ func (al *AgentLoop) ocrPDF( totalPages := mediacache.PDFPageCount(pdfPath) totalStr := mediacache.FormatPageCount(totalPages) - // Start progress indicator + // Send hint message and start progress indicator + if al.bus != nil && channel != "" && chatID != "" { + _ = al.bus.PublishOutbound(ctx, bus.OutboundMessage{ + Channel: channel, + ChatID: chatID, + Content: pdfHintMessage, + SkipPlaceholder: true, + }) + } + + modeLabel := "Processing PDF" + if withFigures { + modeLabel = "Processing PDF (with figures)" + } indicator := al.processingIndicator(ctx, channel, chatID, - fmt.Sprintf("Processing PDF (0/%s)...", totalStr)) + fmt.Sprintf("%s (0/%s)...", modeLabel, totalStr)) defer indicator.Stop() // Determine output directory for OCR results @@ -520,8 +563,11 @@ func (al *AgentLoop) ocrPDF( cmdCtx, cmdCancel := context.WithTimeout(ctx, timeout) defer cmdCancel() - args := make([]string, 0, len(ocrCfg.Args)+4) + args := make([]string, 0, len(ocrCfg.Args)+6) args = append(args, ocrCfg.Args...) + if withFigures { + args = append(args, "--figure", "--figure_letter") + } args = append(args, pdfPath, "-o", outputDir) cmd := exec.CommandContext(cmdCtx, ocrCfg.Command, args...) @@ -556,7 +602,7 @@ func (al *AgentLoop) ocrPDF( line := scanner.Text() if strings.Contains(line, "TextDetector __call__") { page++ - indicator.UpdateLabel(fmt.Sprintf("Processing PDF (%d/%s)...", page, totalStr)) + indicator.UpdateLabel(fmt.Sprintf("%s (%d/%s)...", modeLabel, page, totalStr)) } } diff --git a/pkg/agent/loop_media_test.go b/pkg/agent/loop_media_test.go index e21e89b3b..4b2456659 100644 --- a/pkg/agent/loop_media_test.go +++ b/pkg/agent/loop_media_test.go @@ -151,7 +151,7 @@ func TestFormatDocumentTag_UnknownPages(t *testing.T) { func TestReplacePDFTags_NoPDF(t *testing.T) { al := &AgentLoop{} content := "Check this out [file:/path/to/audio.mp3]" - result := al.replacePDFTags(t.Context(), content, &config.OCRConfig{Command: "echo"}, "", "") + result := al.replacePDFTags(t.Context(), content, &config.OCRConfig{Command: "echo"}, "", "", false) if result != content { t.Errorf("non-PDF should be unchanged, got %q", result) } @@ -160,7 +160,7 @@ func TestReplacePDFTags_NoPDF(t *testing.T) { func TestReplacePDFTags_NoTags(t *testing.T) { al := &AgentLoop{} content := "Hello world" - result := al.replacePDFTags(t.Context(), content, &config.OCRConfig{Command: "echo"}, "", "") + result := al.replacePDFTags(t.Context(), content, &config.OCRConfig{Command: "echo"}, "", "", false) if result != content { t.Errorf("no tags should be unchanged, got %q", result) } @@ -177,3 +177,22 @@ func TestProcessPDFs_NilOCR(t *testing.T) { t.Errorf("content should be unchanged when OCR config is nil") } } + +func TestWantFigures(t *testing.T) { + tests := []struct { + content string + want bool + }{ + {"check this pdf", false}, + {"extract with figures please", true}, + {"Figures included", true}, + {"figure mode", true}, + {"with images", true}, + {"", false}, + } + for _, tt := range tests { + if got := wantFigures(tt.content); got != tt.want { + t.Errorf("wantFigures(%q) = %v, want %v", tt.content, got, tt.want) + } + } +}