feat: figure mode toggle via message keywords + PDF hint message
When sharing a PDF, include keywords like "figures" or "図版" in the message to enable --figure --figure_letter for in-figure text extraction. Cache keys include the figure mode so both variants are stored separately. A one-time hint message is sent to chat when PDF OCR starts, explaining the figure keyword option. Keywords: figure, figures, with images, 図版, 図付き, 画像付き, 図も Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
b4b6c110cc
commit
dad372eb30
2 changed files with 77 additions and 12 deletions
|
|
@ -419,6 +419,27 @@ func injectImageDescriptions(content string, descriptions []string) string {
|
||||||
// in the media cache for PDF OCR results.
|
// in the media cache for PDF OCR results.
|
||||||
const maxPreviewRunes = 500
|
const maxPreviewRunes = 500
|
||||||
|
|
||||||
|
// figureKeywords triggers --figure --figure_letter when found in the message.
|
||||||
|
var figureKeywords = []string{
|
||||||
|
"figure", "figures", "with images",
|
||||||
|
"図版", "図付き", "画像付き", "図も",
|
||||||
|
}
|
||||||
|
|
||||||
|
// wantFigures returns true if the message content contains a figure keyword.
|
||||||
|
func wantFigures(content string) bool {
|
||||||
|
lower := strings.ToLower(content)
|
||||||
|
for _, kw := range figureKeywords {
|
||||||
|
if strings.Contains(lower, kw) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// pdfHintMessage is sent once when a PDF is first processed to explain options.
|
||||||
|
const pdfHintMessage = "PDF OCR in progress. " +
|
||||||
|
"Tip: include \"figures\" or \"図版\" in your message to extract images and in-figure text."
|
||||||
|
|
||||||
// processPDFsInMessages finds [file:/path.pdf] tags in messages and replaces
|
// processPDFsInMessages finds [file:/path.pdf] tags in messages and replaces
|
||||||
// them with [document: preview... (full: /path/to.md, N pages)] tags after
|
// them with [document: preview... (full: /path/to.md, N pages)] tags after
|
||||||
// running OCR. A braille spinner with page progress is shown during processing.
|
// running OCR. A braille spinner with page progress is shown during processing.
|
||||||
|
|
@ -433,7 +454,10 @@ func (al *AgentLoop) processPDFsInMessages(
|
||||||
if !strings.Contains(m.Content, "[file:") {
|
if !strings.Contains(m.Content, "[file:") {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
result[i].Content = al.replacePDFTags(ctx, m.Content, ocrCfg, channel, chatID)
|
withFigures := wantFigures(m.Content)
|
||||||
|
result[i].Content = al.replacePDFTags(
|
||||||
|
ctx, m.Content, ocrCfg, channel, chatID, withFigures,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
@ -445,7 +469,7 @@ const pdfTagPrefix = "[file:"
|
||||||
// replacePDFTags finds [file:*.pdf] tags and replaces them with OCR results.
|
// replacePDFTags finds [file:*.pdf] tags and replaces them with OCR results.
|
||||||
func (al *AgentLoop) replacePDFTags(
|
func (al *AgentLoop) replacePDFTags(
|
||||||
ctx context.Context, content string, ocrCfg *config.OCRConfig,
|
ctx context.Context, content string, ocrCfg *config.OCRConfig,
|
||||||
channel, chatID string,
|
channel, chatID string, withFigures bool,
|
||||||
) string {
|
) string {
|
||||||
var out strings.Builder
|
var out strings.Builder
|
||||||
rest := content
|
rest := content
|
||||||
|
|
@ -469,7 +493,7 @@ func (al *AgentLoop) replacePDFTags(
|
||||||
out.WriteString(rest[:idx])
|
out.WriteString(rest[:idx])
|
||||||
|
|
||||||
if strings.HasSuffix(strings.ToLower(path), ".pdf") {
|
if strings.HasSuffix(strings.ToLower(path), ".pdf") {
|
||||||
out.WriteString(al.ocrPDF(ctx, path, ocrCfg, channel, chatID))
|
out.WriteString(al.ocrPDF(ctx, path, ocrCfg, channel, chatID, withFigures))
|
||||||
} else {
|
} else {
|
||||||
out.WriteString(tag)
|
out.WriteString(tag)
|
||||||
}
|
}
|
||||||
|
|
@ -482,17 +506,23 @@ func (al *AgentLoop) replacePDFTags(
|
||||||
|
|
||||||
// ocrPDF runs OCR on a PDF file and returns a document tag with preview.
|
// ocrPDF runs OCR on a PDF file and returns a document tag with preview.
|
||||||
// Uses the media cache to avoid redundant OCR runs.
|
// Uses the media cache to avoid redundant OCR runs.
|
||||||
|
// When withFigures is true, --figure and --figure_letter flags are added.
|
||||||
func (al *AgentLoop) ocrPDF(
|
func (al *AgentLoop) ocrPDF(
|
||||||
ctx context.Context, pdfPath string, ocrCfg *config.OCRConfig,
|
ctx context.Context, pdfPath string, ocrCfg *config.OCRConfig,
|
||||||
channel, chatID string,
|
channel, chatID string, withFigures bool,
|
||||||
) string {
|
) string {
|
||||||
// Hash the file content for cache lookup
|
// Hash the file content for cache lookup.
|
||||||
|
// Include figure mode in the hash so both variants are cached separately.
|
||||||
pdfData, err := os.ReadFile(pdfPath)
|
pdfData, err := os.ReadFile(pdfPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.WarnCF("agent", "Failed to read PDF", map[string]any{"path": pdfPath, "error": err.Error()})
|
logger.WarnCF("agent", "Failed to read PDF", map[string]any{"path": pdfPath, "error": err.Error()})
|
||||||
return fmt.Sprintf("[file:%s]", pdfPath)
|
return fmt.Sprintf("[file:%s]", pdfPath)
|
||||||
}
|
}
|
||||||
hash := mediacache.HashData(pdfData)
|
hashInput := pdfData
|
||||||
|
if withFigures {
|
||||||
|
hashInput = append(hashInput, []byte(":figures")...)
|
||||||
|
}
|
||||||
|
hash := mediacache.HashData(hashInput)
|
||||||
|
|
||||||
// Check cache
|
// Check cache
|
||||||
if al.mediaCache != nil {
|
if al.mediaCache != nil {
|
||||||
|
|
@ -506,9 +536,22 @@ func (al *AgentLoop) ocrPDF(
|
||||||
totalPages := mediacache.PDFPageCount(pdfPath)
|
totalPages := mediacache.PDFPageCount(pdfPath)
|
||||||
totalStr := mediacache.FormatPageCount(totalPages)
|
totalStr := mediacache.FormatPageCount(totalPages)
|
||||||
|
|
||||||
// Start progress indicator
|
// Send hint message and start progress indicator
|
||||||
|
if al.bus != nil && channel != "" && chatID != "" {
|
||||||
|
_ = al.bus.PublishOutbound(ctx, bus.OutboundMessage{
|
||||||
|
Channel: channel,
|
||||||
|
ChatID: chatID,
|
||||||
|
Content: pdfHintMessage,
|
||||||
|
SkipPlaceholder: true,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
modeLabel := "Processing PDF"
|
||||||
|
if withFigures {
|
||||||
|
modeLabel = "Processing PDF (with figures)"
|
||||||
|
}
|
||||||
indicator := al.processingIndicator(ctx, channel, chatID,
|
indicator := al.processingIndicator(ctx, channel, chatID,
|
||||||
fmt.Sprintf("Processing PDF (0/%s)...", totalStr))
|
fmt.Sprintf("%s (0/%s)...", modeLabel, totalStr))
|
||||||
defer indicator.Stop()
|
defer indicator.Stop()
|
||||||
|
|
||||||
// Determine output directory for OCR results
|
// Determine output directory for OCR results
|
||||||
|
|
@ -520,8 +563,11 @@ func (al *AgentLoop) ocrPDF(
|
||||||
cmdCtx, cmdCancel := context.WithTimeout(ctx, timeout)
|
cmdCtx, cmdCancel := context.WithTimeout(ctx, timeout)
|
||||||
defer cmdCancel()
|
defer cmdCancel()
|
||||||
|
|
||||||
args := make([]string, 0, len(ocrCfg.Args)+4)
|
args := make([]string, 0, len(ocrCfg.Args)+6)
|
||||||
args = append(args, ocrCfg.Args...)
|
args = append(args, ocrCfg.Args...)
|
||||||
|
if withFigures {
|
||||||
|
args = append(args, "--figure", "--figure_letter")
|
||||||
|
}
|
||||||
args = append(args, pdfPath, "-o", outputDir)
|
args = append(args, pdfPath, "-o", outputDir)
|
||||||
|
|
||||||
cmd := exec.CommandContext(cmdCtx, ocrCfg.Command, args...)
|
cmd := exec.CommandContext(cmdCtx, ocrCfg.Command, args...)
|
||||||
|
|
@ -556,7 +602,7 @@ func (al *AgentLoop) ocrPDF(
|
||||||
line := scanner.Text()
|
line := scanner.Text()
|
||||||
if strings.Contains(line, "TextDetector __call__") {
|
if strings.Contains(line, "TextDetector __call__") {
|
||||||
page++
|
page++
|
||||||
indicator.UpdateLabel(fmt.Sprintf("Processing PDF (%d/%s)...", page, totalStr))
|
indicator.UpdateLabel(fmt.Sprintf("%s (%d/%s)...", modeLabel, page, totalStr))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -151,7 +151,7 @@ func TestFormatDocumentTag_UnknownPages(t *testing.T) {
|
||||||
func TestReplacePDFTags_NoPDF(t *testing.T) {
|
func TestReplacePDFTags_NoPDF(t *testing.T) {
|
||||||
al := &AgentLoop{}
|
al := &AgentLoop{}
|
||||||
content := "Check this out [file:/path/to/audio.mp3]"
|
content := "Check this out [file:/path/to/audio.mp3]"
|
||||||
result := al.replacePDFTags(t.Context(), content, &config.OCRConfig{Command: "echo"}, "", "")
|
result := al.replacePDFTags(t.Context(), content, &config.OCRConfig{Command: "echo"}, "", "", false)
|
||||||
if result != content {
|
if result != content {
|
||||||
t.Errorf("non-PDF should be unchanged, got %q", result)
|
t.Errorf("non-PDF should be unchanged, got %q", result)
|
||||||
}
|
}
|
||||||
|
|
@ -160,7 +160,7 @@ func TestReplacePDFTags_NoPDF(t *testing.T) {
|
||||||
func TestReplacePDFTags_NoTags(t *testing.T) {
|
func TestReplacePDFTags_NoTags(t *testing.T) {
|
||||||
al := &AgentLoop{}
|
al := &AgentLoop{}
|
||||||
content := "Hello world"
|
content := "Hello world"
|
||||||
result := al.replacePDFTags(t.Context(), content, &config.OCRConfig{Command: "echo"}, "", "")
|
result := al.replacePDFTags(t.Context(), content, &config.OCRConfig{Command: "echo"}, "", "", false)
|
||||||
if result != content {
|
if result != content {
|
||||||
t.Errorf("no tags should be unchanged, got %q", result)
|
t.Errorf("no tags should be unchanged, got %q", result)
|
||||||
}
|
}
|
||||||
|
|
@ -177,3 +177,22 @@ func TestProcessPDFs_NilOCR(t *testing.T) {
|
||||||
t.Errorf("content should be unchanged when OCR config is nil")
|
t.Errorf("content should be unchanged when OCR config is nil")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestWantFigures(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
content string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"check this pdf", false},
|
||||||
|
{"extract with figures please", true},
|
||||||
|
{"Figures included", true},
|
||||||
|
{"figure mode", true},
|
||||||
|
{"with images", true},
|
||||||
|
{"", false},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
if got := wantFigures(tt.content); got != tt.want {
|
||||||
|
t.Errorf("wantFigures(%q) = %v, want %v", tt.content, got, tt.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue