Address reviewer feedback: Improve WebFetchTool text extraction
This commit is contained in:
Your Name
parent
99ba9051b4
commit
e1687b74fa
2 changed files with 107 additions and 49 deletions
101
pkg/tools/web.go
101
pkg/tools/web.go
|
|
@ -5,6 +5,7 @@ import (
|
|||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
|
@ -17,6 +18,35 @@ const (
|
|||
userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
// Package-level regexp variables for better performance
|
||||
var (
|
||||
scriptTagRegex = regexp.MustCompile(`<script[\s\S]*?</script>`)
|
||||
styleTagRegex = regexp.MustCompile(`<style[\s\S]*?</style>`)
|
||||
allTagsRegex = regexp.MustCompile(`<[^>]+>`)
|
||||
whitespaceRegex = regexp.MustCompile(`[^\S\n]+`)
|
||||
newlineRegex = regexp.MustCompile(`\n{3,}`)
|
||||
|
||||
// Block tag regexes
|
||||
blockTags = []string{"div", "p", "h1", "h2", "h3", "h4", "h5", "h6", "ul", "ol", "li", "table", "tr", "td", "th", "blockquote", "pre", "section", "article", "header", "footer"}
|
||||
|
||||
// Compile regexes for block tags once
|
||||
blockTagRegexes []*regexp.Regexp
|
||||
)
|
||||
|
||||
func init() {
|
||||
// Precompile regexes for block tags
|
||||
blockTagRegexes = make([]*regexp.Regexp, 0, len(blockTags)*2)
|
||||
for _, tag := range blockTags {
|
||||
// Opening tag regex
|
||||
openingRegex := regexp.MustCompile(`<` + tag + `[^>]*>`)
|
||||
blockTagRegexes = append(blockTagRegexes, openingRegex)
|
||||
|
||||
// Closing tag regex
|
||||
closingRegex := regexp.MustCompile(`</` + tag + `>`)
|
||||
blockTagRegexes = append(blockTagRegexes, closingRegex)
|
||||
}
|
||||
}
|
||||
|
||||
type SearchProvider interface {
|
||||
Search(ctx context.Context, query string, count int) (string, error)
|
||||
}
|
||||
|
|
@ -590,26 +620,26 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe
|
|||
}
|
||||
|
||||
func (t *WebFetchTool) extractText(htmlContent string) string {
|
||||
// 首先移除 script 和 style 标签
|
||||
re := regexp.MustCompile(`<script[\s\S]*?</script>`)
|
||||
result := re.ReplaceAllLiteralString(htmlContent, "")
|
||||
re = regexp.MustCompile(`<style[\s\S]*?</style>`)
|
||||
result = re.ReplaceAllLiteralString(result, "")
|
||||
// First remove script and style tags
|
||||
result := scriptTagRegex.ReplaceAllLiteralString(htmlContent, "")
|
||||
result = styleTagRegex.ReplaceAllLiteralString(result, "")
|
||||
|
||||
// 处理 HTML 实体
|
||||
result = t.decodeHTMLEntities(result)
|
||||
// Handle self-closing tags like <br/> by replacing them with spaces
|
||||
selfClosingTagsRegex := regexp.MustCompile(`<br[^>]*>`)
|
||||
result = selfClosingTagsRegex.ReplaceAllLiteralString(result, " ")
|
||||
|
||||
// 移除 HTML 标签，但保留一些结构信息
|
||||
// Remove HTML tags but preserve some structure information
|
||||
result = t.removeHTMLTags(result)
|
||||
|
||||
// Handle HTML entities using standard library
|
||||
result = html.UnescapeString(result)
|
||||
|
||||
// 清理空白字符
|
||||
// Clean up whitespace
|
||||
result = strings.TrimSpace(result)
|
||||
re = regexp.MustCompile(`[^\S\n]+`)
|
||||
result = re.ReplaceAllString(result, " ")
|
||||
re = regexp.MustCompile(`\n{3,}`)
|
||||
result = re.ReplaceAllString(result, "\n\n")
|
||||
result = whitespaceRegex.ReplaceAllString(result, " ")
|
||||
result = newlineRegex.ReplaceAllString(result, "\n\n")
|
||||
|
||||
// 移除空行
|
||||
// Remove empty lines
|
||||
lines := strings.Split(result, "\n")
|
||||
var cleanLines []string
|
||||
for _, line := range lines {
|
||||
|
|
@ -622,44 +652,17 @@ func (t *WebFetchTool) extractText(htmlContent string) string {
|
|||
return strings.Join(cleanLines, "\n")
|
||||
}
|
||||
|
||||
// decodeHTMLEntities 解码常见的 HTML 实体
|
||||
func (t *WebFetchTool) decodeHTMLEntities(s string) string {
|
||||
entities := map[string]string{
|
||||
"&": "&",
|
||||
"<": "<",
|
||||
">": ">",
|
||||
""": "\"",
|
||||
"'": "'",
|
||||
" ": " ",
|
||||
"©": "©",
|
||||
"®": "®",
|
||||
"™": "™",
|
||||
}
|
||||
|
||||
for entity, char := range entities {
|
||||
s = strings.ReplaceAll(s, entity, char)
|
||||
}
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// removeHTMLTags 移除 HTML 标签，但尽量保留内容结构
|
||||
|
||||
// removeHTMLTags removes HTML tags but preserves content structure
|
||||
func (t *WebFetchTool) removeHTMLTags(s string) string {
|
||||
// 对于块级元素，在移除前添加换行符，以保留结构
|
||||
blockTags := []string{"div", "p", "h1", "h2", "h3", "h4", "h5", "h6", "ul", "ol", "li", "table", "tr", "td", "th", "blockquote", "pre", "section", "article", "header", "footer"}
|
||||
|
||||
for _, tag := range blockTags {
|
||||
// 匹配开始标签
|
||||
re := regexp.MustCompile(`<` + tag + `[^>]*>`)
|
||||
s = re.ReplaceAllString(s, "\n")
|
||||
// 匹配结束标签
|
||||
re = regexp.MustCompile(`</` + tag + `>`)
|
||||
s = re.ReplaceAllString(s, "\n")
|
||||
// For block-level elements, add newlines before removal to preserve structure
|
||||
for _, regex := range blockTagRegexes {
|
||||
s = regex.ReplaceAllString(s, "\n")
|
||||
}
|
||||
|
||||
// 移除所有剩余的 HTML 标签
|
||||
re = regexp.MustCompile(`<[^>]+>`)
|
||||
s = re.ReplaceAllLiteralString(s, "")
|
||||
|
||||
// Remove all remaining HTML tags
|
||||
s = allTagsRegex.ReplaceAllLiteralString(s, "")
|
||||
|
||||
return s
|
||||
}
|
||||
|
|
|
|||
|
|
@ -303,6 +303,61 @@ func TestWebFetchTool_extractText(t *testing.T) {
|
|||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "HTML entity decoding",
|
||||
input: "<p>Hello <world> & "quoted" 'apostrophe'</p>",
|
||||
wantFunc: func(t *testing.T, got string) {
|
||||
if !strings.Contains(got, "Hello <world> & \"quoted\" 'apostrophe'") {
|
||||
t.Errorf("Expected HTML entities to be decoded, got: %q", got)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "nested tags",
|
||||
input: "<div><h1>Header</h1><p>Paragraph <strong>bold</strong> text</p></div>",
|
||||
wantFunc: func(t *testing.T, got string) {
|
||||
if !strings.Contains(got, "Header") || !strings.Contains(got, "Paragraph bold text") {
|
||||
t.Errorf("Expected nested tags to be handled correctly, got: %q", got)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "self-closing tags",
|
||||
input: "<p>Text<br/>with<br/>breaks</p><img src=\"image.jpg\" alt=\"image\"/>",
|
||||
wantFunc: func(t *testing.T, got string) {
|
||||
if !strings.Contains(got, "Text with breaks") {
|
||||
t.Errorf("Expected self-closing tags to be handled correctly, got: %q", got)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "complex HTML with multiple block elements",
|
||||
input: "<html><body><header><h1>Website</h1></header><main><section><h2>Section 1</h2><p>Content 1</p></section><section><h2>Section 2</h2><ul><li>Item 1</li><li>Item 2</li></ul></section></main><footer><p>Footer</p></footer></body></html>",
|
||||
wantFunc: func(t *testing.T, got string) {
|
||||
if !strings.Contains(got, "Website") || !strings.Contains(got, "Section 1") ||
|
||||
!strings.Contains(got, "Item 1") || !strings.Contains(got, "Footer") {
|
||||
t.Errorf("Expected complex HTML structure to be preserved, got: %q", got)
|
||||
}
|
||||
// Should have multiple lines
|
||||
lines := strings.Split(got, "\n")
|
||||
if len(lines) < 3 {
|
||||
t.Errorf("Expected multiple lines for complex structure, got %d: %q", len(lines), got)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "HTML entities in attributes",
|
||||
input: "<a href=\"page.html?param=value&other=123\">Link <with> entity</a>",
|
||||
wantFunc: func(t *testing.T, got string) {
|
||||
if !strings.Contains(got, "Link <with> entity") {
|
||||
t.Errorf("Expected content entities to be decoded, got: %q", got)
|
||||
}
|
||||
// Should not contain attribute content
|
||||
if strings.Contains(got, "param=value") {
|
||||
t.Errorf("Expected attribute entities to be stripped, got: %q", got)
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue