From e1687b74fa36d20311dacff4f6cc76764bbf5a07 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 23 Feb 2026 16:20:03 +0800 Subject: [PATCH] Address reviewer feedback: Improve WebFetchTool text extraction --- pkg/tools/web.go | 101 ++++++++++++++++++++++-------------------- pkg/tools/web_test.go | 55 +++++++++++++++++++++++ 2 files changed, 107 insertions(+), 49 deletions(-) diff --git a/pkg/tools/web.go b/pkg/tools/web.go index 1e3f38479..12710b1e4 100644 --- a/pkg/tools/web.go +++ b/pkg/tools/web.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "html" "io" "net/http" "net/url" @@ -17,6 +18,35 @@ const ( userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" ) +// Package-level regexp variables for better performance +var ( + scriptTagRegex = regexp.MustCompile(``) + styleTagRegex = regexp.MustCompile(``) + allTagsRegex = regexp.MustCompile(`<[^>]+>`) + whitespaceRegex = regexp.MustCompile(`[^\S\n]+`) + newlineRegex = regexp.MustCompile(`\n{3,}`) + + // Block tag regexes + blockTags = []string{"div", "p", "h1", "h2", "h3", "h4", "h5", "h6", "ul", "ol", "li", "table", "tr", "td", "th", "blockquote", "pre", "section", "article", "header", "footer"} + + // Compile regexes for block tags once + blockTagRegexes []*regexp.Regexp +) + +func init() { + // Precompile regexes for block tags + blockTagRegexes = make([]*regexp.Regexp, 0, len(blockTags)*2) + for _, tag := range blockTags { + // Opening tag regex + openingRegex := regexp.MustCompile(`<` + tag + `[^>]*>`) + blockTagRegexes = append(blockTagRegexes, openingRegex) + + // Closing tag regex + closingRegex := regexp.MustCompile(``) + blockTagRegexes = append(blockTagRegexes, closingRegex) + } +} + type SearchProvider interface { Search(ctx context.Context, query string, count int) (string, error) } @@ -590,26 +620,26 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe } func (t *WebFetchTool) extractText(htmlContent string) string { - // 首先移除 script 和 style 标签 - re := regexp.MustCompile(``) - result := re.ReplaceAllLiteralString(htmlContent, "") - re = regexp.MustCompile(``) - result = re.ReplaceAllLiteralString(result, "") + // First remove script and style tags + result := scriptTagRegex.ReplaceAllLiteralString(htmlContent, "") + result = styleTagRegex.ReplaceAllLiteralString(result, "") - // 处理 HTML 实体 - result = t.decodeHTMLEntities(result) + // Handle self-closing tags like
by replacing them with spaces + selfClosingTagsRegex := regexp.MustCompile(`]*>`) + result = selfClosingTagsRegex.ReplaceAllLiteralString(result, " ") - // 移除 HTML 标签，但保留一些结构信息 + // Remove HTML tags but preserve some structure information result = t.removeHTMLTags(result) + + // Handle HTML entities using standard library + result = html.UnescapeString(result) - // 清理空白字符 + // Clean up whitespace result = strings.TrimSpace(result) - re = regexp.MustCompile(`[^\S\n]+`) - result = re.ReplaceAllString(result, " ") - re = regexp.MustCompile(`\n{3,}`) - result = re.ReplaceAllString(result, "\n\n") + result = whitespaceRegex.ReplaceAllString(result, " ") + result = newlineRegex.ReplaceAllString(result, "\n\n") - // 移除空行 + // Remove empty lines lines := strings.Split(result, "\n") var cleanLines []string for _, line := range lines { @@ -622,44 +652,17 @@ func (t *WebFetchTool) extractText(htmlContent string) string { return strings.Join(cleanLines, "\n") } -// decodeHTMLEntities 解码常见的 HTML 实体 -func (t *WebFetchTool) decodeHTMLEntities(s string) string { - entities := map[string]string{ - "&": "&", - "<": "<", - ">": ">", - """: "\"", - "'": "'", - " ": " ", - "©": "©", - "®": "®", - "™": "™", - } - - for entity, char := range entities { - s = strings.ReplaceAll(s, entity, char) - } - - return s -} -// removeHTMLTags 移除 HTML 标签，但尽量保留内容结构 + +// removeHTMLTags removes HTML tags but preserves content structure func (t *WebFetchTool) removeHTMLTags(s string) string { - // 对于块级元素，在移除前添加换行符，以保留结构 - blockTags := []string{"div", "p", "h1", "h2", "h3", "h4", "h5", "h6", "ul", "ol", "li", "table", "tr", "td", "th", "blockquote", "pre", "section", "article", "header", "footer"} - - for _, tag := range blockTags { - // 匹配开始标签 - re := regexp.MustCompile(`<` + tag + `[^>]*>`) - s = re.ReplaceAllString(s, "\n") - // 匹配结束标签 - re = regexp.MustCompile(``) - s = re.ReplaceAllString(s, "\n") + // For block-level elements, add newlines before removal to preserve structure + for _, regex := range blockTagRegexes { + s = regex.ReplaceAllString(s, "\n") } - // 移除所有剩余的 HTML 标签 - re = regexp.MustCompile(`<[^>]+>`) - s = re.ReplaceAllLiteralString(s, "") - + // Remove all remaining HTML tags + s = allTagsRegex.ReplaceAllLiteralString(s, "") + return s } diff --git a/pkg/tools/web_test.go b/pkg/tools/web_test.go index 75e0d8d16..8d130716a 100644 --- a/pkg/tools/web_test.go +++ b/pkg/tools/web_test.go @@ -303,6 +303,61 @@ func TestWebFetchTool_extractText(t *testing.T) { } }, }, + { + name: "HTML entity decoding", + input: "

Hello <world> & "quoted" 'apostrophe'

", + wantFunc: func(t *testing.T, got string) { + if !strings.Contains(got, "Hello & \"quoted\" 'apostrophe'") { + t.Errorf("Expected HTML entities to be decoded, got: %q", got) + } + }, + }, + { + name: "nested tags", + input: "

Header

Paragraph bold text

", + wantFunc: func(t *testing.T, got string) { + if !strings.Contains(got, "Header") || !strings.Contains(got, "Paragraph bold text") { + t.Errorf("Expected nested tags to be handled correctly, got: %q", got) + } + }, + }, + { + name: "self-closing tags", + input: "

Text
with
breaks

\"image\"/", + wantFunc: func(t *testing.T, got string) { + if !strings.Contains(got, "Text with breaks") { + t.Errorf("Expected self-closing tags to be handled correctly, got: %q", got) + } + }, + }, + { + name: "complex HTML with multiple block elements", + input: "

Website

Section 1

Content 1

Section 2

  • Item 1
  • Item 2

Footer

", + wantFunc: func(t *testing.T, got string) { + if !strings.Contains(got, "Website") || !strings.Contains(got, "Section 1") || + !strings.Contains(got, "Item 1") || !strings.Contains(got, "Footer") { + t.Errorf("Expected complex HTML structure to be preserved, got: %q", got) + } + // Should have multiple lines + lines := strings.Split(got, "\n") + if len(lines) < 3 { + t.Errorf("Expected multiple lines for complex structure, got %d: %q", len(lines), got) + } + }, + }, + { + name: "HTML entities in attributes", + input: "Link <with> entity", + wantFunc: func(t *testing.T, got string) { + if !strings.Contains(got, "Link entity") { + t.Errorf("Expected content entities to be decoded, got: %q", got) + } + // Should not contain attribute content + if strings.Contains(got, "param=value") { + t.Errorf("Expected attribute entities to be stripped, got: %q", got) + } + }, + }, } for _, tt := range tests {