diff --git a/pkg/skills/loader.go b/pkg/skills/loader.go index bb0abbdcc..5614d732a 100644 --- a/pkg/skills/loader.go +++ b/pkg/skills/loader.go @@ -13,7 +13,11 @@ import ( "github.com/sipeed/picoclaw/pkg/logger" ) -var namePattern = regexp.MustCompile(`^[a-zA-Z0-9]+(-[a-zA-Z0-9]+)*$`) +var ( + namePattern = regexp.MustCompile(`^[a-zA-Z0-9]+(-[a-zA-Z0-9]+)*$`) + reFrontmatter = regexp.MustCompile(`(?s)^---(?:\r\n|\n|\r)(.*?)(?:\r\n|\n|\r)---`) + reStripFrontmatter = regexp.MustCompile(`(?s)^---(?:\r\n|\n|\r)(.*?)(?:\r\n|\n|\r)---(?:\r\n|\n|\r)*`) +) const ( MaxNameLength = 64 @@ -318,11 +322,7 @@ func (sl *SkillsLoader) parseSimpleYAML(content string) map[string]string { } func (sl *SkillsLoader) extractFrontmatter(content string) string { - // Support \n (Unix), \r\n (Windows), and \r (classic Mac) line endings for frontmatter blocks - // (?s) enables DOTALL so . matches newlines; - // ^--- at start, then ... --- at start of line, honoring all three line ending types - re := regexp.MustCompile(`(?s)^---(?:\r\n|\n|\r)(.*?)(?:\r\n|\n|\r)---`) - match := re.FindStringSubmatch(content) + match := reFrontmatter.FindStringSubmatch(content) if len(match) > 1 { return match[1] } @@ -330,12 +330,7 @@ func (sl *SkillsLoader) extractFrontmatter(content string) string { } func (sl *SkillsLoader) stripFrontmatter(content string) string { - // Support \n (Unix), \r\n (Windows), and \r (classic Mac) line endings for frontmatter blocks - // (?s) enables DOTALL so . matches newlines; - // ^--- at start, then ... --- at start of line, honoring all three line ending types - // Match zero or more trailing line endings after closing --- (handles both with and without blank lines) - re := regexp.MustCompile(`(?s)^---(?:\r\n|\n|\r)(.*?)(?:\r\n|\n|\r)---(?:\r\n|\n|\r)*`) - return re.ReplaceAllString(content, "") + return reStripFrontmatter.ReplaceAllString(content, "") } func escapeXML(s string) string { diff --git a/pkg/tools/web.go b/pkg/tools/web.go index 1f5c58ea5..a5d814475 100644 --- a/pkg/tools/web.go +++ b/pkg/tools/web.go @@ -16,6 +16,19 @@ const ( userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" ) +// Pre-compiled regexes for HTML text extraction +var ( + reScript = regexp.MustCompile(``) + reStyle = regexp.MustCompile(``) + reTags = regexp.MustCompile(`<[^>]+>`) + reWhitespace = regexp.MustCompile(`[^\S\n]+`) + reBlankLines = regexp.MustCompile(`\n{3,}`) + + // DuckDuckGo result extraction + reDDGLink = regexp.MustCompile(`]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)`) + reDDGSnippet = regexp.MustCompile(`([\s\S]*?)`) +) + type SearchProvider interface { Search(ctx context.Context, query string, count int) (string, error) } @@ -118,8 +131,7 @@ func (p *DuckDuckGoSearchProvider) extractResults(html string, count int, query // Try finding the result links directly first, as they are the most critical // Pattern: Title // The previous regex was a bit strict. Let's make it more flexible for attributes order/content - reLink := regexp.MustCompile(`]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)`) - matches := reLink.FindAllStringSubmatch(html, count+5) + matches := reDDGLink.FindAllStringSubmatch(html, count+5) if len(matches) == 0 { return fmt.Sprintf("No results found or extraction failed. Query: %s", query), nil @@ -136,8 +148,7 @@ func (p *DuckDuckGoSearchProvider) extractResults(html string, count int, query // A better regex approach: iterate through text and find matches in order // But for now, let's grab all snippets too - reSnippet := regexp.MustCompile(`([\s\S]*?)`) - snippetMatches := reSnippet.FindAllStringSubmatch(html, count+5) + snippetMatches := reDDGSnippet.FindAllStringSubmatch(html, count+5) maxItems := min(len(matches), count) @@ -172,8 +183,7 @@ func (p *DuckDuckGoSearchProvider) extractResults(html string, count int, query } func stripTags(content string) string { - re := regexp.MustCompile(`<[^>]+>`) - return re.ReplaceAllString(content, "") + return reTags.ReplaceAllString(content, "") } type PerplexitySearchProvider struct { @@ -483,19 +493,14 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]interface{}) } func (t *WebFetchTool) extractText(htmlContent string) string { - re := regexp.MustCompile(``) - result := re.ReplaceAllLiteralString(htmlContent, "") - re = regexp.MustCompile(``) - result = re.ReplaceAllLiteralString(result, "") - re = regexp.MustCompile(`<[^>]+>`) - result = re.ReplaceAllLiteralString(result, "") + result := reScript.ReplaceAllLiteralString(htmlContent, "") + result = reStyle.ReplaceAllLiteralString(result, "") + result = reTags.ReplaceAllLiteralString(result, "") result = strings.TrimSpace(result) - re = regexp.MustCompile(`[^\S\n]+`) - result = re.ReplaceAllString(result, " ") - re = regexp.MustCompile(`\n{3,}`) - result = re.ReplaceAllString(result, "\n\n") + result = reWhitespace.ReplaceAllString(result, " ") + result = reBlankLines.ReplaceAllString(result, "\n\n") lines := strings.Split(result, "\n") var cleanLines []string