Merge pull request #2070 from afjcjsbx/feat/improve-web-tools

feat(tools) time range in web_search
This commit is contained in:
daming大铭 2026-03-27 19:36:24 +08:00 committed by GitHub
commit 25ce52715d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 547 additions and 53 deletions

View file

@ -70,12 +70,12 @@ General settings for fetching and processing webpage content.
Baidu Search uses the [Qianfan AI Search API](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5), which is AI-powered and optimized for Chinese-language queries. Baidu Search uses the [Qianfan AI Search API](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5), which is AI-powered and optimized for Chinese-language queries.
| Config | Type | Default | Description | | Config | Type | Default | Description |
|---------------|--------|------------------------------------------------------------------|---------------------------| |---------------|--------|--------------------------------------------------------|---------------------------|
| `enabled` | bool | false | Enable Baidu Search | | `enabled` | bool | false | Enable Baidu Search |
| `api_key` | string | - | Qianfan API key | | `api_key` | string | - | Qianfan API key |
| `base_url` | string | `https://qianfan.baidubce.com/v2/ai_search/web_search` | Baidu Search API URL | | `base_url` | string | `https://qianfan.baidubce.com/v2/ai_search/web_search` | Baidu Search API URL |
| `max_results` | int | 10 | Maximum number of results | | `max_results` | int | 5 | Maximum number of results |
```json ```json
{ {
@ -107,25 +107,25 @@ Baidu Search uses the [Qianfan AI Search API](https://cloud.baidu.com/doc/qianfa
| `enabled` | bool | false | Enable Tavily search | | `enabled` | bool | false | Enable Tavily search |
| `api_key` | string | - | Tavily API key | | `api_key` | string | - | Tavily API key |
| `base_url` | string | - | Custom Tavily API base URL | | `base_url` | string | - | Custom Tavily API base URL |
| `max_results` | int | 0 | Maximum number of results (0 = default) | | `max_results` | int | 5 | Maximum number of results |
### SearXNG ### SearXNG
| Config | Type | Default | Description | | Config | Type | Default | Description |
|---------------|--------|--------------------------|---------------------------| |---------------|--------|-------------------------|---------------------------|
| `enabled` | bool | false | Enable SearXNG search | | `enabled` | bool | false | Enable SearXNG search |
| `base_url` | string | `http://localhost:8888` | SearXNG instance URL | | `base_url` | string | `http://localhost:8888` | SearXNG instance URL |
| `max_results` | int | 5 | Maximum number of results | | `max_results` | int | 5 | Maximum number of results |
### GLM Search ### GLM Search
| Config | Type | Default | Description | | Config | Type | Default | Description |
|-----------------|--------|------------------------------------------------------|---------------------------| |-----------------|--------|---------------------------------------------------|---------------------------|
| `enabled` | bool | false | Enable GLM Search | | `enabled` | bool | false | Enable GLM Search |
| `api_key` | string | - | GLM API key | | `api_key` | string | - | GLM API key |
| `base_url` | string | `https://open.bigmodel.cn/api/paas/v4/web_search` | GLM Search API URL | | `base_url` | string | `https://open.bigmodel.cn/api/paas/v4/web_search` | GLM Search API URL |
| `search_engine` | string | `search_std` | Search engine type | | `search_engine` | string | `search_std` | Search engine type |
| `max_results` | int | 5 | Maximum number of results | | `max_results` | int | 5 | Maximum number of results |
### Additional Web Settings ### Additional Web Settings
@ -134,6 +134,28 @@ Baidu Search uses the [Qianfan AI Search API](https://cloud.baidu.com/doc/qianfa
| `prefer_native` | bool | true | Prefer provider's native search over configured search engines | | `prefer_native` | bool | true | Prefer provider's native search over configured search engines |
| `private_host_whitelist` | string[] | `[]` | Private/internal hosts allowed for web fetching | | `private_host_whitelist` | string[] | `[]` | Private/internal hosts allowed for web fetching |
### `web_search` Tool Parameters
At runtime, the `web_search` tool accepts the following parameters:
| Field | Type | Required | Description |
|-------|------|----------|-------------|
| `query` | string | yes | Search query string |
| `count` | integer | no | Number of results to return. Default: `10`, max: `10` |
| `range` | string | no | Optional time filter: `d` (day), `w` (week), `m` (month), `y` (year) |
If `range` is omitted, PicoClaw performs an unrestricted search.
### Example `web_search` Call
```json
{
"query": "ai agent news",
"count": 10,
"range": "w"
}
```
## Exec Tool ## Exec Tool
The exec tool is used to execute shell commands. The exec tool is used to execute shell commands.

View file

@ -43,7 +43,9 @@ var (
reBlankLines = regexp.MustCompile(`\n{3,}`) reBlankLines = regexp.MustCompile(`\n{3,}`)
// DuckDuckGo result extraction // DuckDuckGo result extraction
reDDGLink = regexp.MustCompile(`<a[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)</a>`) reDDGLink = regexp.MustCompile(
`<a[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)</a>`,
)
reDDGSnippet = regexp.MustCompile(`<a class="result__snippet[^"]*".*?>([\s\S]*?)</a>`) reDDGSnippet = regexp.MustCompile(`<a class="result__snippet[^"]*".*?>([\s\S]*?)</a>`)
) )
@ -86,7 +88,122 @@ func (it *APIKeyIterator) Next() (string, bool) {
} }
type SearchProvider interface { type SearchProvider interface {
Search(ctx context.Context, query string, count int) (string, error) Search(ctx context.Context, query string, count int, rangeCode string) (string, error)
}
func normalizeSearchRange(raw string) (string, error) {
rangeCode := strings.ToLower(strings.TrimSpace(raw))
switch rangeCode {
case "", "d", "w", "m", "y":
return rangeCode, nil
default:
return "", fmt.Errorf("range must be one of: d, w, m, y")
}
}
func mapBraveFreshness(rangeCode string) string {
switch rangeCode {
case "d":
return "pd"
case "w":
return "pw"
case "m":
return "pm"
case "y":
return "py"
default:
return ""
}
}
func mapTavilyTimeRange(rangeCode string) string {
switch rangeCode {
case "d":
return "day"
case "w":
return "week"
case "m":
return "month"
case "y":
return "year"
default:
return ""
}
}
func mapPerplexityRecencyFilter(rangeCode string) string {
switch rangeCode {
case "d":
return "day"
case "w":
return "week"
case "m":
return "month"
case "y":
return "year"
default:
return ""
}
}
func mapDuckDuckGoDateFilter(rangeCode string) string {
switch rangeCode {
case "d":
return "d"
case "w":
return "w"
case "m":
return "m"
case "y":
return "t"
default:
return ""
}
}
func mapSearXNGTimeRange(rangeCode string) string {
switch rangeCode {
case "d":
return "day"
case "w":
return "week"
case "m":
return "month"
case "y":
return "year"
default:
return ""
}
}
func mapGLMRecencyFilter(rangeCode string) string {
switch rangeCode {
case "d":
return "oneDay"
case "w":
return "oneWeek"
case "m":
return "oneMonth"
case "y":
return "oneYear"
default:
return "noLimit"
}
}
func mapBaiduRecencyFilter(rangeCode string) string {
switch rangeCode {
case "d", "w":
// Baidu does not expose a day-level filter. Use the closest supported
// window to keep recency bias instead of silently dropping the filter.
return "week"
case "m":
return "month"
case "y":
return "year"
default:
return ""
}
} }
type BraveSearchProvider struct { type BraveSearchProvider struct {
@ -95,9 +212,17 @@ type BraveSearchProvider struct {
client *http.Client client *http.Client
} }
func (p *BraveSearchProvider) Search(ctx context.Context, query string, count int) (string, error) { func (p *BraveSearchProvider) Search(
ctx context.Context,
query string,
count int,
rangeCode string,
) (string, error) {
searchURL := fmt.Sprintf("https://api.search.brave.com/res/v1/web/search?q=%s&count=%d", searchURL := fmt.Sprintf("https://api.search.brave.com/res/v1/web/search?q=%s&count=%d",
url.QueryEscape(query), count) url.QueryEscape(query), count)
if freshness := mapBraveFreshness(rangeCode); freshness != "" {
searchURL += "&freshness=" + url.QueryEscape(freshness)
}
var lastErr error var lastErr error
iter := p.keyPool.NewIterator() iter := p.keyPool.NewIterator()
@ -186,7 +311,12 @@ type TavilySearchProvider struct {
client *http.Client client *http.Client
} }
func (p *TavilySearchProvider) Search(ctx context.Context, query string, count int) (string, error) { func (p *TavilySearchProvider) Search(
ctx context.Context,
query string,
count int,
rangeCode string,
) (string, error) {
searchURL := p.baseURL searchURL := p.baseURL
if searchURL == "" { if searchURL == "" {
searchURL = "https://api.tavily.com/search" searchURL = "https://api.tavily.com/search"
@ -210,6 +340,9 @@ func (p *TavilySearchProvider) Search(ctx context.Context, query string, count i
"include_raw_content": false, "include_raw_content": false,
"max_results": count, "max_results": count,
} }
if timeRange := mapTavilyTimeRange(rangeCode); timeRange != "" {
payload["time_range"] = timeRange
}
bodyBytes, err := json.Marshal(payload) bodyBytes, err := json.Marshal(payload)
if err != nil { if err != nil {
@ -289,8 +422,16 @@ type DuckDuckGoSearchProvider struct {
client *http.Client client *http.Client
} }
func (p *DuckDuckGoSearchProvider) Search(ctx context.Context, query string, count int) (string, error) { func (p *DuckDuckGoSearchProvider) Search(
ctx context.Context,
query string,
count int,
rangeCode string,
) (string, error) {
searchURL := fmt.Sprintf("https://html.duckduckgo.com/html/?q=%s", url.QueryEscape(query)) searchURL := fmt.Sprintf("https://html.duckduckgo.com/html/?q=%s", url.QueryEscape(query))
if dateFilter := mapDuckDuckGoDateFilter(rangeCode); dateFilter != "" {
searchURL += "&df=" + url.QueryEscape(dateFilter)
}
req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil) req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
if err != nil { if err != nil {
@ -313,7 +454,11 @@ func (p *DuckDuckGoSearchProvider) Search(ctx context.Context, query string, cou
return p.extractResults(string(body), count, query) return p.extractResults(string(body), count, query)
} }
func (p *DuckDuckGoSearchProvider) extractResults(html string, count int, query string) (string, error) { func (p *DuckDuckGoSearchProvider) extractResults(
html string,
count int,
query string,
) (string, error) {
// Simple regex based extraction for DDG HTML // Simple regex based extraction for DDG HTML
// Strategy: Find all result containers or key anchors directly // Strategy: Find all result containers or key anchors directly
@ -381,7 +526,12 @@ type PerplexitySearchProvider struct {
client *http.Client client *http.Client
} }
func (p *PerplexitySearchProvider) Search(ctx context.Context, query string, count int) (string, error) { func (p *PerplexitySearchProvider) Search(
ctx context.Context,
query string,
count int,
rangeCode string,
) (string, error) {
searchURL := "https://api.perplexity.ai/chat/completions" searchURL := "https://api.perplexity.ai/chat/completions"
var lastErr error var lastErr error
@ -401,19 +551,31 @@ func (p *PerplexitySearchProvider) Search(ctx context.Context, query string, cou
"content": "You are a search assistant. Provide concise search results with titles, URLs, and brief descriptions in the following format:\n1. Title\n URL\n Description\n\nDo not add extra commentary.", "content": "You are a search assistant. Provide concise search results with titles, URLs, and brief descriptions in the following format:\n1. Title\n URL\n Description\n\nDo not add extra commentary.",
}, },
{ {
"role": "user", "role": "user",
"content": fmt.Sprintf("Search for: %s. Provide up to %d relevant results.", query, count), "content": fmt.Sprintf(
"Search for: %s. Provide up to %d relevant results.",
query,
count,
),
}, },
}, },
"max_tokens": 1000, "max_tokens": 1000,
} }
if recencyFilter := mapPerplexityRecencyFilter(rangeCode); recencyFilter != "" {
payload["search_recency_filter"] = recencyFilter
}
payloadBytes, err := json.Marshal(payload) payloadBytes, err := json.Marshal(payload)
if err != nil { if err != nil {
return "", fmt.Errorf("failed to marshal request: %w", err) return "", fmt.Errorf("failed to marshal request: %w", err)
} }
req, err := http.NewRequestWithContext(ctx, "POST", searchURL, strings.NewReader(string(payloadBytes))) req, err := http.NewRequestWithContext(
ctx,
"POST",
searchURL,
strings.NewReader(string(payloadBytes)),
)
if err != nil { if err != nil {
return "", fmt.Errorf("failed to create request: %w", err) return "", fmt.Errorf("failed to create request: %w", err)
} }
@ -463,7 +625,11 @@ func (p *PerplexitySearchProvider) Search(ctx context.Context, query string, cou
return fmt.Sprintf("No results for: %s", query), nil return fmt.Sprintf("No results for: %s", query), nil
} }
return fmt.Sprintf("Results for: %s (via Perplexity)\n%s", query, searchResp.Choices[0].Message.Content), nil return fmt.Sprintf(
"Results for: %s (via Perplexity)\n%s",
query,
searchResp.Choices[0].Message.Content,
), nil
} }
return "", fmt.Errorf("all api keys failed, last error: %w", lastErr) return "", fmt.Errorf("all api keys failed, last error: %w", lastErr)
@ -473,10 +639,18 @@ type SearXNGSearchProvider struct {
baseURL string baseURL string
} }
func (p *SearXNGSearchProvider) Search(ctx context.Context, query string, count int) (string, error) { func (p *SearXNGSearchProvider) Search(
ctx context.Context,
query string,
count int,
rangeCode string,
) (string, error) {
searchURL := fmt.Sprintf("%s/search?q=%s&format=json&categories=general", searchURL := fmt.Sprintf("%s/search?q=%s&format=json&categories=general",
strings.TrimSuffix(p.baseURL, "/"), strings.TrimSuffix(p.baseURL, "/"),
url.QueryEscape(query)) url.QueryEscape(query))
if timeRange := mapSearXNGTimeRange(rangeCode); timeRange != "" {
searchURL += "&time_range=" + url.QueryEscape(timeRange)
}
req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil) req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
if err != nil { if err != nil {
@ -539,7 +713,12 @@ type GLMSearchProvider struct {
client *http.Client client *http.Client
} }
func (p *GLMSearchProvider) Search(ctx context.Context, query string, count int) (string, error) { func (p *GLMSearchProvider) Search(
ctx context.Context,
query string,
count int,
rangeCode string,
) (string, error) {
searchURL := p.baseURL searchURL := p.baseURL
if searchURL == "" { if searchURL == "" {
searchURL = "https://open.bigmodel.cn/api/paas/v4/web_search" searchURL = "https://open.bigmodel.cn/api/paas/v4/web_search"
@ -552,6 +731,9 @@ func (p *GLMSearchProvider) Search(ctx context.Context, query string, count int)
"count": count, "count": count,
"content_size": "medium", "content_size": "medium",
} }
if recencyFilter := mapGLMRecencyFilter(rangeCode); recencyFilter != "" {
payload["search_recency_filter"] = recencyFilter
}
bodyBytes, err := json.Marshal(payload) bodyBytes, err := json.Marshal(payload)
if err != nil { if err != nil {
@ -620,7 +802,12 @@ type BaiduSearchProvider struct {
client *http.Client client *http.Client
} }
func (p *BaiduSearchProvider) Search(ctx context.Context, query string, count int) (string, error) { func (p *BaiduSearchProvider) Search(
ctx context.Context,
query string,
count int,
rangeCode string,
) (string, error) {
searchURL := p.baseURL searchURL := p.baseURL
if searchURL == "" { if searchURL == "" {
searchURL = "https://qianfan.baidubce.com/v2/ai_search/web_search" searchURL = "https://qianfan.baidubce.com/v2/ai_search/web_search"
@ -636,6 +823,9 @@ func (p *BaiduSearchProvider) Search(ctx context.Context, query string, count in
"search_source": "baidu_search_v2", "search_source": "baidu_search_v2",
"resource_type_filter": []map[string]any{{"type": "web", "top_k": count}}, "resource_type_filter": []map[string]any{{"type": "web", "top_k": count}},
} }
if recencyFilter := mapBaiduRecencyFilter(rangeCode); recencyFilter != "" {
payload["search_recency_filter"] = recencyFilter
}
bodyBytes, err := json.Marshal(payload) bodyBytes, err := json.Marshal(payload)
if err != nil { if err != nil {
@ -729,7 +919,7 @@ type WebSearchToolOptions struct {
func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) { func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
var provider SearchProvider var provider SearchProvider
maxResults := 5 maxResults := 10
// Priority: Perplexity > Brave > SearXNG > Tavily > DuckDuckGo > Baidu Search > GLM Search // Priority: Perplexity > Brave > SearXNG > Tavily > DuckDuckGo > Baidu Search > GLM Search
if opts.PerplexityEnabled && len(opts.PerplexityAPIKeys) > 0 { if opts.PerplexityEnabled && len(opts.PerplexityAPIKeys) > 0 {
client, err := utils.CreateHTTPClient(opts.Proxy, perplexityTimeout) client, err := utils.CreateHTTPClient(opts.Proxy, perplexityTimeout)
@ -742,7 +932,7 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
client: client, client: client,
} }
if opts.PerplexityMaxResults > 0 { if opts.PerplexityMaxResults > 0 {
maxResults = opts.PerplexityMaxResults maxResults = min(opts.PerplexityMaxResults, 10)
} }
} else if opts.BraveEnabled && len(opts.BraveAPIKeys) > 0 { } else if opts.BraveEnabled && len(opts.BraveAPIKeys) > 0 {
client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout) client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout)
@ -751,12 +941,12 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
} }
provider = &BraveSearchProvider{keyPool: NewAPIKeyPool(opts.BraveAPIKeys), proxy: opts.Proxy, client: client} provider = &BraveSearchProvider{keyPool: NewAPIKeyPool(opts.BraveAPIKeys), proxy: opts.Proxy, client: client}
if opts.BraveMaxResults > 0 { if opts.BraveMaxResults > 0 {
maxResults = opts.BraveMaxResults maxResults = min(opts.BraveMaxResults, 10)
} }
} else if opts.SearXNGEnabled && opts.SearXNGBaseURL != "" { } else if opts.SearXNGEnabled && opts.SearXNGBaseURL != "" {
provider = &SearXNGSearchProvider{baseURL: opts.SearXNGBaseURL} provider = &SearXNGSearchProvider{baseURL: opts.SearXNGBaseURL}
if opts.SearXNGMaxResults > 0 { if opts.SearXNGMaxResults > 0 {
maxResults = opts.SearXNGMaxResults maxResults = min(opts.SearXNGMaxResults, 10)
} }
} else if opts.TavilyEnabled && len(opts.TavilyAPIKeys) > 0 { } else if opts.TavilyEnabled && len(opts.TavilyAPIKeys) > 0 {
client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout) client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout)
@ -770,7 +960,7 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
client: client, client: client,
} }
if opts.TavilyMaxResults > 0 { if opts.TavilyMaxResults > 0 {
maxResults = opts.TavilyMaxResults maxResults = min(opts.TavilyMaxResults, 10)
} }
} else if opts.DuckDuckGoEnabled { } else if opts.DuckDuckGoEnabled {
client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout) client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout)
@ -779,7 +969,7 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
} }
provider = &DuckDuckGoSearchProvider{proxy: opts.Proxy, client: client} provider = &DuckDuckGoSearchProvider{proxy: opts.Proxy, client: client}
if opts.DuckDuckGoMaxResults > 0 { if opts.DuckDuckGoMaxResults > 0 {
maxResults = opts.DuckDuckGoMaxResults maxResults = min(opts.DuckDuckGoMaxResults, 10)
} }
} else if opts.BaiduSearchEnabled && opts.BaiduSearchAPIKey != "" { } else if opts.BaiduSearchEnabled && opts.BaiduSearchAPIKey != "" {
client, err := utils.CreateHTTPClient(opts.Proxy, perplexityTimeout) client, err := utils.CreateHTTPClient(opts.Proxy, perplexityTimeout)
@ -793,7 +983,7 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
client: client, client: client,
} }
if opts.BaiduSearchMaxResults > 0 { if opts.BaiduSearchMaxResults > 0 {
maxResults = opts.BaiduSearchMaxResults maxResults = min(opts.BaiduSearchMaxResults, 10)
} }
} else if opts.GLMSearchEnabled && opts.GLMSearchAPIKey != "" { } else if opts.GLMSearchEnabled && opts.GLMSearchAPIKey != "" {
client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout) client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout)
@ -812,7 +1002,7 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
client: client, client: client,
} }
if opts.GLMSearchMaxResults > 0 { if opts.GLMSearchMaxResults > 0 {
maxResults = opts.GLMSearchMaxResults maxResults = min(opts.GLMSearchMaxResults, 10)
} }
} else { } else {
return nil, nil return nil, nil
@ -829,7 +1019,7 @@ func (t *WebSearchTool) Name() string {
} }
func (t *WebSearchTool) Description() string { func (t *WebSearchTool) Description() string {
return "Search the web for current information. Returns titles, URLs, and snippets from search results." return "Search the web for current information. Supports query, count, and an optional temporal range filter. Returns titles, URLs, and snippets from search results."
} }
func (t *WebSearchTool) Parameters() map[string]any { func (t *WebSearchTool) Parameters() map[string]any {
@ -842,10 +1032,15 @@ func (t *WebSearchTool) Parameters() map[string]any {
}, },
"count": map[string]any{ "count": map[string]any{
"type": "integer", "type": "integer",
"description": "Number of results (1-10)", "description": "Number of results (default: 10, max: 10)",
"minimum": 1.0, "minimum": 1.0,
"maximum": 10.0, "maximum": 10.0,
}, },
"range": map[string]any{
"type": "string",
"description": "Optional time filter: d (day), w (week), m (month), y (year)",
"enum": []string{"d", "w", "m", "y"},
},
}, },
"required": []string{"query"}, "required": []string{"query"},
} }
@ -853,18 +1048,36 @@ func (t *WebSearchTool) Parameters() map[string]any {
func (t *WebSearchTool) Execute(ctx context.Context, args map[string]any) *ToolResult { func (t *WebSearchTool) Execute(ctx context.Context, args map[string]any) *ToolResult {
query, ok := args["query"].(string) query, ok := args["query"].(string)
if !ok { if !ok || strings.TrimSpace(query) == "" {
return ErrorResult("query is required") return ErrorResult("query is required")
} }
query = strings.TrimSpace(query)
count64, err := getInt64Arg(args, "count", int64(t.maxResults))
if err != nil {
return ErrorResult(err.Error())
}
count := t.maxResults count := t.maxResults
if c, ok := args["count"].(float64); ok { if count64 > 0 && count64 <= 10 {
if int(c) > 0 && int(c) <= 10 { count = int(count64)
count = int(c) }
rangeCode, err := normalizeSearchRange("")
if err != nil {
return ErrorResult(err.Error())
}
if rawRange, exists := args["range"]; exists {
rangeStr, ok := rawRange.(string)
if !ok {
return ErrorResult("range must be a string")
}
rangeCode, err = normalizeSearchRange(rangeStr)
if err != nil {
return ErrorResult(err.Error())
} }
} }
result, err := t.provider.Search(ctx, query, count) result, err := t.provider.Search(ctx, query, count, rangeCode)
if err != nil { if err != nil {
return ErrorResult(fmt.Sprintf("search failed: %v", err)) return ErrorResult(fmt.Sprintf("search failed: %v", err))
} }
@ -1038,7 +1251,12 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe
if err != nil { if err != nil {
var maxBytesErr *http.MaxBytesError var maxBytesErr *http.MaxBytesError
if errors.As(err, &maxBytesErr) { if errors.As(err, &maxBytesErr) {
return ErrorResult(fmt.Sprintf("failed to read response: size exceeded %d bytes limit", t.fetchLimitBytes)) return ErrorResult(
fmt.Sprintf(
"failed to read response: size exceeded %d bytes limit",
t.fetchLimitBytes,
),
)
} }
return ErrorResult(err.Error()) return ErrorResult(err.Error())
} }
@ -1088,7 +1306,11 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe
// If the charset is not utf-8, we might have to convert the bodyStr // If the charset is not utf-8, we might have to convert the bodyStr
// before passing it to the HTML/Markdown parser // before passing it to the HTML/Markdown parser
if strings.ToLower(charset) != "utf-8" { if strings.ToLower(charset) != "utf-8" {
logger.WarnCF("tool", "Note: the content is not in UTF-8", map[string]any{"charset": charset}) logger.WarnCF(
"tool",
"Note: the content is not in UTF-8",
map[string]any{"charset": charset},
)
} }
} }
@ -1232,7 +1454,11 @@ func newSafeDialContext(
continue continue
} }
attempted++ attempted++
conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(ipAddr.IP.String(), port)) conn, err := dialer.DialContext(
ctx,
network,
net.JoinHostPort(ipAddr.IP.String(), port),
)
if err == nil { if err == nil {
return conn, nil return conn, nil
} }
@ -1240,10 +1466,17 @@ func newSafeDialContext(
} }
if attempted == 0 { if attempted == 0 {
return nil, fmt.Errorf("all resolved addresses for %s are private, restricted, or not whitelisted", host) return nil, fmt.Errorf(
"all resolved addresses for %s are private, restricted, or not whitelisted",
host,
)
} }
if lastErr != nil { if lastErr != nil {
return nil, fmt.Errorf("failed connecting to public addresses for %s: %w", host, lastErr) return nil, fmt.Errorf(
"failed connecting to public addresses for %s: %w",
host,
lastErr,
)
} }
return nil, fmt.Errorf("failed connecting to public addresses for %s", host) return nil, fmt.Errorf("failed connecting to public addresses for %s", host)
} }

View file

@ -426,6 +426,96 @@ func TestWebTool_WebSearch_MissingQuery(t *testing.T) {
} }
} }
func TestNormalizeSearchRange(t *testing.T) {
tests := []struct {
name string
input string
want string
wantErr bool
}{
{name: "empty", input: "", want: ""},
{name: "day", input: "d", want: "d"},
{name: "week uppercase trimmed", input: " W ", want: "w"},
{name: "month", input: "m", want: "m"},
{name: "year", input: "y", want: "y"},
{name: "invalid", input: "q", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := normalizeSearchRange(tt.input)
if tt.wantErr {
if err == nil {
t.Fatalf("expected error, got nil")
}
return
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got != tt.want {
t.Fatalf("normalizeSearchRange(%q) = %q, want %q", tt.input, got, tt.want)
}
})
}
}
func TestSearchRangeMappings(t *testing.T) {
if got := mapBraveFreshness("d"); got != "pd" {
t.Fatalf("mapBraveFreshness(d) = %q, want pd", got)
}
if got := mapBraveFreshness("y"); got != "py" {
t.Fatalf("mapBraveFreshness(y) = %q, want py", got)
}
if got := mapTavilyTimeRange("w"); got != "week" {
t.Fatalf("mapTavilyTimeRange(w) = %q, want week", got)
}
if got := mapPerplexityRecencyFilter("m"); got != "month" {
t.Fatalf("mapPerplexityRecencyFilter(m) = %q, want month", got)
}
if got := mapDuckDuckGoDateFilter("y"); got != "t" {
t.Fatalf("mapDuckDuckGoDateFilter(y) = %q, want t", got)
}
if got := mapSearXNGTimeRange("d"); got != "day" {
t.Fatalf("mapSearXNGTimeRange(d) = %q, want day", got)
}
if got := mapGLMRecencyFilter("w"); got != "oneWeek" {
t.Fatalf("mapGLMRecencyFilter(w) = %q, want oneWeek", got)
}
if got := mapGLMRecencyFilter(""); got != "noLimit" {
t.Fatalf("mapGLMRecencyFilter(\"\") = %q, want noLimit", got)
}
if got := mapBaiduRecencyFilter("d"); got != "week" {
t.Fatalf("mapBaiduRecencyFilter(d) = %q, want week", got)
}
if got := mapBaiduRecencyFilter("m"); got != "month" {
t.Fatalf("mapBaiduRecencyFilter(m) = %q, want month", got)
}
}
func TestWebTool_WebSearch_InvalidRange(t *testing.T) {
tool, err := NewWebSearchTool(WebSearchToolOptions{
BraveEnabled: true,
BraveAPIKeys: []string{"test-key"},
BraveMaxResults: 5,
})
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
result := tool.Execute(context.Background(), map[string]any{
"query": "test query",
"range": "invalid",
})
if !result.IsError {
t.Fatalf("expected invalid range to return error")
}
if !strings.Contains(result.ForLLM, "range must be one of: d, w, m, y") {
t.Fatalf("unexpected error message: %q", result.ForLLM)
}
}
// TestWebTool_WebFetch_HTMLExtraction verifies HTML text extraction // TestWebTool_WebFetch_HTMLExtraction verifies HTML text extraction
func TestWebTool_WebFetch_HTMLExtraction(t *testing.T) { func TestWebTool_WebFetch_HTMLExtraction(t *testing.T) {
withPrivateWebFetchHostsAllowed(t) withPrivateWebFetchHostsAllowed(t)
@ -1069,6 +1159,45 @@ func TestWebTool_TavilySearch_Success(t *testing.T) {
} }
} }
func TestWebTool_TavilySearch_RangeMapping(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var payload map[string]any
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
t.Fatalf("failed to decode payload: %v", err)
}
if payload["time_range"] != "week" {
t.Fatalf("expected time_range=week, got %v", payload["time_range"])
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(map[string]any{
"results": []map[string]any{
{"title": "Recent result", "url": "https://example.com/recent", "content": "snippet"},
},
})
}))
defer server.Close()
tool, err := NewWebSearchTool(WebSearchToolOptions{
TavilyEnabled: true,
TavilyAPIKeys: []string{"test-key"},
TavilyBaseURL: server.URL,
TavilyMaxResults: 5,
})
if err != nil {
t.Fatalf("NewWebSearchTool() error: %v", err)
}
result := tool.Execute(context.Background(), map[string]any{
"query": "test query",
"range": "w",
})
if result.IsError {
t.Fatalf("expected success, got %s", result.ForLLM)
}
}
// TestWebFetchTool_CloudflareChallenge_RetryWithHonestUA verifies that a 403 response // TestWebFetchTool_CloudflareChallenge_RetryWithHonestUA verifies that a 403 response
// with cf-mitigated: challenge triggers a retry using the honest picoclaw User-Agent, // with cf-mitigated: challenge triggers a retry using the honest picoclaw User-Agent,
// and that the retry response is returned when it succeeds. // and that the retry response is returned when it succeeds.
@ -1297,6 +1426,39 @@ func TestWebTool_TavilySearch_Failover(t *testing.T) {
} }
} }
func TestWebTool_SearXNGSearch_RangeMapping(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.URL.Query().Get("time_range"); got != "year" {
t.Fatalf("expected time_range=year, got %q", got)
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(map[string]any{
"results": []map[string]any{
{"title": "Recent result", "url": "https://example.com/1", "content": "snippet"},
},
})
}))
defer server.Close()
tool, err := NewWebSearchTool(WebSearchToolOptions{
SearXNGEnabled: true,
SearXNGBaseURL: server.URL,
SearXNGMaxResults: 5,
})
if err != nil {
t.Fatalf("NewWebSearchTool() error: %v", err)
}
result := tool.Execute(context.Background(), map[string]any{
"query": "test query",
"range": "y",
})
if result.IsError {
t.Fatalf("expected success, got %s", result.ForLLM)
}
}
func TestWebTool_GLMSearch_Success(t *testing.T) { func TestWebTool_GLMSearch_Success(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" { if r.Method != "POST" {
@ -1365,6 +1527,83 @@ func TestWebTool_GLMSearch_Success(t *testing.T) {
} }
} }
func TestWebTool_GLMSearch_RangeMapping(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var payload map[string]any
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
t.Fatalf("failed to decode payload: %v", err)
}
if payload["search_recency_filter"] != "oneMonth" {
t.Fatalf("expected search_recency_filter=oneMonth, got %v", payload["search_recency_filter"])
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(map[string]any{
"search_result": []map[string]any{
{"title": "Recent GLM Result", "content": "snippet", "link": "https://example.com/glm-range"},
},
})
}))
defer server.Close()
tool, err := NewWebSearchTool(WebSearchToolOptions{
GLMSearchEnabled: true,
GLMSearchAPIKey: "test-glm-key",
GLMSearchBaseURL: server.URL,
GLMSearchEngine: "search_std",
})
if err != nil {
t.Fatalf("NewWebSearchTool() error: %v", err)
}
result := tool.Execute(context.Background(), map[string]any{
"query": "test query",
"range": "m",
})
if result.IsError {
t.Fatalf("expected success, got %s", result.ForLLM)
}
}
func TestWebTool_BaiduSearch_RangeMapping(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var payload map[string]any
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
t.Fatalf("failed to decode payload: %v", err)
}
if payload["search_recency_filter"] != "week" {
t.Fatalf("expected search_recency_filter=week for day fallback, got %v", payload["search_recency_filter"])
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(map[string]any{
"references": []map[string]any{
{"title": "Recent Baidu Result", "url": "https://example.com/baidu", "content": "snippet"},
},
})
}))
defer server.Close()
tool, err := NewWebSearchTool(WebSearchToolOptions{
BaiduSearchEnabled: true,
BaiduSearchAPIKey: "test-baidu-key",
BaiduSearchBaseURL: server.URL,
})
if err != nil {
t.Fatalf("NewWebSearchTool() error: %v", err)
}
result := tool.Execute(context.Background(), map[string]any{
"query": "test query",
"range": "d",
})
if result.IsError {
t.Fatalf("expected success, got %s", result.ForLLM)
}
}
func TestWebTool_GLMSearch_APIError(t *testing.T) { func TestWebTool_GLMSearch_APIError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusUnauthorized) w.WriteHeader(http.StatusUnauthorized)