diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 22a5d40c8..b3e392305 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -166,7 +166,7 @@ func registerSharedTools( cfg.Tools.Web.Proxy, cfg.Tools.Web.Format, cfg.Tools.Web.FetchLimitBytes, - cfg.Tools.Web.PrivateHostWhitelist) + cfg.Tools.Web.PrivateHostWhitelist) if err != nil { logger.ErrorCF("agent", "Failed to create web fetch tool", map[string]any{"error": err.Error()}) } else { diff --git a/pkg/config/config.go b/pkg/config/config.go index 6827cc4d7..fce5fbef9 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -695,10 +695,10 @@ type WebToolsConfig struct { GLMSearch GLMSearchConfig ` json:"glm_search"` // Proxy is an optional proxy URL for web tools (http/https/socks5/socks5h). // For authenticated proxies, prefer HTTP_PROXY/HTTPS_PROXY env vars instead of embedding credentials in config. - Proxy string `json:"proxy,omitempty" env:"PICOCLAW_TOOLS_WEB_PROXY"` - FetchLimitBytes int64 `json:"fetch_limit_bytes,omitempty" env:"PICOCLAW_TOOLS_WEB_FETCH_LIMIT_BYTES"` - Format string `json:"format,omitempty" env:"PICOCLAW_TOOLS_WEB_FORMAT"` - PrivateHostWhitelist FlexibleStringSlice `json:"private_host_whitelist,omitempty" env:"PICOCLAW_TOOLS_WEB_PRIVATE_HOST_WHITELIST"` + Proxy string `json:"proxy,omitempty" env:"PICOCLAW_TOOLS_WEB_PROXY"` + FetchLimitBytes int64 `json:"fetch_limit_bytes,omitempty" env:"PICOCLAW_TOOLS_WEB_FETCH_LIMIT_BYTES"` + Format string `json:"format,omitempty" env:"PICOCLAW_TOOLS_WEB_FORMAT"` + PrivateHostWhitelist FlexibleStringSlice `json:"private_host_whitelist,omitempty" env:"PICOCLAW_TOOLS_WEB_PRIVATE_HOST_WHITELIST"` } type CronToolsConfig struct { diff --git a/pkg/tools/web.go b/pkg/tools/web.go index fed2c5207..810914f2e 100644 --- a/pkg/tools/web.go +++ b/pkg/tools/web.go @@ -16,6 +16,7 @@ import ( "sync/atomic" "time" + "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/utils" ) @@ -29,7 +30,6 @@ const ( defaultMaxChars = 50000 maxRedirects = 5 - format = "plaintext" ) // Pre-compiled regexes for HTML text extraction @@ -790,20 +790,27 @@ type privateHostWhitelist struct { func NewWebFetchTool(maxChars int, format string, fetchLimitBytes int64) (*WebFetchTool, error) { // createHTTPClient cannot fail with an empty proxy string. - return NewWebFetchToolWithProxy(maxChars, "", format, fetchLimitBytes, nil) + return NewWebFetchToolWithConfig(maxChars, "", format, fetchLimitBytes, nil) } // allowPrivateWebFetchHosts controls whether loopback/private hosts are allowed. // This is false in normal runtime to reduce SSRF exposure, and tests can override it temporarily. var allowPrivateWebFetchHosts atomic.Bool -func NewWebFetchToolWithProxy(maxChars int, proxy string, format string, fetchLimitBytes int64) (*WebFetchTool, error) { - return NewWebFetchToolWithConfig(maxChars, proxy, fetchLimitBytes, nil) +func NewWebFetchToolWithProxy( + maxChars int, + proxy string, + format string, + fetchLimitBytes int64, + privateHostWhitelist []string, +) (*WebFetchTool, error) { + return NewWebFetchToolWithConfig(maxChars, proxy, format, fetchLimitBytes, privateHostWhitelist) } func NewWebFetchToolWithConfig( maxChars int, proxy string, + format string, fetchLimitBytes int64, privateHostWhitelist []string, ) (*WebFetchTool, error) { @@ -933,7 +940,26 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe bodyStr := string(body) contentType := resp.Header.Get("Content-Type") - mediaType, _, _ := mime.ParseMediaType(contentType) + mediaType, params, err := mime.ParseMediaType(contentType) + if err != nil { + // The most common error here is "mime: no media type" if the header is empty. + logger.WarnCF("tool", "Failed to parse Content-Type", map[string]any{ + "raw_header": contentType, + "error": err.Error(), + }) + + // security fallback + mediaType = "application/octet-stream" + } + + charset, hasCharset := params["charset"] + if hasCharset { + // If the charset is not utf-8, we might have to convert the bodyStr + // before passing it to the HTML/Markdown parser + if strings.ToLower(charset) != "utf-8" { + logger.WarnCF("tool", "Note: the content is not in UTF-8", map[string]any{"charset": charset}) + } + } var text, extractor string diff --git a/pkg/tools/web_test.go b/pkg/tools/web_test.go index 1bfcd2985..dfb33971a 100644 --- a/pkg/tools/web_test.go +++ b/pkg/tools/web_test.go @@ -17,6 +17,7 @@ import ( const ( testFetchLimit = int64(10 * 1024 * 1024) + format = "plaintext" ) // TestWebTool_WebFetch_Success verifies successful URL fetching @@ -476,7 +477,7 @@ func TestWebTool_WebFetch_PrivateHostAllowedByExactWhitelist(t *testing.T) { defer server.Close() host, _ := serverHostAndPort(t, server.URL) - tool, err := NewWebFetchToolWithConfig(50000, "", testFetchLimit, []string{host}) + tool, err := NewWebFetchToolWithConfig(50000, "", format, testFetchLimit, []string{host}) if err != nil { t.Fatalf("Failed to create web fetch tool: %v", err) } @@ -501,7 +502,7 @@ func TestWebTool_WebFetch_PrivateHostAllowedByCIDRWhitelist(t *testing.T) { defer server.Close() host, _ := serverHostAndPort(t, server.URL) - tool, err := NewWebFetchToolWithConfig(50000, "", testFetchLimit, []string{singleHostCIDR(t, host)}) + tool, err := NewWebFetchToolWithConfig(50000, "", format, testFetchLimit, []string{singleHostCIDR(t, host)}) if err != nil { t.Fatalf("Failed to create web fetch tool: %v", err) } @@ -778,7 +779,7 @@ func TestWebTool_WebFetch_MissingDomain(t *testing.T) { } func TestNewWebFetchToolWithProxy(t *testing.T) { - tool, err := NewWebFetchToolWithProxy(1024, "http://127.0.0.1:7890", format, testFetchLimit) + tool, err := NewWebFetchToolWithProxy(1024, "http://127.0.0.1:7890", format, testFetchLimit, nil) if err != nil { logger.ErrorCF("agent", "Failed to create web fetch tool", map[string]any{"error": err.Error()}) } else if tool.maxChars != 1024 { @@ -789,7 +790,7 @@ func TestNewWebFetchToolWithProxy(t *testing.T) { t.Fatalf("proxy = %q, want %q", tool.proxy, "http://127.0.0.1:7890") } - tool, err = NewWebFetchToolWithProxy(0, "http://127.0.0.1:7890", format, testFetchLimit) + tool, err = NewWebFetchToolWithProxy(0, "http://127.0.0.1:7890", format, testFetchLimit, nil) if err != nil { logger.ErrorCF("agent", "Failed to create web fetch tool", map[string]any{"error": err.Error()}) } @@ -800,7 +801,7 @@ func TestNewWebFetchToolWithProxy(t *testing.T) { } func TestNewWebFetchToolWithConfig_InvalidPrivateHostWhitelist(t *testing.T) { - _, err := NewWebFetchToolWithConfig(1024, "", testFetchLimit, []string{"not-an-ip-or-cidr"}) + _, err := NewWebFetchToolWithConfig(1024, "", format, testFetchLimit, []string{"not-an-ip-or-cidr"}) if err == nil { t.Fatal("expected invalid whitelist entry to fail") }