fix lint + error check

pkg/tools/web.go

@@ -16,6 +16,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/sipeed/picoclaw/pkg/logger"
 	"github.com/sipeed/picoclaw/pkg/utils"
 )
 
@@ -29,7 +30,6 @@ const (
 
 	defaultMaxChars = 50000
 	maxRedirects    = 5
-	format          = "plaintext"
 )
 
 // Pre-compiled regexes for HTML text extraction
@@ -790,20 +790,27 @@ type privateHostWhitelist struct {
 
 func NewWebFetchTool(maxChars int, format string, fetchLimitBytes int64) (*WebFetchTool, error) {
 	// createHTTPClient cannot fail with an empty proxy string.
-	return NewWebFetchToolWithProxy(maxChars, "", format, fetchLimitBytes, nil)
+	return NewWebFetchToolWithConfig(maxChars, "", format, fetchLimitBytes, nil)
 }
 
 // allowPrivateWebFetchHosts controls whether loopback/private hosts are allowed.
 // This is false in normal runtime to reduce SSRF exposure, and tests can override it temporarily.
 var allowPrivateWebFetchHosts atomic.Bool
 
-func NewWebFetchToolWithProxy(maxChars int, proxy string, format string, fetchLimitBytes int64) (*WebFetchTool, error) {
+func NewWebFetchToolWithProxy(
-	return NewWebFetchToolWithConfig(maxChars, proxy, fetchLimitBytes, nil)
+	maxChars int,
+	proxy string,
+	format string,
+	fetchLimitBytes int64,
+	privateHostWhitelist []string,
+) (*WebFetchTool, error) {
+	return NewWebFetchToolWithConfig(maxChars, proxy, format, fetchLimitBytes, privateHostWhitelist)
 }
 
 func NewWebFetchToolWithConfig(
 	maxChars int,
 	proxy string,
+	format string,
 	fetchLimitBytes int64,
 	privateHostWhitelist []string,
 ) (*WebFetchTool, error) {
@@ -933,7 +940,26 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe
 	bodyStr := string(body)
 	contentType := resp.Header.Get("Content-Type")
 
-	mediaType, _, _ := mime.ParseMediaType(contentType)
+	mediaType, params, err := mime.ParseMediaType(contentType)
+	if err != nil {
+		// The most common error here is "mime: no media type" if the header is empty.
+		logger.WarnCF("tool", "Failed to parse Content-Type", map[string]any{
+			"raw_header": contentType,
+			"error":      err.Error(),
+		})
+
+		// security fallback
+		mediaType = "application/octet-stream"
+	}
+
+	charset, hasCharset := params["charset"]
+	if hasCharset {
+		// If the charset is not utf-8, we might have to convert the bodyStr
+		// before passing it to the HTML/Markdown parser
+		if strings.ToLower(charset) != "utf-8" {
+			logger.WarnCF("tool", "Note: the content is not in UTF-8", map[string]any{"charset": charset})
+		}
+	}
 
 	var text, extractor string
 

pkg/tools/web_test.go

@@ -17,6 +17,7 @@ import (
 
 const (
 	testFetchLimit = int64(10 * 1024 * 1024)
+	format         = "plaintext"
 )
 
 // TestWebTool_WebFetch_Success verifies successful URL fetching
@@ -476,7 +477,7 @@ func TestWebTool_WebFetch_PrivateHostAllowedByExactWhitelist(t *testing.T) {
 	defer server.Close()
 
 	host, _ := serverHostAndPort(t, server.URL)
-	tool, err := NewWebFetchToolWithConfig(50000, "", testFetchLimit, []string{host})
+	tool, err := NewWebFetchToolWithConfig(50000, "", format, testFetchLimit, []string{host})
 	if err != nil {
 		t.Fatalf("Failed to create web fetch tool: %v", err)
 	}
@@ -501,7 +502,7 @@ func TestWebTool_WebFetch_PrivateHostAllowedByCIDRWhitelist(t *testing.T) {
 	defer server.Close()
 
 	host, _ := serverHostAndPort(t, server.URL)
-	tool, err := NewWebFetchToolWithConfig(50000, "", testFetchLimit, []string{singleHostCIDR(t, host)})
+	tool, err := NewWebFetchToolWithConfig(50000, "", format, testFetchLimit, []string{singleHostCIDR(t, host)})
 	if err != nil {
 		t.Fatalf("Failed to create web fetch tool: %v", err)
 	}
@@ -778,7 +779,7 @@ func TestWebTool_WebFetch_MissingDomain(t *testing.T) {
 }
 
 func TestNewWebFetchToolWithProxy(t *testing.T) {
-	tool, err := NewWebFetchToolWithProxy(1024, "http://127.0.0.1:7890", format, testFetchLimit)
+	tool, err := NewWebFetchToolWithProxy(1024, "http://127.0.0.1:7890", format, testFetchLimit, nil)
 	if err != nil {
 		logger.ErrorCF("agent", "Failed to create web fetch tool", map[string]any{"error": err.Error()})
 	} else if tool.maxChars != 1024 {
@@ -789,7 +790,7 @@ func TestNewWebFetchToolWithProxy(t *testing.T) {
 		t.Fatalf("proxy = %q, want %q", tool.proxy, "http://127.0.0.1:7890")
 	}
 
-	tool, err = NewWebFetchToolWithProxy(0, "http://127.0.0.1:7890", format, testFetchLimit)
+	tool, err = NewWebFetchToolWithProxy(0, "http://127.0.0.1:7890", format, testFetchLimit, nil)
 	if err != nil {
 		logger.ErrorCF("agent", "Failed to create web fetch tool", map[string]any{"error": err.Error()})
 	}
@@ -800,7 +801,7 @@ func TestNewWebFetchToolWithProxy(t *testing.T) {
 }
 
 func TestNewWebFetchToolWithConfig_InvalidPrivateHostWhitelist(t *testing.T) {
-	_, err := NewWebFetchToolWithConfig(1024, "", testFetchLimit, []string{"not-an-ip-or-cidr"})
+	_, err := NewWebFetchToolWithConfig(1024, "", format, testFetchLimit, []string{"not-an-ip-or-cidr"})
 	if err == nil {
 		t.Fatal("expected invalid whitelist entry to fail")
 	}