diff --git a/pkg/config/config.go b/pkg/config/config.go index 1d98aa334..a03a58ab4 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -842,6 +842,13 @@ func (c ReadFileToolConfig) EffectiveMode() string { } } +type CurlConfig struct { + ToolConfig `json:"-" envPrefix:"PICOCLAW_TOOLS_CURL_"` + AllowedDomains []string `json:"allowed_domains,omitempty" env:"PICOCLAW_TOOLS_CURL_ALLOWED_DOMAINS"` + TimeoutSeconds int `json:"timeout_seconds" env:"PICOCLAW_TOOLS_CURL_TIMEOUT_SECONDS"` + MaxBytes int64 `json:"max_bytes" env:"PICOCLAW_TOOLS_CURL_MAX_BYTES"` +} + type ToolsConfig struct { AllowReadPaths []string `json:"allow_read_paths" yaml:"-" env:"PICOCLAW_TOOLS_ALLOW_READ_PATHS"` AllowWritePaths []string `json:"allow_write_paths" yaml:"-" env:"PICOCLAW_TOOLS_ALLOW_WRITE_PATHS"` @@ -875,6 +882,7 @@ type ToolsConfig struct { Subagent ToolConfig `json:"subagent" yaml:"-" envPrefix:"PICOCLAW_TOOLS_SUBAGENT_"` WebFetch ToolConfig `json:"web_fetch" yaml:"-" envPrefix:"PICOCLAW_TOOLS_WEB_FETCH_"` WriteFile ToolConfig `json:"write_file" yaml:"-" envPrefix:"PICOCLAW_TOOLS_WRITE_FILE_"` + Curl CurlConfig `json:"curl" yaml:"-" envPrefix:"PICOCLAW_TOOLS_CURL_"` } // IsFilterSensitiveDataEnabled returns true if sensitive data filtering is enabled @@ -1362,6 +1370,8 @@ func (t *ToolsConfig) IsToolEnabled(name string) bool { return t.SendTTS.Enabled case "write_file": return t.WriteFile.Enabled + case "curl": + return t.Curl.Enabled case "mcp": return t.MCP.Enabled default: diff --git a/pkg/tools/curl.go b/pkg/tools/curl.go new file mode 100644 index 000000000..336ca1dcc --- /dev/null +++ b/pkg/tools/curl.go @@ -0,0 +1,268 @@ +package tools + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/sipeed/picoclaw/pkg/utils" +) + +const ( + curlDefaultTimeout = 30 * time.Second + curlDefaultMaxBytes = int64(1 << 20) + curlMaxRedirects = 5 +) + +type CurlTool struct { + allowedDomains []string + client *http.Client + maxBytes int64 +} + +type CurlToolOptions struct { + AllowedDomains []string + Proxy string + TimeoutSeconds int + MaxBytes int64 +} + +func NewCurlTool(opts CurlToolOptions) (*CurlTool, error) { + timeout := curlDefaultTimeout + if opts.TimeoutSeconds > 0 { + timeout = time.Duration(opts.TimeoutSeconds) * time.Second + } + + maxBytes := curlDefaultMaxBytes + if opts.MaxBytes > 0 { + maxBytes = opts.MaxBytes + } + + client, err := utils.CreateHTTPClient(opts.Proxy, timeout) + if err != nil { + return nil, fmt.Errorf("failed to create HTTP client for curl tool: %w", err) + } + + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if len(via) >= curlMaxRedirects { + return fmt.Errorf("stopped after %d redirects", curlMaxRedirects) + } + if !isDomainAllowed(req.URL.Hostname(), opts.AllowedDomains) { + return fmt.Errorf("redirect to disallowed domain %q", req.URL.Hostname()) + } + return nil + } + + return &CurlTool{ + allowedDomains: normalizeDomains(opts.AllowedDomains), + client: client, + maxBytes: maxBytes, + }, nil +} + +func (t *CurlTool) Name() string { + return "curl" +} + +func (t *CurlTool) Description() string { + return "Make HTTP requests to external APIs. Use url (required), method (GET/POST/PUT/DELETE/etc), headers (optional map), body (optional string), and timeout (optional seconds). Only allowed domains can be accessed." +} + +func (t *CurlTool) Parameters() map[string]any { + return map[string]any{ + "type": "object", + "properties": map[string]any{ + "url": map[string]any{ + "type": "string", + "description": "URL to request (http/https only, must match allowed domains)", + }, + "method": map[string]any{ + "type": "string", + "description": "HTTP method (GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS)", + "enum": []string{"GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"}, + }, + "headers": map[string]any{ + "type": "object", + "description": "Optional HTTP headers as key-value pairs", + "additionalProperties": map[string]any{ + "type": "string", + }, + }, + "body": map[string]any{ + "type": "string", + "description": "Optional request body (for POST, PUT, PATCH)", + }, + "timeout": map[string]any{ + "type": "integer", + "description": "Optional timeout in seconds (default: 30, max: 120)", + "minimum": 1.0, + "maximum": 120.0, + }, + }, + "required": []string{"url"}, + } +} + +func (t *CurlTool) Execute(ctx context.Context, args map[string]any) *ToolResult { + urlStr, ok := args["url"].(string) + if !ok || strings.TrimSpace(urlStr) == "" { + return ErrorResult("url is required") + } + + parsedURL, err := url.Parse(urlStr) + if err != nil { + return ErrorResult(fmt.Sprintf("invalid URL: %v", err)) + } + + if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { + return ErrorResult("only http/https URLs are allowed") + } + + if parsedURL.Host == "" { + return ErrorResult("missing domain in URL") + } + + if !isDomainAllowed(parsedURL.Hostname(), t.allowedDomains) { + return ErrorResult(fmt.Sprintf("domain %q is not in the allowed domains list", parsedURL.Hostname())) + } + + method := "GET" + if m, ok := args["method"].(string); ok && m != "" { + method = strings.ToUpper(m) + } + + var bodyReader io.Reader + if bodyStr, ok := args["body"].(string); ok && bodyStr != "" { + bodyReader = strings.NewReader(bodyStr) + } + + req, err := http.NewRequestWithContext(ctx, method, urlStr, bodyReader) + if err != nil { + return ErrorResult(fmt.Sprintf("failed to create request: %v", err)) + } + + req.Header.Set("User-Agent", fmt.Sprintf("picoclaw/curl (+https://github.com/sipeed/picoclaw)")) + + if headers, ok := args["headers"].(map[string]any); ok { + for k, v := range headers { + if vs, ok := v.(string); ok { + req.Header.Set(k, vs) + } + } + } + + timeout := curlDefaultTimeout + if tSec, ok := args["timeout"].(float64); ok && tSec > 0 { + timeout = time.Duration(tSec) * time.Second + if timeout > 120*time.Second { + timeout = 120 * time.Second + } + } + + ctxWithTimeout, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + req = req.WithContext(ctxWithTimeout) + + resp, err := t.client.Do(req) + if err != nil { + if ctxWithTimeout.Err() == context.DeadlineExceeded { + return ErrorResult(fmt.Sprintf("request timed out after %v", timeout)) + } + return ErrorResult(fmt.Sprintf("request failed: %v", err)) + } + defer resp.Body.Close() + + body, err := io.ReadAll(io.LimitReader(resp.Body, t.maxBytes)) + if err != nil { + if err == io.ErrUnexpectedEOF || strings.Contains(err.Error(), "http: request body too large") { + return ErrorResult(fmt.Sprintf("response body exceeded %d bytes limit", t.maxBytes)) + } + return ErrorResult(fmt.Sprintf("failed to read response: %v", err)) + } + + var headersOut map[string]string + if resp.Header != nil { + headersOut = make(map[string]string) + for k, v := range resp.Header { + if len(v) > 0 { + headersOut[k] = v[0] + } + } + } + + result := map[string]any{ + "status": resp.StatusCode, + "status_text": resp.Status, + "headers": headersOut, + "body": string(body), + "url": urlStr, + "method": method, + "truncated": int64(len(body)) >= t.maxBytes, + } + + return &ToolResult{ + ForLLM: formatCurlResult(result), + ForUser: fmt.Sprintf("HTTP %d from %s %s", resp.StatusCode, method, urlStr), + } +} + +func formatCurlResult(result map[string]any) string { + var b strings.Builder + b.WriteString(fmt.Sprintf("Status: %v\n", result["status"])) + b.WriteString(fmt.Sprintf("URL: %v\n", result["url"])) + b.WriteString(fmt.Sprintf("Method: %v\n", result["method"])) + + if headers, ok := result["headers"].(map[string]string); ok && len(headers) > 0 { + b.WriteString("Headers:\n") + for k, v := range headers { + b.WriteString(fmt.Sprintf(" %s: %s\n", k, v)) + } + } + + if truncated, ok := result["truncated"].(bool); ok && truncated { + b.WriteString("\n[Response body truncated due to size limit]\n") + } + + if body, ok := result["body"].(string); ok { + b.WriteString("\nBody:\n") + b.WriteString(body) + } + + return b.String() +} + +func isDomainAllowed(hostname string, allowedDomains []string) bool { + if len(allowedDomains) == 0 { + return true + } + hostname = strings.ToLower(strings.TrimSuffix(hostname, ".")) + for _, domain := range allowedDomains { + if hostname == domain || strings.HasSuffix(hostname, "."+domain) { + return true + } + } + return false +} + +func normalizeDomains(domains []string) []string { + result := make([]string, 0, len(domains)) + seen := make(map[string]struct{}) + for _, d := range domains { + d = strings.ToLower(strings.TrimSpace(d)) + d = strings.TrimPrefix(d, "http://") + d = strings.TrimPrefix(d, "https://") + d = strings.TrimSuffix(d, "/") + if d == "" { + continue + } + if _, exists := seen[d]; !exists { + seen[d] = struct{}{} + result = append(result, d) + } + } + return result +} diff --git a/pkg/tools/curl_test.go b/pkg/tools/curl_test.go new file mode 100644 index 000000000..8cb637eda --- /dev/null +++ b/pkg/tools/curl_test.go @@ -0,0 +1,242 @@ +package tools + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "testing" +) + +func TestCurlTool_Name(t *testing.T) { + tool, err := NewCurlTool(CurlToolOptions{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tool.Name() != "curl" { + t.Errorf("expected name 'curl', got %q", tool.Name()) + } +} + +func TestCurlTool_Description(t *testing.T) { + tool, err := NewCurlTool(CurlToolOptions{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tool.Description() == "" { + t.Error("description should not be empty") + } +} + +func TestCurlTool_Parameters(t *testing.T) { + tool, err := NewCurlTool(CurlToolOptions{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + params := tool.Parameters() + if params["type"] != "object" { + t.Errorf("expected type 'object', got %v", params["type"]) + } + props := params["properties"].(map[string]any) + if _, ok := props["url"]; !ok { + t.Error("parameters should include 'url'") + } + if _, ok := props["method"]; !ok { + t.Error("parameters should include 'method'") + } + required := params["required"].([]string) + found := false + for _, r := range required { + if r == "url" { + found = true + break + } + } + if !found { + t.Error("'url' should be required") + } +} + +func TestCurlTool_Execute_MissingURL(t *testing.T) { + tool, err := NewCurlTool(CurlToolOptions{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + result := tool.Execute(context.Background(), map[string]any{}) + if !result.IsError { + t.Error("expected error for missing url") + } +} + +func TestCurlTool_Execute_InvalidURL(t *testing.T) { + tool, err := NewCurlTool(CurlToolOptions{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + result := tool.Execute(context.Background(), map[string]any{"url": "://invalid"}) + if !result.IsError { + t.Error("expected error for invalid URL") + } +} + +func TestCurlTool_Execute_NonHTTPScheme(t *testing.T) { + tool, err := NewCurlTool(CurlToolOptions{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + result := tool.Execute(context.Background(), map[string]any{"url": "ftp://example.com/file"}) + if !result.IsError { + t.Error("expected error for non-http scheme") + } +} + +func TestCurlTool_Execute_DomainNotAllowed(t *testing.T) { + tool, err := NewCurlTool(CurlToolOptions{ + AllowedDomains: []string{"api.example.com"}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + result := tool.Execute(context.Background(), map[string]any{"url": "https://evil.com/data"}) + if !result.IsError { + t.Error("expected error for disallowed domain") + } +} + +func TestCurlTool_Execute_GET(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" { + t.Errorf("expected GET, got %s", r.Method) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) + })) + defer server.Close() + + tool, err := NewCurlTool(CurlToolOptions{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + result := tool.Execute(context.Background(), map[string]any{"url": server.URL}) + if result.IsError { + t.Fatalf("unexpected error: %s", result.ForLLM) + } + if !containsStr(result.ForLLM, `"status": "ok"`) && !containsStr(result.ForLLM, `status`) { + t.Errorf("expected response body, got: %s", result.ForLLM) + } +} + +func TestCurlTool_Execute_POST(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + t.Errorf("expected POST, got %s", r.Method) + } + w.WriteHeader(http.StatusCreated) + w.Write([]byte("created")) + })) + defer server.Close() + + tool, err := NewCurlTool(CurlToolOptions{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + result := tool.Execute(context.Background(), map[string]any{ + "url": server.URL, + "method": "POST", + "body": `{"key":"value"}`, + }) + if result.IsError { + t.Fatalf("unexpected error: %s", result.ForLLM) + } + if !containsStr(result.ForLLM, "201") { + t.Errorf("expected status 201, got: %s", result.ForLLM) + } +} + +func TestCurlTool_Execute_CustomHeaders(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + auth := r.Header.Get("Authorization") + if auth != "Bearer test-token" { + t.Errorf("expected Authorization header 'Bearer test-token', got %q", auth) + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + tool, err := NewCurlTool(CurlToolOptions{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + result := tool.Execute(context.Background(), map[string]any{ + "url": server.URL, + "method": "GET", + "headers": map[string]any{"Authorization": "Bearer test-token"}, + }) + if result.IsError { + t.Fatalf("unexpected error: %s", result.ForLLM) + } +} + +func TestCurlTool_Execute_DomainWhitelistSubdomain(t *testing.T) { + tool, err := NewCurlTool(CurlToolOptions{ + AllowedDomains: []string{"example.com"}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + parsed := mustParseURL("https://api.example.com/v1") + if !isDomainAllowed(parsed.Hostname(), tool.allowedDomains) { + t.Error("subdomain should be allowed when parent domain is in whitelist") + } +} + +func TestCurlTool_NormalizeDomains(t *testing.T) { + domains := normalizeDomains([]string{ + "HTTPS://Example.COM/", + "http://test.com", + " api.io ", + "", + "Example.COM", + }) + if len(domains) != 3 { + t.Errorf("expected 3 domains, got %d: %v", len(domains), domains) + } + expected := []string{"example.com", "test.com", "api.io"} + for i, d := range expected { + if domains[i] != d { + t.Errorf("expected domain[%d] = %q, got %q", i, d, domains[i]) + } + } +} + +func TestCurlTool_EmptyAllowedDomains(t *testing.T) { + if !isDomainAllowed("any.com", []string{}) { + t.Error("should allow any domain when whitelist is empty") + } + if !isDomainAllowed("any.com", nil) { + t.Error("should allow any domain when whitelist is nil") + } +} + +func containsStr(s, substr string) bool { + return len(s) >= len(substr) && searchStr(s, substr) +} + +func searchStr(s, substr string) bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} + +func mustParseURL(raw string) *url.URL { + u, err := url.Parse(raw) + if err != nil { + panic(err) + } + return u +}