feat(tools): add curl tool with domain whitelist and configurable limits (gh:1845)
This commit is contained in:
Fakhri A
parent
6a8552a664
commit
fe6833d5c3
3 changed files with 520 additions and 0 deletions
|
|
@ -842,6 +842,13 @@ func (c ReadFileToolConfig) EffectiveMode() string {
|
|||
}
|
||||
}
|
||||
|
||||
type CurlConfig struct {
|
||||
ToolConfig `json:"-" envPrefix:"PICOCLAW_TOOLS_CURL_"`
|
||||
AllowedDomains []string `json:"allowed_domains,omitempty" env:"PICOCLAW_TOOLS_CURL_ALLOWED_DOMAINS"`
|
||||
TimeoutSeconds int `json:"timeout_seconds" env:"PICOCLAW_TOOLS_CURL_TIMEOUT_SECONDS"`
|
||||
MaxBytes int64 `json:"max_bytes" env:"PICOCLAW_TOOLS_CURL_MAX_BYTES"`
|
||||
}
|
||||
|
||||
type ToolsConfig struct {
|
||||
AllowReadPaths []string `json:"allow_read_paths" yaml:"-" env:"PICOCLAW_TOOLS_ALLOW_READ_PATHS"`
|
||||
AllowWritePaths []string `json:"allow_write_paths" yaml:"-" env:"PICOCLAW_TOOLS_ALLOW_WRITE_PATHS"`
|
||||
|
|
@ -875,6 +882,7 @@ type ToolsConfig struct {
|
|||
Subagent ToolConfig `json:"subagent" yaml:"-" envPrefix:"PICOCLAW_TOOLS_SUBAGENT_"`
|
||||
WebFetch ToolConfig `json:"web_fetch" yaml:"-" envPrefix:"PICOCLAW_TOOLS_WEB_FETCH_"`
|
||||
WriteFile ToolConfig `json:"write_file" yaml:"-" envPrefix:"PICOCLAW_TOOLS_WRITE_FILE_"`
|
||||
Curl CurlConfig `json:"curl" yaml:"-" envPrefix:"PICOCLAW_TOOLS_CURL_"`
|
||||
}
|
||||
|
||||
// IsFilterSensitiveDataEnabled returns true if sensitive data filtering is enabled
|
||||
|
|
@ -1362,6 +1370,8 @@ func (t *ToolsConfig) IsToolEnabled(name string) bool {
|
|||
return t.SendTTS.Enabled
|
||||
case "write_file":
|
||||
return t.WriteFile.Enabled
|
||||
case "curl":
|
||||
return t.Curl.Enabled
|
||||
case "mcp":
|
||||
return t.MCP.Enabled
|
||||
default:
|
||||
|
|
|
|||
268
pkg/tools/curl.go
Normal file
268
pkg/tools/curl.go
Normal file
|
|
@ -0,0 +1,268 @@
|
|||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/sipeed/picoclaw/pkg/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
curlDefaultTimeout = 30 * time.Second
|
||||
curlDefaultMaxBytes = int64(1 << 20)
|
||||
curlMaxRedirects = 5
|
||||
)
|
||||
|
||||
type CurlTool struct {
|
||||
allowedDomains []string
|
||||
client *http.Client
|
||||
maxBytes int64
|
||||
}
|
||||
|
||||
type CurlToolOptions struct {
|
||||
AllowedDomains []string
|
||||
Proxy string
|
||||
TimeoutSeconds int
|
||||
MaxBytes int64
|
||||
}
|
||||
|
||||
func NewCurlTool(opts CurlToolOptions) (*CurlTool, error) {
|
||||
timeout := curlDefaultTimeout
|
||||
if opts.TimeoutSeconds > 0 {
|
||||
timeout = time.Duration(opts.TimeoutSeconds) * time.Second
|
||||
}
|
||||
|
||||
maxBytes := curlDefaultMaxBytes
|
||||
if opts.MaxBytes > 0 {
|
||||
maxBytes = opts.MaxBytes
|
||||
}
|
||||
|
||||
client, err := utils.CreateHTTPClient(opts.Proxy, timeout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create HTTP client for curl tool: %w", err)
|
||||
}
|
||||
|
||||
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= curlMaxRedirects {
|
||||
return fmt.Errorf("stopped after %d redirects", curlMaxRedirects)
|
||||
}
|
||||
if !isDomainAllowed(req.URL.Hostname(), opts.AllowedDomains) {
|
||||
return fmt.Errorf("redirect to disallowed domain %q", req.URL.Hostname())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return &CurlTool{
|
||||
allowedDomains: normalizeDomains(opts.AllowedDomains),
|
||||
client: client,
|
||||
maxBytes: maxBytes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (t *CurlTool) Name() string {
|
||||
return "curl"
|
||||
}
|
||||
|
||||
func (t *CurlTool) Description() string {
|
||||
return "Make HTTP requests to external APIs. Use url (required), method (GET/POST/PUT/DELETE/etc), headers (optional map), body (optional string), and timeout (optional seconds). Only allowed domains can be accessed."
|
||||
}
|
||||
|
||||
func (t *CurlTool) Parameters() map[string]any {
|
||||
return map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"url": map[string]any{
|
||||
"type": "string",
|
||||
"description": "URL to request (http/https only, must match allowed domains)",
|
||||
},
|
||||
"method": map[string]any{
|
||||
"type": "string",
|
||||
"description": "HTTP method (GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS)",
|
||||
"enum": []string{"GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"},
|
||||
},
|
||||
"headers": map[string]any{
|
||||
"type": "object",
|
||||
"description": "Optional HTTP headers as key-value pairs",
|
||||
"additionalProperties": map[string]any{
|
||||
"type": "string",
|
||||
},
|
||||
},
|
||||
"body": map[string]any{
|
||||
"type": "string",
|
||||
"description": "Optional request body (for POST, PUT, PATCH)",
|
||||
},
|
||||
"timeout": map[string]any{
|
||||
"type": "integer",
|
||||
"description": "Optional timeout in seconds (default: 30, max: 120)",
|
||||
"minimum": 1.0,
|
||||
"maximum": 120.0,
|
||||
},
|
||||
},
|
||||
"required": []string{"url"},
|
||||
}
|
||||
}
|
||||
|
||||
func (t *CurlTool) Execute(ctx context.Context, args map[string]any) *ToolResult {
|
||||
urlStr, ok := args["url"].(string)
|
||||
if !ok || strings.TrimSpace(urlStr) == "" {
|
||||
return ErrorResult("url is required")
|
||||
}
|
||||
|
||||
parsedURL, err := url.Parse(urlStr)
|
||||
if err != nil {
|
||||
return ErrorResult(fmt.Sprintf("invalid URL: %v", err))
|
||||
}
|
||||
|
||||
if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
|
||||
return ErrorResult("only http/https URLs are allowed")
|
||||
}
|
||||
|
||||
if parsedURL.Host == "" {
|
||||
return ErrorResult("missing domain in URL")
|
||||
}
|
||||
|
||||
if !isDomainAllowed(parsedURL.Hostname(), t.allowedDomains) {
|
||||
return ErrorResult(fmt.Sprintf("domain %q is not in the allowed domains list", parsedURL.Hostname()))
|
||||
}
|
||||
|
||||
method := "GET"
|
||||
if m, ok := args["method"].(string); ok && m != "" {
|
||||
method = strings.ToUpper(m)
|
||||
}
|
||||
|
||||
var bodyReader io.Reader
|
||||
if bodyStr, ok := args["body"].(string); ok && bodyStr != "" {
|
||||
bodyReader = strings.NewReader(bodyStr)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, urlStr, bodyReader)
|
||||
if err != nil {
|
||||
return ErrorResult(fmt.Sprintf("failed to create request: %v", err))
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", fmt.Sprintf("picoclaw/curl (+https://github.com/sipeed/picoclaw)"))
|
||||
|
||||
if headers, ok := args["headers"].(map[string]any); ok {
|
||||
for k, v := range headers {
|
||||
if vs, ok := v.(string); ok {
|
||||
req.Header.Set(k, vs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
timeout := curlDefaultTimeout
|
||||
if tSec, ok := args["timeout"].(float64); ok && tSec > 0 {
|
||||
timeout = time.Duration(tSec) * time.Second
|
||||
if timeout > 120*time.Second {
|
||||
timeout = 120 * time.Second
|
||||
}
|
||||
}
|
||||
|
||||
ctxWithTimeout, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
req = req.WithContext(ctxWithTimeout)
|
||||
|
||||
resp, err := t.client.Do(req)
|
||||
if err != nil {
|
||||
if ctxWithTimeout.Err() == context.DeadlineExceeded {
|
||||
return ErrorResult(fmt.Sprintf("request timed out after %v", timeout))
|
||||
}
|
||||
return ErrorResult(fmt.Sprintf("request failed: %v", err))
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, t.maxBytes))
|
||||
if err != nil {
|
||||
if err == io.ErrUnexpectedEOF || strings.Contains(err.Error(), "http: request body too large") {
|
||||
return ErrorResult(fmt.Sprintf("response body exceeded %d bytes limit", t.maxBytes))
|
||||
}
|
||||
return ErrorResult(fmt.Sprintf("failed to read response: %v", err))
|
||||
}
|
||||
|
||||
var headersOut map[string]string
|
||||
if resp.Header != nil {
|
||||
headersOut = make(map[string]string)
|
||||
for k, v := range resp.Header {
|
||||
if len(v) > 0 {
|
||||
headersOut[k] = v[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result := map[string]any{
|
||||
"status": resp.StatusCode,
|
||||
"status_text": resp.Status,
|
||||
"headers": headersOut,
|
||||
"body": string(body),
|
||||
"url": urlStr,
|
||||
"method": method,
|
||||
"truncated": int64(len(body)) >= t.maxBytes,
|
||||
}
|
||||
|
||||
return &ToolResult{
|
||||
ForLLM: formatCurlResult(result),
|
||||
ForUser: fmt.Sprintf("HTTP %d from %s %s", resp.StatusCode, method, urlStr),
|
||||
}
|
||||
}
|
||||
|
||||
func formatCurlResult(result map[string]any) string {
|
||||
var b strings.Builder
|
||||
b.WriteString(fmt.Sprintf("Status: %v\n", result["status"]))
|
||||
b.WriteString(fmt.Sprintf("URL: %v\n", result["url"]))
|
||||
b.WriteString(fmt.Sprintf("Method: %v\n", result["method"]))
|
||||
|
||||
if headers, ok := result["headers"].(map[string]string); ok && len(headers) > 0 {
|
||||
b.WriteString("Headers:\n")
|
||||
for k, v := range headers {
|
||||
b.WriteString(fmt.Sprintf(" %s: %s\n", k, v))
|
||||
}
|
||||
}
|
||||
|
||||
if truncated, ok := result["truncated"].(bool); ok && truncated {
|
||||
b.WriteString("\n[Response body truncated due to size limit]\n")
|
||||
}
|
||||
|
||||
if body, ok := result["body"].(string); ok {
|
||||
b.WriteString("\nBody:\n")
|
||||
b.WriteString(body)
|
||||
}
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func isDomainAllowed(hostname string, allowedDomains []string) bool {
|
||||
if len(allowedDomains) == 0 {
|
||||
return true
|
||||
}
|
||||
hostname = strings.ToLower(strings.TrimSuffix(hostname, "."))
|
||||
for _, domain := range allowedDomains {
|
||||
if hostname == domain || strings.HasSuffix(hostname, "."+domain) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func normalizeDomains(domains []string) []string {
|
||||
result := make([]string, 0, len(domains))
|
||||
seen := make(map[string]struct{})
|
||||
for _, d := range domains {
|
||||
d = strings.ToLower(strings.TrimSpace(d))
|
||||
d = strings.TrimPrefix(d, "http://")
|
||||
d = strings.TrimPrefix(d, "https://")
|
||||
d = strings.TrimSuffix(d, "/")
|
||||
if d == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[d]; !exists {
|
||||
seen[d] = struct{}{}
|
||||
result = append(result, d)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
242
pkg/tools/curl_test.go
Normal file
242
pkg/tools/curl_test.go
Normal file
|
|
@ -0,0 +1,242 @@
|
|||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCurlTool_Name(t *testing.T) {
|
||||
tool, err := NewCurlTool(CurlToolOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if tool.Name() != "curl" {
|
||||
t.Errorf("expected name 'curl', got %q", tool.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCurlTool_Description(t *testing.T) {
|
||||
tool, err := NewCurlTool(CurlToolOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if tool.Description() == "" {
|
||||
t.Error("description should not be empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCurlTool_Parameters(t *testing.T) {
|
||||
tool, err := NewCurlTool(CurlToolOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
params := tool.Parameters()
|
||||
if params["type"] != "object" {
|
||||
t.Errorf("expected type 'object', got %v", params["type"])
|
||||
}
|
||||
props := params["properties"].(map[string]any)
|
||||
if _, ok := props["url"]; !ok {
|
||||
t.Error("parameters should include 'url'")
|
||||
}
|
||||
if _, ok := props["method"]; !ok {
|
||||
t.Error("parameters should include 'method'")
|
||||
}
|
||||
required := params["required"].([]string)
|
||||
found := false
|
||||
for _, r := range required {
|
||||
if r == "url" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("'url' should be required")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCurlTool_Execute_MissingURL(t *testing.T) {
|
||||
tool, err := NewCurlTool(CurlToolOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
result := tool.Execute(context.Background(), map[string]any{})
|
||||
if !result.IsError {
|
||||
t.Error("expected error for missing url")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCurlTool_Execute_InvalidURL(t *testing.T) {
|
||||
tool, err := NewCurlTool(CurlToolOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
result := tool.Execute(context.Background(), map[string]any{"url": "://invalid"})
|
||||
if !result.IsError {
|
||||
t.Error("expected error for invalid URL")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCurlTool_Execute_NonHTTPScheme(t *testing.T) {
|
||||
tool, err := NewCurlTool(CurlToolOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
result := tool.Execute(context.Background(), map[string]any{"url": "ftp://example.com/file"})
|
||||
if !result.IsError {
|
||||
t.Error("expected error for non-http scheme")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCurlTool_Execute_DomainNotAllowed(t *testing.T) {
|
||||
tool, err := NewCurlTool(CurlToolOptions{
|
||||
AllowedDomains: []string{"api.example.com"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
result := tool.Execute(context.Background(), map[string]any{"url": "https://evil.com/data"})
|
||||
if !result.IsError {
|
||||
t.Error("expected error for disallowed domain")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCurlTool_Execute_GET(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != "GET" {
|
||||
t.Errorf("expected GET, got %s", r.Method)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
tool, err := NewCurlTool(CurlToolOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
result := tool.Execute(context.Background(), map[string]any{"url": server.URL})
|
||||
if result.IsError {
|
||||
t.Fatalf("unexpected error: %s", result.ForLLM)
|
||||
}
|
||||
if !containsStr(result.ForLLM, `"status": "ok"`) && !containsStr(result.ForLLM, `status`) {
|
||||
t.Errorf("expected response body, got: %s", result.ForLLM)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCurlTool_Execute_POST(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != "POST" {
|
||||
t.Errorf("expected POST, got %s", r.Method)
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
w.Write([]byte("created"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
tool, err := NewCurlTool(CurlToolOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
result := tool.Execute(context.Background(), map[string]any{
|
||||
"url": server.URL,
|
||||
"method": "POST",
|
||||
"body": `{"key":"value"}`,
|
||||
})
|
||||
if result.IsError {
|
||||
t.Fatalf("unexpected error: %s", result.ForLLM)
|
||||
}
|
||||
if !containsStr(result.ForLLM, "201") {
|
||||
t.Errorf("expected status 201, got: %s", result.ForLLM)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCurlTool_Execute_CustomHeaders(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
auth := r.Header.Get("Authorization")
|
||||
if auth != "Bearer test-token" {
|
||||
t.Errorf("expected Authorization header 'Bearer test-token', got %q", auth)
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
tool, err := NewCurlTool(CurlToolOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
result := tool.Execute(context.Background(), map[string]any{
|
||||
"url": server.URL,
|
||||
"method": "GET",
|
||||
"headers": map[string]any{"Authorization": "Bearer test-token"},
|
||||
})
|
||||
if result.IsError {
|
||||
t.Fatalf("unexpected error: %s", result.ForLLM)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCurlTool_Execute_DomainWhitelistSubdomain(t *testing.T) {
|
||||
tool, err := NewCurlTool(CurlToolOptions{
|
||||
AllowedDomains: []string{"example.com"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
parsed := mustParseURL("https://api.example.com/v1")
|
||||
if !isDomainAllowed(parsed.Hostname(), tool.allowedDomains) {
|
||||
t.Error("subdomain should be allowed when parent domain is in whitelist")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCurlTool_NormalizeDomains(t *testing.T) {
|
||||
domains := normalizeDomains([]string{
|
||||
"HTTPS://Example.COM/",
|
||||
"http://test.com",
|
||||
" api.io ",
|
||||
"",
|
||||
"Example.COM",
|
||||
})
|
||||
if len(domains) != 3 {
|
||||
t.Errorf("expected 3 domains, got %d: %v", len(domains), domains)
|
||||
}
|
||||
expected := []string{"example.com", "test.com", "api.io"}
|
||||
for i, d := range expected {
|
||||
if domains[i] != d {
|
||||
t.Errorf("expected domain[%d] = %q, got %q", i, d, domains[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCurlTool_EmptyAllowedDomains(t *testing.T) {
|
||||
if !isDomainAllowed("any.com", []string{}) {
|
||||
t.Error("should allow any domain when whitelist is empty")
|
||||
}
|
||||
if !isDomainAllowed("any.com", nil) {
|
||||
t.Error("should allow any domain when whitelist is nil")
|
||||
}
|
||||
}
|
||||
|
||||
func containsStr(s, substr string) bool {
|
||||
return len(s) >= len(substr) && searchStr(s, substr)
|
||||
}
|
||||
|
||||
func searchStr(s, substr string) bool {
|
||||
for i := 0; i <= len(s)-len(substr); i++ {
|
||||
if s[i:i+len(substr)] == substr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func mustParseURL(raw string) *url.URL {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return u
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue