diff --git a/config/config.example.json b/config/config.example.json index aa75c8338..409449a7d 100644 --- a/config/config.example.json +++ b/config/config.example.json @@ -115,6 +115,16 @@ "api_key": "YOUR_BRAVE_API_KEY", "max_results": 5 } + }, + "browser": { + "enabled": false, + "protocol": "cdp", + "cdp_url": "ws://localhost:3000", + "ws_url": "", + "token": "YOUR_BROWSER_TOKEN", + "stealth": true, + "launch_timeout": 120000, + "action_timeout": 30000 } }, "heartbeat": { diff --git a/go.mod b/go.mod index 98aecd6ab..0b887d6ee 100644 --- a/go.mod +++ b/go.mod @@ -14,23 +14,25 @@ require ( github.com/mymmrac/telego v1.6.0 github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1 github.com/openai/openai-go/v3 v3.22.0 + github.com/playwright-community/playwright-go v0.5200.1 github.com/slack-go/slack v0.17.3 github.com/tencent-connect/botgo v0.2.1 golang.org/x/oauth2 v0.35.0 ) - - require ( github.com/andybalholm/brotli v1.2.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect + github.com/deckarep/golang-set/v2 v2.7.0 // indirect github.com/github/copilot-sdk/go v0.1.23 - github.com/google/jsonschema-go v0.4.2 // indirect + github.com/go-jose/go-jose/v3 v3.0.4 // indirect github.com/go-resty/resty/v2 v2.17.1 // indirect + github.com/go-stack/stack v1.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/jsonschema-go v0.4.2 // indirect github.com/grbit/go-json v0.11.0 // indirect github.com/klauspost/compress v1.18.4 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect @@ -47,5 +49,4 @@ require ( golang.org/x/net v0.50.0 // indirect golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.41.0 // indirect - ) diff --git a/go.sum b/go.sum index 6a565b93e..0238280f0 100644 --- a/go.sum +++ b/go.sum @@ -29,15 +29,21 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set/v2 v2.7.0 h1:gIloKvD7yH2oip4VLhsv3JyLLFnC0Y2mlusgcvJYW5k= +github.com/deckarep/golang-set/v2 v2.7.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/github/copilot-sdk/go v0.1.23 h1:uExtO/inZQndCZMiSAA1hvXINiz9tqo/MZgQzFzurxw= github.com/github/copilot-sdk/go v0.1.23/go.mod h1:GdwwBfMbm9AABLEM3x5IZKw4ZfwCYxZ1BgyytmZenQ0= +github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY= +github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q= github.com/go-resty/resty/v2 v2.17.1 h1:x3aMpHK1YM9e4va/TMDRlusDDoZiQ+ViDu/WpA6xTM4= github.com/go-resty/resty/v2 v2.17.1/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= @@ -58,6 +64,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -84,6 +92,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/larksuite/oapi-sdk-go/v3 v3.5.3 h1:xvf8Dv29kBXC5/DNDCLhHkAFW8l/0LlQJimO5Zn+JUk= github.com/larksuite/oapi-sdk-go/v3 v3.5.3/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= +github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mymmrac/telego v1.6.0 h1:Zc8rgyHozvd/7ZgyrigyHdAF9koHYMfilYfyB6wlFC0= github.com/mymmrac/telego v1.6.0/go.mod h1:xt6ZWA8zi8KmuzryE1ImEdl9JSwjHNpM4yhC7D8hU4Y= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -99,6 +109,8 @@ github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1/go.mod h1:ln3IqPYYocZbYvl github.com/openai/openai-go/v3 v3.22.0 h1:6MEoNoV8sbjOVmXdvhmuX3BjVbVdcExbVyGixiyJ8ys= github.com/openai/openai-go/v3 v3.22.0/go.mod h1:cdufnVK14cWcT9qA1rRtrXx4FTRsgbDPW7Ia7SS5cZo= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/playwright-community/playwright-go v0.5200.1 h1:Sm2oOuhqt0M5Y4kUi/Qh9w4cyyi3ZIWTBeGKImc2UVo= +github.com/playwright-community/playwright-go v0.5200.1/go.mod h1:UnnyQZaqUOO5ywAZu60+N4EiWReUqX1MQBBA3Oofvf8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -110,6 +122,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -154,6 +167,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -205,6 +219,7 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -212,6 +227,7 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index f3dd94090..e23da0791 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -86,6 +86,24 @@ func createToolRegistry(workspace string, restrict bool, cfg *config.Config, msg registry.Register(tools.NewI2CTool()) registry.Register(tools.NewSPITool()) + // Browser tool (Playwright — supports CDP and Playwright Wire Protocol) + if cfg.Tools.Browser.Enabled { + wsURL := cfg.Tools.Browser.CdpURL + if cfg.Tools.Browser.WsURL != "" { + wsURL = cfg.Tools.Browser.WsURL + } + if wsURL != "" { + registry.Register(tools.NewBrowserTool(tools.BrowserToolOptions{ + Protocol: cfg.Tools.Browser.Protocol, + WsURL: wsURL, + Token: cfg.Tools.Browser.Token, + Stealth: cfg.Tools.Browser.Stealth, + LaunchTimeout: cfg.Tools.Browser.LaunchTimeout, + ActionTimeout: cfg.Tools.Browser.ActionTimeout, + })) + } + } + // Message tool - available to both agent and subagent // Subagent uses it to communicate directly with user messageTool := tools.NewMessageTool() diff --git a/pkg/config/config.go b/pkg/config/config.go index d76ec8095..fcdd21881 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -210,8 +210,20 @@ type WebToolsConfig struct { DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"` } +type BrowserConfig struct { + Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_BROWSER_ENABLED"` + Protocol string `json:"protocol" env:"PICOCLAW_TOOLS_BROWSER_PROTOCOL"` + CdpURL string `json:"cdp_url" env:"PICOCLAW_TOOLS_BROWSER_CDP_URL"` + WsURL string `json:"ws_url" env:"PICOCLAW_TOOLS_BROWSER_WS_URL"` + Token string `json:"token" env:"PICOCLAW_TOOLS_BROWSER_TOKEN"` + Stealth bool `json:"stealth" env:"PICOCLAW_TOOLS_BROWSER_STEALTH"` + LaunchTimeout int `json:"launch_timeout" env:"PICOCLAW_TOOLS_BROWSER_LAUNCH_TIMEOUT"` + ActionTimeout int `json:"action_timeout" env:"PICOCLAW_TOOLS_BROWSER_ACTION_TIMEOUT"` +} + type ToolsConfig struct { - Web WebToolsConfig `json:"web"` + Web WebToolsConfig `json:"web"` + Browser BrowserConfig `json:"browser"` } func DefaultConfig() *Config { diff --git a/pkg/tools/browser.go b/pkg/tools/browser.go new file mode 100644 index 000000000..fe6ccd62a --- /dev/null +++ b/pkg/tools/browser.go @@ -0,0 +1,674 @@ +package tools + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "strings" + "sync" + "time" + + "github.com/playwright-community/playwright-go" +) + +// BrowserToolOptions configures the browser tool. +type BrowserToolOptions struct { + Protocol string // "cdp" (default) or "playwright" + WsURL string // WebSocket URL for the remote browser + Token string // Auth token (used for CDP/Browserless) + Stealth bool // Request stealth mode via launch params + LaunchTimeout int // Connection timeout in ms (default 120000) + ActionTimeout int // Per-action timeout in ms (default 30000) +} + +// BrowserTool provides browser automation via Playwright. +// Supports two connection protocols: +// - "cdp": Chromium via ConnectOverCDP (e.g. Browserless) +// - "playwright": Firefox via Connect (e.g. Camoufox) +type BrowserTool struct { + wsURL string + token string + protocol string + stealth bool + launchTimeout time.Duration + actionTimeout time.Duration + + pw *playwright.Playwright + browser playwright.Browser + page playwright.Page + connected bool + mu sync.Mutex +} + +// NewBrowserTool creates a new browser tool with the given options. +func NewBrowserTool(opts BrowserToolOptions) *BrowserTool { + protocol := opts.Protocol + if protocol == "" { + protocol = "cdp" + } + + launchTimeout := 120 * time.Second + if opts.LaunchTimeout > 0 { + launchTimeout = time.Duration(opts.LaunchTimeout) * time.Millisecond + } + + actionTimeout := 30 * time.Second + if opts.ActionTimeout > 0 { + actionTimeout = time.Duration(opts.ActionTimeout) * time.Millisecond + } + + return &BrowserTool{ + wsURL: opts.WsURL, + token: opts.Token, + protocol: protocol, + stealth: opts.Stealth, + launchTimeout: launchTimeout, + actionTimeout: actionTimeout, + } +} + +func (t *BrowserTool) Name() string { return "browser" } + +func (t *BrowserTool) Description() string { + return "Control a remote browser to navigate web pages, take screenshots, click elements, fill forms, and extract content. " + + "Supports Chromium (CDP) and Firefox (Playwright Wire Protocol)." +} + +func (t *BrowserTool) Parameters() map[string]interface{} { + return map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "action": map[string]interface{}{ + "type": "string", + "description": "The browser action to perform", + "enum": []string{ + "navigate", "click", "type", "screenshot", "get_text", + "evaluate", "wait", "scroll", "hover", "select", + "pdf", "cookies", "close", + }, + }, + "url": map[string]interface{}{ + "type": "string", + "description": "URL to navigate to (for 'navigate' action)", + }, + "selector": map[string]interface{}{ + "type": "string", + "description": "CSS selector for the target element", + }, + "text": map[string]interface{}{ + "type": "string", + "description": "Text to type into the element (for 'type' action)", + }, + "expression": map[string]interface{}{ + "type": "string", + "description": "JavaScript expression to evaluate (for 'evaluate' action)", + }, + "direction": map[string]interface{}{ + "type": "string", + "description": "Scroll direction: 'up' or 'down' (for 'scroll' action)", + "enum": []string{"up", "down"}, + }, + "amount": map[string]interface{}{ + "type": "number", + "description": "Scroll amount in pixels (default 500)", + }, + "values": map[string]interface{}{ + "type": "array", + "items": map[string]interface{}{"type": "string"}, + "description": "Values to select (for 'select' action)", + }, + "cookie_action": map[string]interface{}{ + "type": "string", + "description": "Cookie sub-action: 'get', 'set', 'delete', 'clear'", + "enum": []string{"get", "set", "delete", "clear"}, + }, + "cookie_name": map[string]interface{}{ + "type": "string", + "description": "Cookie name (for set/delete)", + }, + "cookie_value": map[string]interface{}{ + "type": "string", + "description": "Cookie value (for set)", + }, + "cookie_domain": map[string]interface{}{ + "type": "string", + "description": "Cookie domain (for set)", + }, + "cookie_url": map[string]interface{}{ + "type": "string", + "description": "Cookie URL (for set, alternative to domain)", + }, + "timeout": map[string]interface{}{ + "type": "number", + "description": "Action timeout in milliseconds (overrides default)", + }, + "full_page": map[string]interface{}{ + "type": "boolean", + "description": "Take full-page screenshot (default true)", + }, + }, + "required": []string{"action"}, + } +} + +func (t *BrowserTool) Execute(ctx context.Context, args map[string]interface{}) *ToolResult { + action, _ := args["action"].(string) + if action == "" { + return &ToolResult{ForLLM: "Error: 'action' parameter is required"} + } + + // Ensure browser connection (except for close action) + if action != "close" { + if err := t.ensureConnected(); err != nil { + return &ToolResult{ForLLM: fmt.Sprintf("Error connecting to browser: %v", err)} + } + } + + switch action { + case "navigate": + return t.doNavigate(args) + case "click": + return t.doClick(args) + case "type": + return t.doType(args) + case "screenshot": + return t.doScreenshot(args) + case "get_text": + return t.doGetText(args) + case "evaluate": + return t.doEvaluate(args) + case "wait": + return t.doWait(args) + case "scroll": + return t.doScroll(args) + case "hover": + return t.doHover(args) + case "select": + return t.doSelect(args) + case "pdf": + return t.doPDF(args) + case "cookies": + return t.doCookies(args) + case "close": + return t.doClose() + default: + return &ToolResult{ForLLM: fmt.Sprintf("Error: unknown action '%s'", action)} + } +} + +// ensureConnected establishes a browser connection if not already connected. +func (t *BrowserTool) ensureConnected() error { + t.mu.Lock() + defer t.mu.Unlock() + + if t.connected && t.page != nil { + return nil + } + + pw, err := playwright.Run() + if err != nil { + return fmt.Errorf("failed to start playwright driver: %w", err) + } + + timeoutMs := float64(t.launchTimeout.Milliseconds()) + + var browser playwright.Browser + + switch t.protocol { + case "playwright": + // Firefox via Playwright Wire Protocol (e.g. Camoufox) + browser, err = pw.Firefox.Connect(t.wsURL, playwright.BrowserTypeConnectOptions{ + Timeout: &timeoutMs, + }) + if err != nil { + pw.Stop() + return fmt.Errorf("failed to connect via Playwright protocol to %s: %w", t.wsURL, err) + } + default: + // Chromium via CDP (e.g. Browserless) + cdpURL := t.buildCdpURL() + browser, err = pw.Chromium.ConnectOverCDP(cdpURL, playwright.BrowserTypeConnectOverCDPOptions{ + Timeout: &timeoutMs, + }) + if err != nil { + pw.Stop() + return fmt.Errorf("failed to connect via CDP to %s: %w", cdpURL, err) + } + } + + // Reuse existing page from browser context if available (common for CDP) + var page playwright.Page + contexts := browser.Contexts() + if len(contexts) > 0 && len(contexts[0].Pages()) > 0 { + page = contexts[0].Pages()[0] + } else { + page, err = browser.NewPage() + if err != nil { + browser.Close() + pw.Stop() + return fmt.Errorf("failed to create new page: %w", err) + } + } + + t.pw = pw + t.browser = browser + t.page = page + t.connected = true + + return nil +} + +// buildCdpURL constructs the CDP WebSocket URL with token and launch params. +func (t *BrowserTool) buildCdpURL() string { + u := t.wsURL + + params := []string{} + if t.token != "" { + params = append(params, "token="+t.token) + } + if t.stealth { + params = append(params, "stealth=true") + } + if t.launchTimeout > 0 { + launchJSON := fmt.Sprintf(`{"timeout":%d}`, t.launchTimeout.Milliseconds()) + params = append(params, "launch="+launchJSON) + } + + if len(params) > 0 { + separator := "?" + if strings.Contains(u, "?") { + separator = "&" + } + u += separator + strings.Join(params, "&") + } + + return u +} + +// getTimeout returns the action timeout, optionally overridden by args. +func (t *BrowserTool) getTimeout(args map[string]interface{}) float64 { + if v, ok := args["timeout"].(float64); ok && v > 0 { + return v + } + return float64(t.actionTimeout.Milliseconds()) +} + +func (t *BrowserTool) doNavigate(args map[string]interface{}) *ToolResult { + url, _ := args["url"].(string) + if url == "" { + return &ToolResult{ForLLM: "Error: 'url' parameter is required for navigate action"} + } + + timeout := t.getTimeout(args) + resp, err := t.page.Goto(url, playwright.PageGotoOptions{ + Timeout: &timeout, + WaitUntil: playwright.WaitUntilStateDomcontentloaded, + }) + if err != nil { + return &ToolResult{ForLLM: fmt.Sprintf("Error navigating to %s: %v", url, err)} + } + + status := 0 + if resp != nil { + status = resp.Status() + } + + title, _ := t.page.Title() + + return &ToolResult{ + ForLLM: fmt.Sprintf("Navigated to %s (status: %d, title: %q)", url, status, title), + } +} + +func (t *BrowserTool) doClick(args map[string]interface{}) *ToolResult { + selector, _ := args["selector"].(string) + if selector == "" { + return &ToolResult{ForLLM: "Error: 'selector' parameter is required for click action"} + } + + timeout := t.getTimeout(args) + err := t.page.Click(selector, playwright.PageClickOptions{ + Timeout: &timeout, + }) + if err != nil { + return &ToolResult{ForLLM: fmt.Sprintf("Error clicking %q: %v", selector, err)} + } + + return &ToolResult{ForLLM: fmt.Sprintf("Clicked element: %s", selector)} +} + +func (t *BrowserTool) doType(args map[string]interface{}) *ToolResult { + selector, _ := args["selector"].(string) + if selector == "" { + return &ToolResult{ForLLM: "Error: 'selector' parameter is required for type action"} + } + text, _ := args["text"].(string) + + timeout := t.getTimeout(args) + err := t.page.Fill(selector, text, playwright.PageFillOptions{ + Timeout: &timeout, + }) + if err != nil { + return &ToolResult{ForLLM: fmt.Sprintf("Error typing into %q: %v", selector, err)} + } + + return &ToolResult{ForLLM: fmt.Sprintf("Typed %d characters into %s", len(text), selector)} +} + +func (t *BrowserTool) doScreenshot(args map[string]interface{}) *ToolResult { + fullPage := true + if v, ok := args["full_page"].(bool); ok { + fullPage = v + } + + selector, _ := args["selector"].(string) + + var data []byte + var err error + + if selector != "" { + // Screenshot a specific element + locator := t.page.Locator(selector) + timeout := t.getTimeout(args) + data, err = locator.Screenshot(playwright.LocatorScreenshotOptions{ + Timeout: &timeout, + }) + } else { + // Full page or viewport screenshot + data, err = t.page.Screenshot(playwright.PageScreenshotOptions{ + FullPage: playwright.Bool(fullPage), + }) + } + + if err != nil { + return &ToolResult{ForLLM: fmt.Sprintf("Error taking screenshot: %v", err)} + } + + encoded := base64.StdEncoding.EncodeToString(data) + + title, _ := t.page.Title() + url := t.page.URL() + + return &ToolResult{ + ForLLM: fmt.Sprintf("Screenshot taken (page: %q, url: %s, size: %d bytes). Image data returned as base64.", title, url, len(data)), + ForUser: fmt.Sprintf("![screenshot](data:image/png;base64,%s)", encoded), + } +} + +func (t *BrowserTool) doGetText(args map[string]interface{}) *ToolResult { + selector, _ := args["selector"].(string) + if selector == "" { + selector = "body" + } + + timeout := t.getTimeout(args) + text, err := t.page.InnerText(selector, playwright.PageInnerTextOptions{ + Timeout: &timeout, + }) + if err != nil { + return &ToolResult{ForLLM: fmt.Sprintf("Error getting text from %q: %v", selector, err)} + } + + // Truncate very long text + const maxLen = 50000 + if len(text) > maxLen { + text = text[:maxLen] + "\n... [truncated]" + } + + return &ToolResult{ForLLM: fmt.Sprintf("Text from %q (%d chars):\n%s", selector, len(text), text)} +} + +func (t *BrowserTool) doEvaluate(args map[string]interface{}) *ToolResult { + expression, _ := args["expression"].(string) + if expression == "" { + return &ToolResult{ForLLM: "Error: 'expression' parameter is required for evaluate action"} + } + + result, err := t.page.Evaluate(expression) + if err != nil { + return &ToolResult{ForLLM: fmt.Sprintf("Error evaluating JS: %v", err)} + } + + resultJSON, _ := json.Marshal(result) + return &ToolResult{ForLLM: fmt.Sprintf("Evaluation result: %s", string(resultJSON))} +} + +func (t *BrowserTool) doWait(args map[string]interface{}) *ToolResult { + selector, _ := args["selector"].(string) + if selector == "" { + return &ToolResult{ForLLM: "Error: 'selector' parameter is required for wait action"} + } + + timeout := t.getTimeout(args) + locator := t.page.Locator(selector) + err := locator.WaitFor(playwright.LocatorWaitForOptions{ + State: playwright.WaitForSelectorStateVisible, + Timeout: &timeout, + }) + if err != nil { + return &ToolResult{ForLLM: fmt.Sprintf("Error waiting for %q: %v", selector, err)} + } + + return &ToolResult{ForLLM: fmt.Sprintf("Element %q is now visible", selector)} +} + +func (t *BrowserTool) doScroll(args map[string]interface{}) *ToolResult { + direction, _ := args["direction"].(string) + if direction == "" { + direction = "down" + } + + amount := 500.0 + if v, ok := args["amount"].(float64); ok && v > 0 { + amount = v + } + + if direction == "up" { + amount = -amount + } + + js := fmt.Sprintf("window.scrollBy(0, %f)", amount) + _, err := t.page.Evaluate(js) + if err != nil { + return &ToolResult{ForLLM: fmt.Sprintf("Error scrolling: %v", err)} + } + + return &ToolResult{ForLLM: fmt.Sprintf("Scrolled %s by %.0f pixels", direction, amount)} +} + +func (t *BrowserTool) doHover(args map[string]interface{}) *ToolResult { + selector, _ := args["selector"].(string) + if selector == "" { + return &ToolResult{ForLLM: "Error: 'selector' parameter is required for hover action"} + } + + timeout := t.getTimeout(args) + err := t.page.Hover(selector, playwright.PageHoverOptions{ + Timeout: &timeout, + }) + if err != nil { + return &ToolResult{ForLLM: fmt.Sprintf("Error hovering on %q: %v", selector, err)} + } + + return &ToolResult{ForLLM: fmt.Sprintf("Hovering on element: %s", selector)} +} + +func (t *BrowserTool) doSelect(args map[string]interface{}) *ToolResult { + selector, _ := args["selector"].(string) + if selector == "" { + return &ToolResult{ForLLM: "Error: 'selector' parameter is required for select action"} + } + + rawValues, _ := args["values"].([]interface{}) + values := make([]string, 0, len(rawValues)) + for _, v := range rawValues { + if s, ok := v.(string); ok { + values = append(values, s) + } + } + + timeout := t.getTimeout(args) + chosen, err := t.page.SelectOption(selector, playwright.SelectOptionValues{ + Values: &values, + }, playwright.PageSelectOptionOptions{ + Timeout: &timeout, + }) + if err != nil { + return &ToolResult{ForLLM: fmt.Sprintf("Error selecting options in %q: %v", selector, err)} + } + + return &ToolResult{ForLLM: fmt.Sprintf("Selected %d option(s) in %s: %v", len(chosen), selector, chosen)} +} + +func (t *BrowserTool) doPDF(args map[string]interface{}) *ToolResult { + if t.protocol == "playwright" { + return &ToolResult{ForLLM: "Error: PDF generation is only supported with Chromium (CDP protocol). Current protocol is 'playwright' (Firefox)."} + } + + data, err := t.page.PDF() + if err != nil { + return &ToolResult{ForLLM: fmt.Sprintf("Error generating PDF: %v", err)} + } + + encoded := base64.StdEncoding.EncodeToString(data) + + return &ToolResult{ + ForLLM: fmt.Sprintf("PDF generated (%d bytes). Data returned as base64.", len(data)), + ForUser: encoded, + } +} + +func (t *BrowserTool) doCookies(args map[string]interface{}) *ToolResult { + cookieAction, _ := args["cookie_action"].(string) + if cookieAction == "" { + return &ToolResult{ForLLM: "Error: 'cookie_action' parameter is required (get, set, delete, clear)"} + } + + browserCtx := t.page.Context() + + switch cookieAction { + case "get": + cookies, err := browserCtx.Cookies() + if err != nil { + return &ToolResult{ForLLM: fmt.Sprintf("Error getting cookies: %v", err)} + } + data, _ := json.MarshalIndent(cookies, "", " ") + return &ToolResult{ForLLM: fmt.Sprintf("Cookies (%d):\n%s", len(cookies), string(data))} + + case "set": + name, _ := args["cookie_name"].(string) + value, _ := args["cookie_value"].(string) + domain, _ := args["cookie_domain"].(string) + cookieURL, _ := args["cookie_url"].(string) + + if name == "" { + return &ToolResult{ForLLM: "Error: 'cookie_name' is required for set"} + } + + cookie := playwright.OptionalCookie{ + Name: name, + Value: value, + } + + if cookieURL != "" { + cookie.URL = &cookieURL + } else if domain != "" { + cookie.Domain = &domain + defaultPath := "/" + cookie.Path = &defaultPath + } else { + // Use current page URL as fallback + pageURL := t.page.URL() + cookie.URL = &pageURL + } + + err := browserCtx.AddCookies([]playwright.OptionalCookie{cookie}) + if err != nil { + return &ToolResult{ForLLM: fmt.Sprintf("Error setting cookie: %v", err)} + } + return &ToolResult{ForLLM: fmt.Sprintf("Cookie %q set successfully", name)} + + case "delete": + name, _ := args["cookie_name"].(string) + if name == "" { + return &ToolResult{ForLLM: "Error: 'cookie_name' is required for delete"} + } + + // Get all cookies, clear, re-add all except target + cookies, err := browserCtx.Cookies() + if err != nil { + return &ToolResult{ForLLM: fmt.Sprintf("Error getting cookies for delete: %v", err)} + } + + err = browserCtx.ClearCookies() + if err != nil { + return &ToolResult{ForLLM: fmt.Sprintf("Error clearing cookies: %v", err)} + } + + var toReAdd []playwright.OptionalCookie + for _, c := range cookies { + if c.Name == name { + continue + } + oc := playwright.OptionalCookie{ + Name: c.Name, + Value: c.Value, + } + if c.Domain != "" { + oc.Domain = &c.Domain + oc.Path = &c.Path + } + toReAdd = append(toReAdd, oc) + } + + if len(toReAdd) > 0 { + browserCtx.AddCookies(toReAdd) + } + + return &ToolResult{ForLLM: fmt.Sprintf("Cookie %q deleted", name)} + + case "clear": + err := browserCtx.ClearCookies() + if err != nil { + return &ToolResult{ForLLM: fmt.Sprintf("Error clearing cookies: %v", err)} + } + return &ToolResult{ForLLM: "All cookies cleared"} + + default: + return &ToolResult{ForLLM: fmt.Sprintf("Error: unknown cookie_action %q", cookieAction)} + } +} + +func (t *BrowserTool) doClose() *ToolResult { + t.mu.Lock() + defer t.mu.Unlock() + + if !t.connected { + return &ToolResult{ForLLM: "Browser is not connected"} + } + + var errs []string + + if t.browser != nil { + if err := t.browser.Close(); err != nil { + errs = append(errs, fmt.Sprintf("browser close: %v", err)) + } + } + if t.pw != nil { + if err := t.pw.Stop(); err != nil { + errs = append(errs, fmt.Sprintf("playwright stop: %v", err)) + } + } + + t.page = nil + t.browser = nil + t.pw = nil + t.connected = false + + if len(errs) > 0 { + return &ToolResult{ForLLM: fmt.Sprintf("Browser closed with warnings: %s", strings.Join(errs, "; "))} + } + + return &ToolResult{ForLLM: "Browser closed successfully"} +} diff --git a/pkg/tools/browser_test.go b/pkg/tools/browser_test.go new file mode 100644 index 000000000..ac522b859 --- /dev/null +++ b/pkg/tools/browser_test.go @@ -0,0 +1,309 @@ +package tools + +import ( + "context" + "os" + "testing" +) + +func TestBrowserTool_Unit_Name(t *testing.T) { + bt := NewBrowserTool(BrowserToolOptions{WsURL: "ws://localhost:3000"}) + if bt.Name() != "browser" { + t.Errorf("expected name 'browser', got %q", bt.Name()) + } +} + +func TestBrowserTool_Unit_Parameters(t *testing.T) { + bt := NewBrowserTool(BrowserToolOptions{WsURL: "ws://localhost:3000"}) + params := bt.Parameters() + props, ok := params["properties"].(map[string]interface{}) + if !ok { + t.Fatal("expected properties map") + } + if _, ok := props["action"]; !ok { + t.Error("expected 'action' in parameters") + } + if _, ok := props["url"]; !ok { + t.Error("expected 'url' in parameters") + } + if _, ok := props["selector"]; !ok { + t.Error("expected 'selector' in parameters") + } +} + +func TestBrowserTool_Unit_MissingAction(t *testing.T) { + bt := NewBrowserTool(BrowserToolOptions{WsURL: "ws://localhost:3000"}) + result := bt.Execute(context.Background(), map[string]interface{}{}) + if result.ForLLM != "Error: 'action' parameter is required" { + t.Errorf("unexpected result: %s", result.ForLLM) + } +} + +func TestBrowserTool_Unit_UnknownAction(t *testing.T) { + bt := NewBrowserTool(BrowserToolOptions{WsURL: "ws://localhost:3000"}) + // Unknown action should fail at connect step since no browser is available, + // but we test the action routing by checking close (which doesn't need connection) + result := bt.Execute(context.Background(), map[string]interface{}{ + "action": "close", + }) + if result.ForLLM != "Browser is not connected" { + t.Errorf("unexpected result for close without connect: %s", result.ForLLM) + } +} + +func TestBrowserTool_Unit_CloseWithoutConnect(t *testing.T) { + bt := NewBrowserTool(BrowserToolOptions{WsURL: "ws://localhost:3000"}) + result := bt.Execute(context.Background(), map[string]interface{}{ + "action": "close", + }) + if result.ForLLM != "Browser is not connected" { + t.Errorf("expected 'Browser is not connected', got %q", result.ForLLM) + } +} + +func TestBrowserTool_Unit_NavigateMissingURL(t *testing.T) { + bt := NewBrowserTool(BrowserToolOptions{WsURL: "ws://localhost:3000"}) + // This will fail at ensureConnected, but we verify the tool doesn't panic + result := bt.Execute(context.Background(), map[string]interface{}{ + "action": "navigate", + }) + if result.ForLLM == "" { + t.Error("expected error message for navigate without URL") + } +} + +func TestBrowserTool_Unit_DefaultProtocol(t *testing.T) { + bt := NewBrowserTool(BrowserToolOptions{WsURL: "ws://localhost:3000"}) + if bt.protocol != "cdp" { + t.Errorf("expected default protocol 'cdp', got %q", bt.protocol) + } +} + +func TestBrowserTool_Unit_PlaywrightProtocol(t *testing.T) { + bt := NewBrowserTool(BrowserToolOptions{ + WsURL: "ws://localhost:3000", + Protocol: "playwright", + }) + if bt.protocol != "playwright" { + t.Errorf("expected protocol 'playwright', got %q", bt.protocol) + } +} + +func TestBrowserTool_Unit_BuildCdpURL(t *testing.T) { + bt := NewBrowserTool(BrowserToolOptions{ + WsURL: "ws://localhost:3000", + Token: "test-token", + Stealth: true, + }) + url := bt.buildCdpURL() + if url == "" { + t.Fatal("expected non-empty URL") + } + if !contains(url, "token=test-token") { + t.Errorf("expected token in URL, got %q", url) + } + if !contains(url, "stealth=true") { + t.Errorf("expected stealth in URL, got %q", url) + } +} + +func TestBrowserTool_Unit_PDFBlockedOnPlaywright(t *testing.T) { + bt := NewBrowserTool(BrowserToolOptions{ + WsURL: "ws://localhost:3000", + Protocol: "playwright", + }) + // Manually set connected to skip ensureConnected + bt.connected = true + bt.page = nil // Will be nil but doPDF checks protocol first + + result := bt.doPDF(map[string]interface{}{}) + if result.ForLLM == "" || !contains(result.ForLLM, "only supported with Chromium") { + t.Errorf("expected PDF blocked message for playwright protocol, got %q", result.ForLLM) + } +} + +func contains(s, substr string) bool { + return len(s) >= len(substr) && (s == substr || len(s) > 0 && containsSubstring(s, substr)) +} + +func containsSubstring(s, substr string) bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} + +// --- Integration tests --- +// These require a running browser instance. +// Set BROWSER_TEST_CDP_URL + BROWSER_TEST_TOKEN for CDP (Browserless) +// Set BROWSER_TEST_PW_URL for Playwright Wire Protocol (Camoufox) + +func TestBrowserTool_Integration_CDP(t *testing.T) { + cdpURL := os.Getenv("BROWSER_TEST_CDP_URL") + token := os.Getenv("BROWSER_TEST_TOKEN") + if cdpURL == "" { + t.Skip("BROWSER_TEST_CDP_URL not set, skipping CDP integration test") + } + + bt := NewBrowserTool(BrowserToolOptions{ + Protocol: "cdp", + WsURL: cdpURL, + Token: token, + Stealth: true, + }) + + ctx := context.Background() + runBrowserIntegrationTests(t, ctx, bt, true) +} + +func TestBrowserTool_Integration_Playwright(t *testing.T) { + pwURL := os.Getenv("BROWSER_TEST_PW_URL") + if pwURL == "" { + t.Skip("BROWSER_TEST_PW_URL not set, skipping Playwright integration test") + } + + bt := NewBrowserTool(BrowserToolOptions{ + Protocol: "playwright", + WsURL: pwURL, + }) + + ctx := context.Background() + runBrowserIntegrationTests(t, ctx, bt, false) +} + +func runBrowserIntegrationTests(t *testing.T, ctx context.Context, bt *BrowserTool, supportsPDF bool) { + t.Helper() + + // Navigate + t.Run("Navigate", func(t *testing.T) { + result := bt.Execute(ctx, map[string]interface{}{ + "action": "navigate", + "url": "https://example.com", + }) + if result.ForLLM == "" { + t.Fatal("expected non-empty result") + } + if contains(result.ForLLM, "Error") { + t.Fatalf("navigate failed: %s", result.ForLLM) + } + t.Log(result.ForLLM) + }) + + // Screenshot + t.Run("Screenshot", func(t *testing.T) { + result := bt.Execute(ctx, map[string]interface{}{ + "action": "screenshot", + }) + if contains(result.ForLLM, "Error") { + t.Fatalf("screenshot failed: %s", result.ForLLM) + } + if result.ForUser == "" { + t.Error("expected screenshot data in ForUser") + } + t.Log(result.ForLLM) + }) + + // GetText + t.Run("GetText", func(t *testing.T) { + result := bt.Execute(ctx, map[string]interface{}{ + "action": "get_text", + "selector": "h1", + }) + if contains(result.ForLLM, "Error") { + t.Fatalf("get_text failed: %s", result.ForLLM) + } + if !contains(result.ForLLM, "Example Domain") { + t.Errorf("expected 'Example Domain' in text, got %q", result.ForLLM) + } + t.Log(result.ForLLM) + }) + + // Evaluate + t.Run("Evaluate", func(t *testing.T) { + result := bt.Execute(ctx, map[string]interface{}{ + "action": "evaluate", + "expression": "document.title", + }) + if contains(result.ForLLM, "Error") { + t.Fatalf("evaluate failed: %s", result.ForLLM) + } + t.Log(result.ForLLM) + }) + + // Scroll + t.Run("Scroll", func(t *testing.T) { + result := bt.Execute(ctx, map[string]interface{}{ + "action": "scroll", + "direction": "down", + "amount": float64(200), + }) + if contains(result.ForLLM, "Error") { + t.Fatalf("scroll failed: %s", result.ForLLM) + } + t.Log(result.ForLLM) + }) + + // Cookies + t.Run("Cookies", func(t *testing.T) { + // Set cookie + result := bt.Execute(ctx, map[string]interface{}{ + "action": "cookies", + "cookie_action": "set", + "cookie_name": "test_cookie", + "cookie_value": "test_value", + "cookie_domain": "example.com", + }) + if contains(result.ForLLM, "Error") { + t.Fatalf("cookie set failed: %s", result.ForLLM) + } + + // Get cookies + result = bt.Execute(ctx, map[string]interface{}{ + "action": "cookies", + "cookie_action": "get", + }) + if contains(result.ForLLM, "Error") { + t.Fatalf("cookie get failed: %s", result.ForLLM) + } + if !contains(result.ForLLM, "test_cookie") { + t.Error("expected test_cookie in cookie list") + } + + // Delete cookie + result = bt.Execute(ctx, map[string]interface{}{ + "action": "cookies", + "cookie_action": "delete", + "cookie_name": "test_cookie", + }) + if contains(result.ForLLM, "Error") { + t.Fatalf("cookie delete failed: %s", result.ForLLM) + } + t.Log("Cookies test passed") + }) + + // PDF (only for CDP) + if supportsPDF { + t.Run("PDF", func(t *testing.T) { + result := bt.Execute(ctx, map[string]interface{}{ + "action": "pdf", + }) + if contains(result.ForLLM, "Error") { + t.Fatalf("pdf failed: %s", result.ForLLM) + } + t.Log(result.ForLLM) + }) + } + + // Close + t.Run("Close", func(t *testing.T) { + result := bt.Execute(ctx, map[string]interface{}{ + "action": "close", + }) + if contains(result.ForLLM, "Error") { + t.Fatalf("close failed: %s", result.ForLLM) + } + t.Log(result.ForLLM) + }) +}