From bf377189027e3002f41fe58857f78b2e1e921779 Mon Sep 17 00:00:00 2001 From: strelov1 Date: Sat, 14 Feb 2026 23:21:58 -0300 Subject: [PATCH] feat: add CDP browser automation tool Add browser tool that connects to any Chrome DevTools Protocol compatible browser (Browserless, headless Chrome, etc.) via chromedp. Supported actions: navigate, click, type, screenshot, get_text, evaluate, wait, scroll, hover, select, pdf, cookies, close. Features: - Lazy connection with persistent session reuse - Auto-reconnect on connection drop - Configurable launch/action timeouts - Stealth mode support - Screenshot and PDF generation with base64 output Co-Authored-By: Claude Opus 4.6 --- config/config.example.json | 8 + go.mod | 7 + go.sum | 15 + pkg/agent/loop.go | 11 + pkg/config/config.go | 18 +- pkg/tools/browser.go | 792 +++++++++++++++++++++++++++++++++++++ pkg/tools/browser_test.go | 258 ++++++++++++ 7 files changed, 1108 insertions(+), 1 deletion(-) create mode 100644 pkg/tools/browser.go create mode 100644 pkg/tools/browser_test.go diff --git a/config/config.example.json b/config/config.example.json index 288e16c58..d2573e72b 100644 --- a/config/config.example.json +++ b/config/config.example.json @@ -107,6 +107,14 @@ "api_key": "YOUR_BRAVE_API_KEY", "max_results": 5 } + }, + "browser": { + "enabled": false, + "cdp_url": "ws://localhost:3000", + "token": "YOUR_CDP_TOKEN", + "stealth": true, + "launch_timeout": 120000, + "action_timeout": 30000 } }, "heartbeat": { diff --git a/go.mod b/go.mod index f4c233ea8..39b8f83b5 100644 --- a/go.mod +++ b/go.mod @@ -24,8 +24,15 @@ require ( github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327 // indirect + github.com/chromedp/chromedp v0.14.2 // indirect + github.com/chromedp/sysutil v1.1.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect + github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect github.com/go-resty/resty/v2 v2.17.1 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.4.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/grbit/go-json v0.11.0 // indirect github.com/klauspost/compress v1.18.4 // indirect diff --git a/go.sum b/go.sum index 9174d2889..0f489772c 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,12 @@ github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5m github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327 h1:UQ4AU+BGti3Sy/aLU8KVseYKNALcX9UXY6DfpwQ6J8E= +github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k= +github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM= +github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo= +github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= +github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM= github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI= @@ -32,6 +38,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs= +github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M= github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q= github.com/go-resty/resty/v2 v2.17.1 h1:x3aMpHK1YM9e4va/TMDRlusDDoZiQ+ViDu/WpA6xTM4= @@ -39,6 +47,12 @@ github.com/go-resty/resty/v2 v2.17.1/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2m github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= +github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -199,6 +213,7 @@ golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 73e837119..548028520 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -81,6 +81,17 @@ func createToolRegistry(workspace string, restrict bool, cfg *config.Config, msg } registry.Register(tools.NewWebFetchTool(50000)) + // Browser tool (CDP) + if cfg.Tools.Browser.Enabled && cfg.Tools.Browser.CdpURL != "" { + registry.Register(tools.NewBrowserTool(tools.BrowserToolOptions{ + CdpURL: cfg.Tools.Browser.CdpURL, + Token: cfg.Tools.Browser.Token, + Stealth: cfg.Tools.Browser.Stealth, + LaunchTimeout: cfg.Tools.Browser.LaunchTimeout, + ActionTimeout: cfg.Tools.Browser.ActionTimeout, + })) + } + // Hardware tools (I2C, SPI) - Linux only, returns error on other platforms registry.Register(tools.NewI2CTool()) registry.Register(tools.NewSPITool()) diff --git a/pkg/config/config.go b/pkg/config/config.go index bbfa2e40b..020aac9e3 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -198,8 +198,18 @@ type WebToolsConfig struct { DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"` } +type BrowserConfig struct { + Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_BROWSER_ENABLED"` + CdpURL string `json:"cdp_url" env:"PICOCLAW_TOOLS_BROWSER_CDP_URL"` + Token string `json:"token" env:"PICOCLAW_TOOLS_BROWSER_TOKEN"` + Stealth bool `json:"stealth" env:"PICOCLAW_TOOLS_BROWSER_STEALTH"` + LaunchTimeout int `json:"launch_timeout" env:"PICOCLAW_TOOLS_BROWSER_LAUNCH_TIMEOUT"` + ActionTimeout int `json:"action_timeout" env:"PICOCLAW_TOOLS_BROWSER_ACTION_TIMEOUT"` +} + type ToolsConfig struct { - Web WebToolsConfig `json:"web"` + Web WebToolsConfig `json:"web"` + Browser BrowserConfig `json:"browser"` } func DefaultConfig() *Config { @@ -301,6 +311,12 @@ func DefaultConfig() *Config { MaxResults: 5, }, }, + Browser: BrowserConfig{ + Enabled: false, + Stealth: true, + LaunchTimeout: 120000, + ActionTimeout: 30000, + }, }, Heartbeat: HeartbeatConfig{ Enabled: true, diff --git a/pkg/tools/browser.go b/pkg/tools/browser.go new file mode 100644 index 000000000..5a96fa705 --- /dev/null +++ b/pkg/tools/browser.go @@ -0,0 +1,792 @@ +package tools + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "net/url" + "strings" + "sync" + "time" + + "github.com/chromedp/cdproto/cdp" + "github.com/chromedp/cdproto/network" + "github.com/chromedp/cdproto/page" + "github.com/chromedp/chromedp" + "github.com/sipeed/picoclaw/pkg/logger" +) + +// BrowserToolOptions configures the browser tool. +type BrowserToolOptions struct { + CdpURL string + Token string + Stealth bool + LaunchTimeout int // ms + ActionTimeout int // ms +} + +// BrowserTool provides browser automation via Chrome DevTools Protocol. +// It connects to a remote browser via CDP WebSocket and supports navigation, +// clicking, typing, screenshots, text extraction, JS evaluation, and waiting. +type BrowserTool struct { + cdpURL string + token string + stealth bool + launchTimeout time.Duration + actionTimeout time.Duration + + allocCtx context.Context + allocCancel context.CancelFunc + browserCtx context.Context + browserCancel context.CancelFunc + connected bool + mu sync.Mutex +} + +// NewBrowserTool creates a new BrowserTool with the given options. +func NewBrowserTool(opts BrowserToolOptions) *BrowserTool { + launchTimeout := 120 * time.Second + if opts.LaunchTimeout > 0 { + launchTimeout = time.Duration(opts.LaunchTimeout) * time.Millisecond + } + + actionTimeout := 30 * time.Second + if opts.ActionTimeout > 0 { + actionTimeout = time.Duration(opts.ActionTimeout) * time.Millisecond + } + + return &BrowserTool{ + cdpURL: opts.CdpURL, + token: opts.Token, + stealth: opts.Stealth, + launchTimeout: launchTimeout, + actionTimeout: actionTimeout, + } +} + +func (t *BrowserTool) Name() string { + return "browser" +} + +func (t *BrowserTool) Description() string { + return "Control a browser via CDP (Chrome DevTools Protocol). " + + "Actions: navigate, click, type, screenshot, get_text, evaluate, wait, scroll, hover, select, pdf, cookies, close. " + + "The browser session persists across calls until explicitly closed." +} + +func (t *BrowserTool) Parameters() map[string]interface{} { + return map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "action": map[string]interface{}{ + "type": "string", + "description": "Action to perform: " + + "navigate (open URL), click (click element), type (type text into input), " + + "screenshot (capture page/element), get_text (extract text), evaluate (run JS), " + + "wait (wait for element), scroll (scroll page/element), hover (mouse over element), " + + "select (choose dropdown option), pdf (save page as PDF), " + + "cookies (get/set/delete cookies), close (end session)", + "enum": []string{ + "navigate", "click", "type", "screenshot", "get_text", "evaluate", + "wait", "scroll", "hover", "select", "pdf", "cookies", "close", + }, + }, + "url": map[string]interface{}{ + "type": "string", + "description": "URL to navigate to (for 'navigate' action)", + }, + "selector": map[string]interface{}{ + "type": "string", + "description": "CSS selector for target element (for click, type, get_text, screenshot, wait, scroll, hover, select)", + }, + "text": map[string]interface{}{ + "type": "string", + "description": "Text to type (for 'type' action)", + }, + "expression": map[string]interface{}{ + "type": "string", + "description": "JavaScript expression to evaluate (for 'evaluate' action)", + }, + "timeout_ms": map[string]interface{}{ + "type": "integer", + "description": "Timeout in milliseconds (for 'wait' action, default: 30000)", + }, + "value": map[string]interface{}{ + "type": "string", + "description": "Value to select (for 'select' action — option value attribute)", + }, + "direction": map[string]interface{}{ + "type": "string", + "description": "Scroll direction: 'up' or 'down' (for 'scroll' action, default: 'down')", + "enum": []string{"up", "down"}, + }, + "distance": map[string]interface{}{ + "type": "integer", + "description": "Scroll distance in pixels (for 'scroll' action, default: 500)", + }, + "wait_for_navigation": map[string]interface{}{ + "type": "boolean", + "description": "Wait for page navigation after click (for 'click' action, default: false)", + }, + "cookie_action": map[string]interface{}{ + "type": "string", + "description": "Cookie sub-action: 'get' (list all), 'set' (add cookie), 'delete' (remove cookie), 'clear' (remove all)", + "enum": []string{"get", "set", "delete", "clear"}, + }, + "cookie_name": map[string]interface{}{ + "type": "string", + "description": "Cookie name (for cookies set/delete)", + }, + "cookie_value": map[string]interface{}{ + "type": "string", + "description": "Cookie value (for cookies set)", + }, + "cookie_domain": map[string]interface{}{ + "type": "string", + "description": "Cookie domain (for cookies set/delete)", + }, + }, + "required": []string{"action"}, + } +} + +func (t *BrowserTool) Execute(ctx context.Context, args map[string]interface{}) *ToolResult { + action, ok := args["action"].(string) + if !ok { + return ErrorResult("action is required") + } + + if action == "close" { + return t.doClose() + } + + if err := t.ensureConnected(); err != nil { + return ErrorResult(fmt.Sprintf("failed to connect to browser: %v", err)) + } + + switch action { + case "navigate": + return t.doNavigate(ctx, args) + case "click": + return t.doClick(ctx, args) + case "type": + return t.doType(ctx, args) + case "screenshot": + return t.doScreenshot(ctx, args) + case "get_text": + return t.doGetText(ctx, args) + case "evaluate": + return t.doEvaluate(ctx, args) + case "wait": + return t.doWait(ctx, args) + case "scroll": + return t.doScroll(ctx, args) + case "hover": + return t.doHover(ctx, args) + case "select": + return t.doSelect(ctx, args) + case "pdf": + return t.doPDF(ctx, args) + case "cookies": + return t.doCookies(ctx, args) + default: + return ErrorResult(fmt.Sprintf("unknown action: %s", action)) + } +} + +// buildCdpURL constructs the WebSocket URL with token, stealth, and launch timeout params. +// The cdp_url config should point to the CDP WebSocket endpoint as-is. +func (t *BrowserTool) buildCdpURL() string { + u, err := url.Parse(t.cdpURL) + if err != nil { + return t.cdpURL + } + + q := u.Query() + if t.token != "" { + q.Set("token", t.token) + } + if t.stealth { + q.Set("stealth", "true") + } + q.Set("launch", fmt.Sprintf(`{"timeout":%d}`, int(t.launchTimeout.Milliseconds()))) + u.RawQuery = q.Encode() + + return u.String() +} + +// ensureConnected lazily connects to the remote browser, reusing the connection. +func (t *BrowserTool) ensureConnected() error { + t.mu.Lock() + defer t.mu.Unlock() + + if t.connected && t.browserCtx != nil { + if t.browserCtx.Err() == nil { + return nil + } + logger.InfoCF("browser", "Browser context expired, reconnecting", nil) + t.cleanupLocked() + } + + wsURL := t.buildCdpURL() + logger.InfoCF("browser", "Connecting to remote browser", + map[string]interface{}{ + "url": t.cdpURL, + }) + + // NoModifyURL prevents chromedp from fetching /json/version which returns + // internal Chrome URLs inaccessible outside the container. + allocCtx, allocCancel := chromedp.NewRemoteAllocator(context.Background(), wsURL, chromedp.NoModifyURL) + + browserCtx, browserCancel := chromedp.NewContext(allocCtx) + + // Establish the connection with a timeout. + // IMPORTANT: don't use context.WithTimeout(browserCtx, ...) because chromedp's + // RemoteAllocator closes the WebSocket when the allocate context is done. + // Instead, run in a goroutine with a channel-based timeout. + done := make(chan error, 1) + go func() { + done <- chromedp.Run(browserCtx) + }() + + select { + case err := <-done: + if err != nil { + browserCancel() + allocCancel() + return fmt.Errorf("browser connection failed: %w", err) + } + case <-time.After(t.launchTimeout): + browserCancel() + allocCancel() + return fmt.Errorf("browser connection timed out after %s", t.launchTimeout) + } + + t.allocCtx = allocCtx + t.allocCancel = allocCancel + t.browserCtx = browserCtx + t.browserCancel = browserCancel + t.connected = true + logger.InfoCF("browser", "Connected to remote browser", nil) + return nil +} + +// cleanupLocked releases browser resources. Must be called with mu held. +func (t *BrowserTool) cleanupLocked() { + if t.browserCancel != nil { + t.browserCancel() + t.browserCancel = nil + } + if t.allocCancel != nil { + t.allocCancel() + t.allocCancel = nil + } + t.browserCtx = nil + t.allocCtx = nil + t.connected = false +} + +// runAction executes chromedp actions with the configured action timeout. +func (t *BrowserTool) runAction(ctx context.Context, actions ...chromedp.Action) error { + t.mu.Lock() + browserCtx := t.browserCtx + t.mu.Unlock() + + if browserCtx == nil { + return fmt.Errorf("browser not connected") + } + + actionCtx, cancel := context.WithTimeout(browserCtx, t.actionTimeout) + defer cancel() + + return chromedp.Run(actionCtx, actions...) +} + +// getBrowserCtx returns the current browser context safely. +func (t *BrowserTool) getBrowserCtx() context.Context { + t.mu.Lock() + defer t.mu.Unlock() + return t.browserCtx +} + +func (t *BrowserTool) doNavigate(ctx context.Context, args map[string]interface{}) *ToolResult { + navURL, ok := args["url"].(string) + if !ok || navURL == "" { + return ErrorResult("url is required for navigate action") + } + + var title string + var location string + + err := t.runAction(ctx, + chromedp.Navigate(navURL), + chromedp.WaitReady("body"), + chromedp.Title(&title), + chromedp.Location(&location), + ) + if err != nil { + return ErrorResult(fmt.Sprintf("navigate failed: %v", err)) + } + + result := fmt.Sprintf("Navigated to: %s\nTitle: %s\nURL: %s", navURL, title, location) + return SilentResult(result) +} + +func (t *BrowserTool) doClick(ctx context.Context, args map[string]interface{}) *ToolResult { + selector, ok := args["selector"].(string) + if !ok || selector == "" { + return ErrorResult("selector is required for click action") + } + + waitNav, _ := args["wait_for_navigation"].(bool) + + if waitNav { + // Click and wait for navigation to complete + browserCtx := t.getBrowserCtx() + if browserCtx == nil { + return ErrorResult("browser not connected") + } + + actionCtx, cancel := context.WithTimeout(browserCtx, t.actionTimeout) + defer cancel() + + err := chromedp.Run(actionCtx, + chromedp.WaitVisible(selector), + chromedp.Click(selector), + chromedp.WaitReady("body"), + ) + if err != nil { + return ErrorResult(fmt.Sprintf("click failed on '%s': %v", selector, err)) + } + + var title, location string + _ = chromedp.Run(actionCtx, + chromedp.Title(&title), + chromedp.Location(&location), + ) + return SilentResult(fmt.Sprintf("Clicked element: %s\nNavigated to: %s\nTitle: %s", selector, location, title)) + } + + err := t.runAction(ctx, + chromedp.WaitVisible(selector), + chromedp.Click(selector), + ) + if err != nil { + return ErrorResult(fmt.Sprintf("click failed on '%s': %v", selector, err)) + } + + return SilentResult(fmt.Sprintf("Clicked element: %s", selector)) +} + +func (t *BrowserTool) doType(ctx context.Context, args map[string]interface{}) *ToolResult { + selector, ok := args["selector"].(string) + if !ok || selector == "" { + return ErrorResult("selector is required for type action") + } + + text, ok := args["text"].(string) + if !ok { + return ErrorResult("text is required for type action") + } + + err := t.runAction(ctx, + chromedp.WaitVisible(selector), + chromedp.Clear(selector), + chromedp.SendKeys(selector, text), + ) + if err != nil { + return ErrorResult(fmt.Sprintf("type failed on '%s': %v", selector, err)) + } + + return SilentResult(fmt.Sprintf("Typed %d chars into: %s", len(text), selector)) +} + +func (t *BrowserTool) doScreenshot(ctx context.Context, args map[string]interface{}) *ToolResult { + var buf []byte + var err error + + selector, _ := args["selector"].(string) + + if selector != "" { + err = t.runAction(ctx, + chromedp.WaitVisible(selector), + chromedp.Screenshot(selector, &buf, chromedp.NodeVisible), + ) + } else { + err = t.runAction(ctx, + chromedp.FullScreenshot(&buf, 90), + ) + } + + if err != nil { + return ErrorResult(fmt.Sprintf("screenshot failed: %v", err)) + } + + encoded := base64.StdEncoding.EncodeToString(buf) + + llmContent := fmt.Sprintf("Screenshot taken (%d bytes PNG, %d chars base64)", len(buf), len(encoded)) + if len(encoded) > 500 { + llmContent += fmt.Sprintf("\nBase64 preview: %s...", encoded[:500]) + } else { + llmContent += fmt.Sprintf("\nBase64: %s", encoded) + } + + return &ToolResult{ + ForLLM: llmContent, + ForUser: fmt.Sprintf("data:image/png;base64,%s", encoded), + Silent: false, + } +} + +func (t *BrowserTool) doGetText(ctx context.Context, args map[string]interface{}) *ToolResult { + selector, _ := args["selector"].(string) + if selector == "" { + selector = "body" + } + + var text string + err := t.runAction(ctx, + chromedp.WaitReady(selector), + chromedp.Text(selector, &text), + ) + + if err != nil { + return ErrorResult(fmt.Sprintf("get_text failed: %v", err)) + } + + text = strings.TrimSpace(text) + + const maxTextLen = 10000 + truncated := false + if len(text) > maxTextLen { + text = text[:maxTextLen] + truncated = true + } + + if truncated { + text += "\n\n[... truncated at 10000 chars]" + } + + return SilentResult(text) +} + +func (t *BrowserTool) doEvaluate(ctx context.Context, args map[string]interface{}) *ToolResult { + expression, ok := args["expression"].(string) + if !ok || expression == "" { + return ErrorResult("expression is required for evaluate action") + } + + var result interface{} + err := t.runAction(ctx, + chromedp.Evaluate(expression, &result), + ) + if err != nil { + return ErrorResult(fmt.Sprintf("evaluate failed: %v", err)) + } + + resultStr := fmt.Sprintf("%v", result) + + const maxResultLen = 10000 + if len(resultStr) > maxResultLen { + resultStr = resultStr[:maxResultLen] + "\n[... truncated]" + } + + return SilentResult(fmt.Sprintf("JS eval result: %s", resultStr)) +} + +func (t *BrowserTool) doWait(ctx context.Context, args map[string]interface{}) *ToolResult { + selector, ok := args["selector"].(string) + if !ok || selector == "" { + return ErrorResult("selector is required for wait action") + } + + timeout := t.actionTimeout + if ms, ok := args["timeout_ms"].(float64); ok && ms > 0 { + timeout = time.Duration(ms) * time.Millisecond + } + + browserCtx := t.getBrowserCtx() + if browserCtx == nil { + return ErrorResult("browser not connected") + } + + waitCtx, cancel := context.WithTimeout(browserCtx, timeout) + defer cancel() + + err := chromedp.Run(waitCtx, + chromedp.WaitVisible(selector), + ) + if err != nil { + return ErrorResult(fmt.Sprintf("wait timed out for '%s': %v", selector, err)) + } + + return SilentResult(fmt.Sprintf("Element visible: %s", selector)) +} + +func (t *BrowserTool) doScroll(ctx context.Context, args map[string]interface{}) *ToolResult { + direction := "down" + if d, ok := args["direction"].(string); ok && d != "" { + direction = d + } + + distance := 500 + if d, ok := args["distance"].(float64); ok && d > 0 { + distance = int(d) + } + + if direction == "up" { + distance = -distance + } + + selector, _ := args["selector"].(string) + + var js string + if selector != "" { + js = fmt.Sprintf(`document.querySelector(%q).scrollBy(0, %d)`, selector, distance) + } else { + js = fmt.Sprintf(`window.scrollBy(0, %d)`, distance) + } + + err := t.runAction(ctx, chromedp.Evaluate(js, nil)) + if err != nil { + return ErrorResult(fmt.Sprintf("scroll failed: %v", err)) + } + + target := "page" + if selector != "" { + target = selector + } + return SilentResult(fmt.Sprintf("Scrolled %s %s by %dpx", target, direction, abs(distance))) +} + +func (t *BrowserTool) doHover(ctx context.Context, args map[string]interface{}) *ToolResult { + selector, ok := args["selector"].(string) + if !ok || selector == "" { + return ErrorResult("selector is required for hover action") + } + + err := t.runAction(ctx, + chromedp.WaitVisible(selector), + chromedp.MouseClickXY(0, 0, chromedp.ButtonNone), // reset position + chromedp.ActionFunc(func(ctx context.Context) error { + // Get element position and hover over it + var nodes []*cdp.Node + if err := chromedp.Nodes(selector, &nodes, chromedp.AtLeast(1)).Do(ctx); err != nil { + return err + } + return chromedp.MouseClickNode(nodes[0], chromedp.ButtonNone).Do(ctx) + }), + ) + if err != nil { + // Fallback: use JS-based hover via dispatchEvent + jsHover := fmt.Sprintf(` + var el = document.querySelector(%q); + if (el) { + el.dispatchEvent(new MouseEvent('mouseover', {bubbles: true})); + el.dispatchEvent(new MouseEvent('mouseenter', {bubbles: true})); + 'hovered' + } else { + 'element not found' + } + `, selector) + var result string + err2 := t.runAction(ctx, chromedp.Evaluate(jsHover, &result)) + if err2 != nil { + return ErrorResult(fmt.Sprintf("hover failed on '%s': %v", selector, err)) + } + if result == "element not found" { + return ErrorResult(fmt.Sprintf("hover failed: element '%s' not found", selector)) + } + } + + return SilentResult(fmt.Sprintf("Hovered over: %s", selector)) +} + +func (t *BrowserTool) doSelect(ctx context.Context, args map[string]interface{}) *ToolResult { + selector, ok := args["selector"].(string) + if !ok || selector == "" { + return ErrorResult("selector is required for select action") + } + + value, ok := args["value"].(string) + if !ok || value == "" { + return ErrorResult("value is required for select action") + } + + err := t.runAction(ctx, + chromedp.WaitVisible(selector), + chromedp.SetValue(selector, value), + // Trigger change event so JS frameworks pick up the selection + chromedp.Evaluate(fmt.Sprintf( + `var el = document.querySelector(%q); if(el) el.dispatchEvent(new Event('change', {bubbles:true}))`, + selector, + ), nil), + ) + if err != nil { + return ErrorResult(fmt.Sprintf("select failed on '%s': %v", selector, err)) + } + + return SilentResult(fmt.Sprintf("Selected value '%s' in: %s", value, selector)) +} + +func (t *BrowserTool) doPDF(ctx context.Context, args map[string]interface{}) *ToolResult { + browserCtx := t.getBrowserCtx() + if browserCtx == nil { + return ErrorResult("browser not connected") + } + + actionCtx, cancel := context.WithTimeout(browserCtx, t.actionTimeout) + defer cancel() + + var buf []byte + err := chromedp.Run(actionCtx, chromedp.ActionFunc(func(ctx context.Context) error { + var err error + buf, _, err = page.PrintToPDF(). + WithPrintBackground(true). + WithPreferCSSPageSize(true). + Do(ctx) + return err + })) + if err != nil { + return ErrorResult(fmt.Sprintf("pdf failed: %v", err)) + } + + encoded := base64.StdEncoding.EncodeToString(buf) + + llmContent := fmt.Sprintf("PDF generated (%d bytes, %d chars base64)", len(buf), len(encoded)) + + return &ToolResult{ + ForLLM: llmContent, + ForUser: fmt.Sprintf("data:application/pdf;base64,%s", encoded), + Silent: false, + } +} + +func (t *BrowserTool) doCookies(ctx context.Context, args map[string]interface{}) *ToolResult { + cookieAction, _ := args["cookie_action"].(string) + if cookieAction == "" { + cookieAction = "get" + } + + browserCtx := t.getBrowserCtx() + if browserCtx == nil { + return ErrorResult("browser not connected") + } + + actionCtx, cancel := context.WithTimeout(browserCtx, t.actionTimeout) + defer cancel() + + switch cookieAction { + case "get": + var cookies []*network.Cookie + err := chromedp.Run(actionCtx, chromedp.ActionFunc(func(ctx context.Context) error { + var err error + cookies, err = network.GetCookies().Do(ctx) + return err + })) + if err != nil { + return ErrorResult(fmt.Sprintf("get cookies failed: %v", err)) + } + + type cookieSummary struct { + Name string `json:"name"` + Value string `json:"value"` + Domain string `json:"domain"` + Path string `json:"path"` + Secure bool `json:"secure"` + HTTPOnly bool `json:"httpOnly"` + } + + summaries := make([]cookieSummary, 0, len(cookies)) + for _, c := range cookies { + summaries = append(summaries, cookieSummary{ + Name: c.Name, + Value: c.Value, + Domain: c.Domain, + Path: c.Path, + Secure: c.Secure, + HTTPOnly: c.HTTPOnly, + }) + } + + data, _ := json.MarshalIndent(summaries, "", " ") + return SilentResult(fmt.Sprintf("Cookies (%d):\n%s", len(cookies), string(data))) + + case "set": + name, _ := args["cookie_name"].(string) + value, _ := args["cookie_value"].(string) + domain, _ := args["cookie_domain"].(string) + + if name == "" { + return ErrorResult("cookie_name is required for cookies set") + } + + err := chromedp.Run(actionCtx, chromedp.ActionFunc(func(ctx context.Context) error { + cp := &network.CookieParam{ + Name: name, + Value: value, + } + if domain != "" { + cp.Domain = domain + } + return network.SetCookies([]*network.CookieParam{cp}).Do(ctx) + })) + if err != nil { + return ErrorResult(fmt.Sprintf("set cookie failed: %v", err)) + } + return SilentResult(fmt.Sprintf("Cookie set: %s=%s", name, value)) + + case "delete": + name, _ := args["cookie_name"].(string) + domain, _ := args["cookie_domain"].(string) + + if name == "" { + return ErrorResult("cookie_name is required for cookies delete") + } + + err := chromedp.Run(actionCtx, chromedp.ActionFunc(func(ctx context.Context) error { + dp := &network.DeleteCookiesParams{Name: name} + if domain != "" { + dp.Domain = domain + } + return dp.Do(ctx) + })) + if err != nil { + return ErrorResult(fmt.Sprintf("delete cookie failed: %v", err)) + } + return SilentResult(fmt.Sprintf("Cookie deleted: %s", name)) + + case "clear": + err := chromedp.Run(actionCtx, chromedp.ActionFunc(func(ctx context.Context) error { + return network.ClearBrowserCookies().Do(ctx) + })) + if err != nil { + return ErrorResult(fmt.Sprintf("clear cookies failed: %v", err)) + } + return SilentResult("All cookies cleared") + + default: + return ErrorResult(fmt.Sprintf("unknown cookie_action: %s (use get/set/delete/clear)", cookieAction)) + } +} + +func (t *BrowserTool) doClose() *ToolResult { + t.mu.Lock() + defer t.mu.Unlock() + + if !t.connected { + return SilentResult("Browser already closed") + } + + t.cleanupLocked() + logger.InfoCF("browser", "Browser session closed", nil) + return SilentResult("Browser session closed") +} + +func abs(x int) int { + if x < 0 { + return -x + } + return x +} diff --git a/pkg/tools/browser_test.go b/pkg/tools/browser_test.go new file mode 100644 index 000000000..ccd67cc7f --- /dev/null +++ b/pkg/tools/browser_test.go @@ -0,0 +1,258 @@ +package tools + +import ( + "context" + "os" + "testing" +) + +// TestBrowserTool_Unit tests browser tool without a real browser connection. +func TestBrowserTool_Unit(t *testing.T) { + tool := NewBrowserTool(BrowserToolOptions{ + CdpURL: "ws://localhost:19999", + Token: "test", + Stealth: true, + LaunchTimeout: 5000, + ActionTimeout: 5000, + }) + + t.Run("Name", func(t *testing.T) { + if tool.Name() != "browser" { + t.Errorf("expected 'browser', got %q", tool.Name()) + } + }) + + t.Run("Parameters", func(t *testing.T) { + params := tool.Parameters() + props, ok := params["properties"].(map[string]interface{}) + if !ok { + t.Fatal("expected properties map") + } + for _, key := range []string{"action", "url", "selector", "text", "expression", "timeout_ms"} { + if _, ok := props[key]; !ok { + t.Errorf("missing parameter: %s", key) + } + } + required, ok := params["required"].([]string) + if !ok || len(required) != 1 || required[0] != "action" { + t.Errorf("expected required=[action], got %v", required) + } + }) + + t.Run("MissingAction", func(t *testing.T) { + result := tool.Execute(context.Background(), map[string]interface{}{}) + if !result.IsError { + t.Error("expected error for missing action") + } + }) + + t.Run("UnknownAction", func(t *testing.T) { + result := tool.Execute(context.Background(), map[string]interface{}{"action": "fly"}) + if !result.IsError { + t.Error("expected error for unknown action") + } + }) + + t.Run("CloseWithoutConnect", func(t *testing.T) { + result := tool.Execute(context.Background(), map[string]interface{}{"action": "close"}) + if result.IsError { + t.Error("close on unconnected browser should not error") + } + }) + + t.Run("NavigateMissingURL", func(t *testing.T) { + // Will try to connect and fail (no server), but let's check arg validation + // The tool tries to connect first, so it will fail at connection + result := tool.Execute(context.Background(), map[string]interface{}{"action": "navigate"}) + if !result.IsError { + t.Error("expected error") + } + }) + + t.Run("BuildCdpURL", func(t *testing.T) { + url := tool.buildCdpURL() + if url == "" { + t.Error("expected non-empty CDP URL") + } + // Check token is in URL + if !contains(url, "token=test") { + t.Errorf("expected token in URL, got: %s", url) + } + if !contains(url, "stealth=true") { + t.Errorf("expected stealth in URL, got: %s", url) + } + if !contains(url, "launch=") { + t.Errorf("expected launch timeout in URL, got: %s", url) + } + }) +} + +func contains(s, substr string) bool { + return len(s) >= len(substr) && searchString(s, substr) +} + +func searchString(s, substr string) bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} + +// TestBrowserTool_Integration runs against a real Browserless instance. +// Set BROWSER_TEST_CDP_URL and BROWSER_TEST_TOKEN to enable. +func TestBrowserTool_Integration(t *testing.T) { + cdpURL := os.Getenv("BROWSER_TEST_CDP_URL") + token := os.Getenv("BROWSER_TEST_TOKEN") + if cdpURL == "" { + t.Skip("BROWSER_TEST_CDP_URL not set, skipping integration test") + } + + tool := NewBrowserTool(BrowserToolOptions{ + CdpURL: cdpURL, + Token: token, + Stealth: true, + LaunchTimeout: 120000, + ActionTimeout: 30000, + }) + + ctx := context.Background() + + t.Run("NavigateAndGetText", func(t *testing.T) { + // Navigate + result := tool.Execute(ctx, map[string]interface{}{ + "action": "navigate", + "url": "https://example.com", + }) + if result.IsError { + t.Fatalf("navigate failed: %s", result.ForLLM) + } + t.Logf("Navigate result: %s", result.ForLLM) + + // Get text + result = tool.Execute(ctx, map[string]interface{}{ + "action": "get_text", + }) + if result.IsError { + t.Fatalf("get_text failed: %s", result.ForLLM) + } + t.Logf("Page text (first 200 chars): %.200s", result.ForLLM) + + if !contains(result.ForLLM, "Example Domain") { + t.Error("expected 'Example Domain' in page text") + } + }) + + t.Run("Screenshot", func(t *testing.T) { + result := tool.Execute(ctx, map[string]interface{}{ + "action": "screenshot", + }) + if result.IsError { + t.Fatalf("screenshot failed: %s", result.ForLLM) + } + t.Logf("Screenshot result (LLM): %.100s", result.ForLLM) + if result.ForUser == "" { + t.Error("expected ForUser with base64 data") + } + }) + + t.Run("Evaluate", func(t *testing.T) { + result := tool.Execute(ctx, map[string]interface{}{ + "action": "evaluate", + "expression": "document.title", + }) + if result.IsError { + t.Fatalf("evaluate failed: %s", result.ForLLM) + } + t.Logf("Evaluate result: %s", result.ForLLM) + if !contains(result.ForLLM, "Example Domain") { + t.Error("expected 'Example Domain' in eval result") + } + }) + + t.Run("Scroll", func(t *testing.T) { + result := tool.Execute(ctx, map[string]interface{}{ + "action": "scroll", + "direction": "down", + "distance": float64(300), + }) + if result.IsError { + t.Fatalf("scroll failed: %s", result.ForLLM) + } + t.Logf("Scroll result: %s", result.ForLLM) + }) + + t.Run("Hover", func(t *testing.T) { + result := tool.Execute(ctx, map[string]interface{}{ + "action": "hover", + "selector": "a", + }) + if result.IsError { + t.Fatalf("hover failed: %s", result.ForLLM) + } + t.Logf("Hover result: %s", result.ForLLM) + }) + + t.Run("Cookies", func(t *testing.T) { + // Set cookie + result := tool.Execute(ctx, map[string]interface{}{ + "action": "cookies", + "cookie_action": "set", + "cookie_name": "test_cookie", + "cookie_value": "hello123", + "cookie_domain": "example.com", + }) + if result.IsError { + t.Fatalf("set cookie failed: %s", result.ForLLM) + } + t.Logf("Set cookie: %s", result.ForLLM) + + // Get cookies + result = tool.Execute(ctx, map[string]interface{}{ + "action": "cookies", + "cookie_action": "get", + }) + if result.IsError { + t.Fatalf("get cookies failed: %s", result.ForLLM) + } + t.Logf("Cookies: %s", result.ForLLM) + if !contains(result.ForLLM, "test_cookie") { + t.Error("expected test_cookie in cookies list") + } + + // Delete cookie + result = tool.Execute(ctx, map[string]interface{}{ + "action": "cookies", + "cookie_action": "delete", + "cookie_name": "test_cookie", + "cookie_domain": "example.com", + }) + if result.IsError { + t.Fatalf("delete cookie failed: %s", result.ForLLM) + } + t.Logf("Delete cookie: %s", result.ForLLM) + }) + + t.Run("PDF", func(t *testing.T) { + result := tool.Execute(ctx, map[string]interface{}{ + "action": "pdf", + }) + if result.IsError { + t.Fatalf("pdf failed: %s", result.ForLLM) + } + t.Logf("PDF result: %s", result.ForLLM) + if result.ForUser == "" || !contains(result.ForUser, "data:application/pdf") { + t.Error("expected PDF data URI in ForUser") + } + }) + + t.Run("Close", func(t *testing.T) { + result := tool.Execute(ctx, map[string]interface{}{ + "action": "close", + }) + if result.IsError { + t.Fatalf("close failed: %s", result.ForLLM) + } + }) +}