From 9d2a5bcab2ab0c300abbf117f0d799f07a306e70 Mon Sep 17 00:00:00 2001 From: zhenghuizli Date: Tue, 7 Apr 2026 20:50:37 +0800 Subject: [PATCH] feat(tool): add browser automation via Chrome DevTools Protocol (CDP) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement a BrowserTool that provides browser automation capabilities through direct CDP WebSocket connection to Chrome/Chromium. This addresses Issue #293 (Browser Automation) on the project roadmap. Key features: - Go-native CDP client using gorilla/websocket (zero new dependencies) - 12 actions: navigate, state, click, type, fill, select, screenshot, get_text, scroll, keys, evaluate, close - [N]-indexed DOM state extraction (borrowed from opencli) for LLM-friendly structured output at zero vision token cost - 13 stealth anti-detection patches (navigator.webdriver, chrome stub, plugin spoofing, CDP stack trace cleanup, etc.) - Cross-platform Chrome auto-detection (macOS/Linux/Windows + CHROME_PATH) - Screenshot delivery via MediaStore + channel system to users - Browsing history tracking with compact summary in state output - SSRF protection using net.ParseIP for full CIDR coverage (private, loopback, link-local, metadata endpoints) - Concurrent-safe lazy CDP connection with sync.Mutex - Optional via //go:build cdp tag — zero impact on default builds Build: - Default: go build — no browser code included (stub returns error) - Full: go build -tags cdp — complete browser support enabled New files: - pkg/tools/browser.go — BrowserTool core (Tool interface) - pkg/tools/browser_cdp.go — CDP WebSocket client - pkg/tools/browser_cdp_util.go — Chrome path detection - pkg/tools/browser_stealth.go — Anti-detection JS generator - pkg/tools/browser_stub.go — Stub for non-CDP builds - pkg/tools/browser_test.go — Unit tests (8 tests) - pkg/tools/browser_integration_test.go — Integration tests (9 tests) Modified files: - pkg/config/config.go — BrowserToolConfig + IsToolEnabled("browser") - pkg/agent/instance.go — Browser tool registration Ref: #293 --- pkg/agent/instance.go | 10 + pkg/config/config.go | 13 + pkg/tools/browser.go | 889 ++++++++++++++++++++++++++ pkg/tools/browser_cdp.go | 530 +++++++++++++++ pkg/tools/browser_cdp_util.go | 98 +++ pkg/tools/browser_integration_test.go | 250 ++++++++ pkg/tools/browser_stealth.go | 234 +++++++ pkg/tools/browser_stub.go | 42 ++ pkg/tools/browser_test.go | 202 ++++++ 9 files changed, 2268 insertions(+) create mode 100644 pkg/tools/browser.go create mode 100644 pkg/tools/browser_cdp.go create mode 100644 pkg/tools/browser_cdp_util.go create mode 100644 pkg/tools/browser_integration_test.go create mode 100644 pkg/tools/browser_stealth.go create mode 100644 pkg/tools/browser_stub.go create mode 100644 pkg/tools/browser_test.go diff --git a/pkg/agent/instance.go b/pkg/agent/instance.go index bacfa49c5..6a77ade5b 100644 --- a/pkg/agent/instance.go +++ b/pkg/agent/instance.go @@ -100,6 +100,16 @@ func NewAgentInstance( } } + if cfg.Tools.IsToolEnabled("browser") { + browserTool, err := tools.NewBrowserTool(cfg.Tools.Browser) + if err != nil { + logger.WarnCF("agent", "Browser tool unavailable (optional)", + map[string]any{"error": err.Error()}) + } else { + toolsRegistry.Register(browserTool) + } + } + if cfg.Tools.IsToolEnabled("edit_file") { toolsRegistry.Register(tools.NewEditFileTool(workspace, restrict, allowWritePaths)) } diff --git a/pkg/config/config.go b/pkg/config/config.go index 1d98aa334..571e6a21b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -875,6 +875,7 @@ type ToolsConfig struct { Subagent ToolConfig `json:"subagent" yaml:"-" envPrefix:"PICOCLAW_TOOLS_SUBAGENT_"` WebFetch ToolConfig `json:"web_fetch" yaml:"-" envPrefix:"PICOCLAW_TOOLS_WEB_FETCH_"` WriteFile ToolConfig `json:"write_file" yaml:"-" envPrefix:"PICOCLAW_TOOLS_WRITE_FILE_"` + Browser BrowserToolConfig `json:"browser" yaml:"browser,omitempty"` } // IsFilterSensitiveDataEnabled returns true if sensitive data filtering is enabled @@ -895,6 +896,16 @@ type SearchCacheConfig struct { TTLSeconds int `json:"ttl_seconds" env:"PICOCLAW_SKILLS_SEARCH_CACHE_TTL_SECONDS"` } +// BrowserToolConfig holds configuration for the browser automation tool. +// The browser tool provides CDP-based browser control and requires Chrome/Chromium. +type BrowserToolConfig struct { + ToolConfig `envPrefix:"PICOCLAW_TOOLS_BROWSER_"` + CDPEndpoint string `json:"cdp_endpoint" yaml:"cdp_endpoint,omitempty" env:"PICOCLAW_TOOLS_BROWSER_CDP_ENDPOINT"` + Timeout int `json:"timeout_seconds" yaml:"timeout_seconds,omitempty" env:"PICOCLAW_TOOLS_BROWSER_TIMEOUT"` + Stealth bool `json:"stealth" yaml:"stealth,omitempty" env:"PICOCLAW_TOOLS_BROWSER_STEALTH"` + AllowEval bool `json:"allow_evaluate" yaml:"allow_evaluate,omitempty" env:"PICOCLAW_TOOLS_BROWSER_ALLOW_EVALUATE"` +} + type SkillsRegistriesConfig struct { ClawHub ClawHubRegistryConfig `json:"clawhub" yaml:"clawhub,omitempty"` } @@ -1364,6 +1375,8 @@ func (t *ToolsConfig) IsToolEnabled(name string) bool { return t.WriteFile.Enabled case "mcp": return t.MCP.Enabled + case "browser": + return t.Browser.Enabled default: return true } diff --git a/pkg/tools/browser.go b/pkg/tools/browser.go new file mode 100644 index 000000000..4703bd804 --- /dev/null +++ b/pkg/tools/browser.go @@ -0,0 +1,889 @@ +//go:build cdp + +package tools + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "net" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/media" +) + +// pageVisit records a single page navigation for browsing history. +type pageVisit struct { + URL string + Title string + Timestamp time.Time +} + +const maxHistorySize = 50 // keep last 50 pages to bound memory + +// BrowserTool provides browser automation capabilities via Chrome DevTools Protocol. +// It enables AI agents to navigate web pages, interact with elements, and extract data. +type BrowserTool struct { + cfg config.BrowserToolConfig + cdp *CDPClient + cdpMu sync.Mutex // guards lazy cdp connection + chromePath string + stealthJS string + mediaStore media.MediaStore + history []pageVisit // browsing history, most recent last +} + +// NewBrowserTool creates a new BrowserTool. It verifies that Chrome is available +// and attempts to connect to the CDP endpoint. Returns an error if Chrome is +// not found or the connection fails. +func NewBrowserTool(cfg config.BrowserToolConfig) (*BrowserTool, error) { + chromePath, err := FindChromePath() + if err != nil { + return nil, fmt.Errorf( + "Chrome/Chromium required for browser tool but not found. "+ + "Install Chrome and restart, or set CHROME_PATH env var. "+ + "Error: %w", err) + } + + logger.InfoCF("tool", "Chrome found for browser tool", + map[string]any{"path": chromePath}) + + var stealthJS string + if cfg.Stealth { + stealthJS = generateStealthJS() + } + + return &BrowserTool{ + cfg: cfg, + chromePath: chromePath, + stealthJS: stealthJS, + }, nil +} + +// connectIfNeeded lazily connects to the CDP endpoint. +// It is safe for concurrent use. +func (t *BrowserTool) connectIfNeeded() error { + t.cdpMu.Lock() + defer t.cdpMu.Unlock() + + if t.cdp != nil { + return nil + } + + endpoint := t.cfg.CDPEndpoint + if endpoint == "" { + endpoint = "http://127.0.0.1:9222" + } + + cdp, err := NewCDPClient(endpoint) + if err != nil { + return fmt.Errorf( + "failed to connect to Chrome CDP at %s. "+ + "Make sure Chrome is running with: %s --remote-debugging-port=9222. "+ + "Error: %w", endpoint, t.chromePath, err) + } + + // Enable required CDP domains + if err := cdp.EnablePage(); err != nil { + cdp.Close() + return fmt.Errorf("failed to enable Page domain: %w", err) + } + if err := cdp.EnableRuntime(); err != nil { + cdp.Close() + return fmt.Errorf("failed to enable Runtime domain: %w", err) + } + if err := cdp.EnableDOM(); err != nil { + cdp.Close() + return fmt.Errorf("failed to enable DOM domain: %w", err) + } + + // Inject stealth JS if configured + if t.stealthJS != "" { + if err := cdp.InjectScript(t.stealthJS); err != nil { + logger.WarnCF("tool", "Failed to inject stealth JS", + map[string]any{"error": err.Error()}) + } + } + + t.cdp = cdp + return nil +} + +func (t *BrowserTool) Name() string { return "browser" } + +func (t *BrowserTool) Description() string { + return `Browser automation via CDP. Actions: navigate, state, click, type, fill, select, screenshot, get_text, scroll, keys, evaluate, close. +Workflow: navigate url → state (get [N] indices) → click/type [N] → state (verify). Always run state after navigate or click.` +} + +func (t *BrowserTool) Parameters() map[string]any { + return map[string]any{ + "type": "object", + "properties": map[string]any{ + "action": map[string]any{ + "type": "string", + "description": "Browser action to perform", + "enum": []string{"navigate", "state", "click", "type", "fill", "select", "screenshot", "get_text", "scroll", "keys", "evaluate", "close"}, + }, + "url": map[string]any{ + "type": "string", + "description": "URL for navigate action", + }, + "index": map[string]any{ + "type": "integer", + "description": "Element index [N] from state output for click/type/fill/select/get_text", + }, + "text": map[string]any{ + "type": "string", + "description": "Text for type/fill actions, option value for select, key name for keys", + }, + "direction": map[string]any{ + "type": "string", + "description": "Scroll direction: up or down", + "enum": []string{"up", "down"}, + }, + "code": map[string]any{ + "type": "string", + "description": "JavaScript code for evaluate action (requires allow_evaluate=true in config)", + }, + }, + "required": []string{"action"}, + } +} + +func (t *BrowserTool) Execute(ctx context.Context, args map[string]any) *ToolResult { + action, _ := args["action"].(string) + if action == "" { + return ErrorResult("action is required") + } + + // Connect lazily on first use + if action != "close" { + if err := t.connectIfNeeded(); err != nil { + return ErrorResult(err.Error()) + } + } + + timeout := time.Duration(t.cfg.Timeout) * time.Second + if timeout <= 0 { + timeout = 30 * time.Second + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + switch action { + case "navigate": + return t.executeNavigate(ctx, args) + case "state": + return t.executeState(ctx) + case "click": + return t.executeClick(ctx, args) + case "type": + return t.executeType(ctx, args) + case "fill": + return t.executeFill(ctx, args) + case "select": + return t.executeSelect(ctx, args) + case "screenshot": + return t.executeScreenshot(ctx) + case "get_text": + return t.executeGetText(ctx, args) + case "scroll": + return t.executeScroll(ctx, args) + case "keys": + return t.executeKeys(ctx, args) + case "evaluate": + return t.executeEvaluate(ctx, args) + case "close": + return t.executeClose() + default: + return ErrorResult(fmt.Sprintf("unknown browser action: %s", action)) + } +} + +// --- Action implementations --- + +func (t *BrowserTool) executeNavigate(ctx context.Context, args map[string]any) *ToolResult { + urlStr, _ := args["url"].(string) + if urlStr == "" { + return ErrorResult("url is required for navigate action") + } + + // Validate URL + if err := validateBrowserURL(urlStr); err != nil { + return ErrorResult(err.Error()) + } + + if err := t.cdp.Navigate(ctx, urlStr); err != nil { + return ErrorResult(fmt.Sprintf("navigation failed: %v", err)) + } + + // Record URL in history (title will be updated when state is called) + t.recordVisit(urlStr, "") + + return SilentResult(fmt.Sprintf("Navigated to %s. Run 'state' to inspect page elements.", urlStr)) +} + +func (t *BrowserTool) executeState(ctx context.Context) *ToolResult { + // JavaScript that extracts interactive elements with [N] indices + js := `(function() { + var selectors = 'a, button, input, select, textarea, [role="button"], [role="link"], [role="tab"], [onclick], [tabindex]:not([tabindex="-1"])'; + var elements = document.querySelectorAll(selectors); + var result = []; + var idx = 0; + for (var i = 0; i < elements.length; i++) { + var el = elements[i]; + var rect = el.getBoundingClientRect(); + if (rect.width === 0 && rect.height === 0) continue; + if (getComputedStyle(el).visibility === 'hidden') continue; + if (getComputedStyle(el).display === 'none') continue; + var tag = el.tagName.toLowerCase(); + var info = { + i: idx, + tag: tag, + text: (el.textContent || '').trim().slice(0, 80).replace(/\s+/g, ' ') + }; + if (el.getAttribute('role')) info.role = el.getAttribute('role'); + if (el.getAttribute('type')) info.type = el.getAttribute('type'); + if (el.getAttribute('name')) info.name = el.getAttribute('name'); + if (el.getAttribute('placeholder')) info.placeholder = el.getAttribute('placeholder'); + if (el.value !== undefined && el.value !== '') info.value = String(el.value).slice(0, 80); + if (tag === 'a' && el.getAttribute('href')) info.href = el.getAttribute('href').slice(0, 120); + if (el.getAttribute('aria-label')) info.label = el.getAttribute('aria-label').slice(0, 80); + if (el.disabled) info.disabled = true; + // Store selector path for interaction + el.setAttribute('data-pcw-idx', String(idx)); + result.push(info); + idx++; + } + return JSON.stringify({ + title: document.title, + url: location.href, + elements: result, + count: idx + }); +})();` + + raw, err := t.cdp.Evaluate(ctx, js) + if err != nil { + return ErrorResult(fmt.Sprintf("failed to extract page state: %v", err)) + } + + // Parse the JSON string returned by JS + var stateStr string + if err := json.Unmarshal(raw, &stateStr); err != nil { + return ErrorResult(fmt.Sprintf("failed to parse state JSON: %v", err)) + } + + var state struct { + Title string `json:"title"` + URL string `json:"url"` + Elements []map[string]any `json:"elements"` + Count int `json:"count"` + } + if err := json.Unmarshal([]byte(stateStr), &state); err != nil { + return ErrorResult(fmt.Sprintf("failed to parse state: %v", err)) + } + + // Update title in history if it changed (SPA navigations) + t.updateCurrentTitle(state.Title) + + // Format output for LLM consumption + var sb strings.Builder + + // Include browsing history summary for context + if summary := t.historySummary(); summary != "" { + sb.WriteString(summary) + sb.WriteByte('\n') + } + + sb.WriteString(fmt.Sprintf("Current page: %s\nURL: %s\n\n", state.Title, state.URL)) + sb.WriteString(fmt.Sprintf("Interactive elements (%d):\n", state.Count)) + + for _, el := range state.Elements { + idxVal, ok := el["i"].(float64) + if !ok { + continue + } + idx := int(idxVal) + tag, _ := el["tag"].(string) + text, _ := el["text"].(string) + + sb.WriteString(fmt.Sprintf("[%d] %s", idx, tag)) + if role, ok := el["role"].(string); ok { + sb.WriteString(fmt.Sprintf(" role=%q", role)) + } + if typ, ok := el["type"].(string); ok { + sb.WriteString(fmt.Sprintf(" type=%q", typ)) + } + if name, ok := el["name"].(string); ok { + sb.WriteString(fmt.Sprintf(" name=%q", name)) + } + if placeholder, ok := el["placeholder"].(string); ok { + sb.WriteString(fmt.Sprintf(" placeholder=%q", placeholder)) + } + if value, ok := el["value"].(string); ok { + sb.WriteString(fmt.Sprintf(" value=%q", value)) + } + if href, ok := el["href"].(string); ok { + sb.WriteString(fmt.Sprintf(" href=%q", href)) + } + if label, ok := el["label"].(string); ok { + sb.WriteString(fmt.Sprintf(" aria-label=%q", label)) + } + if disabled, ok := el["disabled"].(bool); ok && disabled { + sb.WriteString(" [disabled]") + } + if text != "" { + sb.WriteString(fmt.Sprintf(" %q", text)) + } + sb.WriteByte('\n') + } + + return SilentResult(sb.String()) +} + +func (t *BrowserTool) executeClick(ctx context.Context, args map[string]any) *ToolResult { + index, ok := getIntArg(args, "index") + if !ok { + return ErrorResult("index is required for click action (use [N] from state output)") + } + + // Click using the data-pcw-idx attribute we set during state + js := fmt.Sprintf(`(function() { + var el = document.querySelector('[data-pcw-idx="%d"]'); + if (!el) return JSON.stringify({error: 'Element [%d] not found. Run state to refresh indices.'}); + el.scrollIntoView({block: 'center', behavior: 'instant'}); + el.click(); + return JSON.stringify({ok: true, tag: el.tagName.toLowerCase(), text: (el.textContent || '').trim().slice(0, 40)}); +})();`, index, index) + + raw, err := t.cdp.Evaluate(ctx, js) + if err != nil { + return ErrorResult(fmt.Sprintf("click failed: %v", err)) + } + + var resultStr string + if err := json.Unmarshal(raw, &resultStr); err != nil { + return ErrorResult(fmt.Sprintf("failed to parse click result: %v", err)) + } + + var result struct { + OK bool `json:"ok"` + Error string `json:"error"` + Tag string `json:"tag"` + Text string `json:"text"` + } + if err := json.Unmarshal([]byte(resultStr), &result); err != nil { + return ErrorResult(fmt.Sprintf("failed to parse click result: %v", err)) + } + + if result.Error != "" { + return ErrorResult(result.Error) + } + + return SilentResult(fmt.Sprintf("Clicked [%d] <%s> %q. Run 'state' to see updated page.", index, result.Tag, result.Text)) +} + +func (t *BrowserTool) executeType(ctx context.Context, args map[string]any) *ToolResult { + index, ok := getIntArg(args, "index") + if !ok { + return ErrorResult("index is required for type action") + } + text, _ := args["text"].(string) + if text == "" { + return ErrorResult("text is required for type action") + } + + // Focus the element + focusJS := fmt.Sprintf(`(function() { + var el = document.querySelector('[data-pcw-idx="%d"]'); + if (!el) return JSON.stringify({error: 'Element [%d] not found. Run state to refresh indices.'}); + el.scrollIntoView({block: 'center', behavior: 'instant'}); + el.focus(); + return JSON.stringify({ok: true}); +})();`, index, index) + + raw, err := t.cdp.Evaluate(ctx, focusJS) + if err != nil { + return ErrorResult(fmt.Sprintf("focus failed: %v", err)) + } + + var focusStr string + if err := json.Unmarshal(raw, &focusStr); err != nil { + return ErrorResult(fmt.Sprintf("failed to parse focus result: %v", err)) + } + if strings.Contains(focusStr, "error") { + var result struct{ Error string `json:"error"` } + json.Unmarshal([]byte(focusStr), &result) + if result.Error != "" { + return ErrorResult(result.Error) + } + } + + // Type using CDP Input.insertText + if err := t.cdp.InsertText(text); err != nil { + return ErrorResult(fmt.Sprintf("type failed: %v", err)) + } + + return SilentResult(fmt.Sprintf("Typed %q into [%d].", text, index)) +} + +func (t *BrowserTool) executeFill(ctx context.Context, args map[string]any) *ToolResult { + index, ok := getIntArg(args, "index") + if !ok { + return ErrorResult("index is required for fill action") + } + text, _ := args["text"].(string) + if text == "" { + return ErrorResult("text is required for fill action") + } + + // Focus, select all existing text, then type new text + clearJS := fmt.Sprintf(`(function() { + var el = document.querySelector('[data-pcw-idx="%d"]'); + if (!el) return JSON.stringify({error: 'Element [%d] not found.'}); + el.scrollIntoView({block: 'center', behavior: 'instant'}); + el.focus(); + el.value = ''; + el.dispatchEvent(new Event('input', {bubbles: true})); + return JSON.stringify({ok: true}); +})();`, index, index) + + raw, err := t.cdp.Evaluate(ctx, clearJS) + if err != nil { + return ErrorResult(fmt.Sprintf("fill failed (clear step): %v", err)) + } + + var clearStr string + if err := json.Unmarshal(raw, &clearStr); err != nil { + return ErrorResult(fmt.Sprintf("failed to parse fill result: %v", err)) + } + if strings.Contains(clearStr, "error") { + var result struct{ Error string `json:"error"` } + json.Unmarshal([]byte(clearStr), &result) + if result.Error != "" { + return ErrorResult(result.Error) + } + } + + // Type new value + if err := t.cdp.InsertText(text); err != nil { + return ErrorResult(fmt.Sprintf("fill failed (type step): %v", err)) + } + + return SilentResult(fmt.Sprintf("Filled [%d] with %q.", index, text)) +} + +func (t *BrowserTool) executeSelect(ctx context.Context, args map[string]any) *ToolResult { + index, ok := getIntArg(args, "index") + if !ok { + return ErrorResult("index is required for select action") + } + text, _ := args["text"].(string) + if text == "" { + return ErrorResult("text (option value) is required for select action") + } + + textJSON, _ := json.Marshal(text) + js := fmt.Sprintf(`(function() { + var el = document.querySelector('[data-pcw-idx="%d"]'); + if (!el) return JSON.stringify({error: 'Element [%d] not found.'}); + if (el.tagName.toLowerCase() !== 'select') return JSON.stringify({error: 'Element [%d] is not a