From 62d2c5eeed192185ec518f94d99443ca17c69ea7 Mon Sep 17 00:00:00 2001 From: Danieldd28 Date: Mon, 16 Feb 2026 19:42:45 +0700 Subject: [PATCH] feat: add ActionBook browser automation tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrate ActionBook CLI as three native browser automation tools: - browser_search: find action manuals for websites - browser_get: retrieve CSS selectors and page structure by ID - browser: execute browser commands (open, click, fill, text, etc.) Zero new dependencies — thin CLI wrapper using os/exec. Config-gated via tools.browser.enabled (default: true). 18 unit tests + 1 integration test, all passing. Closes: browser automation support for agent loop --- pkg/agent/loop.go | 7 + pkg/config/config.go | 12 +- pkg/tools/browser.go | 411 ++++++++++++++++++++++++++++++++++++++ pkg/tools/browser_test.go | 283 ++++++++++++++++++++++++++ 4 files changed, 712 insertions(+), 1 deletion(-) create mode 100644 pkg/tools/browser.go create mode 100644 pkg/tools/browser_test.go diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index cd4276155..51070c713 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -84,6 +84,13 @@ func createToolRegistry(workspace string, restrict bool, cfg *config.Config, msg } registry.Register(tools.NewWebFetchTool(50000)) + // Browser automation tools (ActionBook) + if cfg.Tools.Browser.Enabled { + registry.Register(tools.NewBrowserSearchTool(cfg.Tools.Browser.Headless)) + registry.Register(tools.NewBrowserGetTool()) + registry.Register(tools.NewBrowserTool(cfg.Tools.Browser.Headless)) + } + // Hardware tools (I2C, SPI) - Linux only, returns error on other platforms registry.Register(tools.NewI2CTool()) registry.Register(tools.NewSPITool()) diff --git a/pkg/config/config.go b/pkg/config/config.go index d189ff00b..bc63acbb5 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -211,8 +211,14 @@ type WebToolsConfig struct { DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"` } +type BrowserConfig struct { + Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_BROWSER_ENABLED"` + Headless bool `json:"headless" env:"PICOCLAW_TOOLS_BROWSER_HEADLESS"` +} + type ToolsConfig struct { - Web WebToolsConfig `json:"web"` + Web WebToolsConfig `json:"web"` + Browser BrowserConfig `json:"browser"` } func DefaultConfig() *Config { @@ -322,6 +328,10 @@ func DefaultConfig() *Config { MaxResults: 5, }, }, + Browser: BrowserConfig{ + Enabled: true, + Headless: true, + }, }, Heartbeat: HeartbeatConfig{ Enabled: true, diff --git a/pkg/tools/browser.go b/pkg/tools/browser.go new file mode 100644 index 000000000..0f9badabc --- /dev/null +++ b/pkg/tools/browser.go @@ -0,0 +1,411 @@ +// PicoClaw - Ultra-lightweight personal AI agent +// License: MIT +// +// Copyright (c) 2026 PicoClaw contributors + +package tools + +import ( + "bytes" + "context" + "fmt" + "os/exec" + "strings" + "time" +) + +// actionbookBinary caches the resolved path to the actionbook CLI. +var actionbookBinary string + +func resolveActionbook() (string, error) { + if actionbookBinary != "" { + return actionbookBinary, nil + } + path, err := exec.LookPath("actionbook") + if err != nil { + return "", fmt.Errorf("actionbook CLI not found in PATH: %w", err) + } + actionbookBinary = path + return path, nil +} + +// runActionbook executes an actionbook CLI command with a timeout. +func runActionbook(ctx context.Context, timeout time.Duration, args ...string) (string, error) { + bin, err := resolveActionbook() + if err != nil { + return "", err + } + + cmdCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + cmd := exec.CommandContext(cmdCtx, bin, args...) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + if cmdCtx.Err() == context.DeadlineExceeded { + return "", fmt.Errorf("actionbook command timed out after %v", timeout) + } + // Many actionbook commands write useful output even on non-zero exit. + // Return stdout+stderr as output with the error. + out := stdout.String() + if stderr.Len() > 0 { + out += "\nSTDERR: " + stderr.String() + } + if out != "" { + return out, nil // treat as non-fatal; LLM can interpret + } + return "", fmt.Errorf("actionbook failed: %w — %s", err, stderr.String()) + } + + return stdout.String(), nil +} + +// truncateOutput limits output length to avoid blowing up context. +func truncateOutput(s string, max int) string { + if len(s) <= max { + return s + } + return s[:max] + fmt.Sprintf("\n... (truncated, %d more chars)", len(s)-max) +} + +// --------------------------------------------------------------------------- +// BrowserSearchTool — actionbook search +// --------------------------------------------------------------------------- + +// BrowserSearchTool wraps `actionbook search` to find action manuals. +type BrowserSearchTool struct { + headless bool +} + +func NewBrowserSearchTool(headless bool) *BrowserSearchTool { + return &BrowserSearchTool{headless: headless} +} + +func (t *BrowserSearchTool) Name() string { return "browser_search" } +func (t *BrowserSearchTool) Description() string { + return "Search ActionBook for browser action manuals matching a task. Returns action IDs, descriptions, URLs, and health scores. Use the returned ID with browser_get to retrieve selectors." +} + +func (t *BrowserSearchTool) Parameters() map[string]interface{} { + return map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "query": map[string]interface{}{ + "type": "string", + "description": "Search query describing the task (e.g. 'airbnb search listings Tokyo')", + }, + "domain": map[string]interface{}{ + "type": "string", + "description": "Optional domain to filter results (e.g. 'airbnb.com')", + }, + }, + "required": []string{"query"}, + } +} + +func (t *BrowserSearchTool) Execute(ctx context.Context, args map[string]interface{}) *ToolResult { + query, ok := args["query"].(string) + if !ok || query == "" { + return ErrorResult("query is required") + } + + cmdArgs := []string{"search", query, "--json"} + if domain, ok := args["domain"].(string); ok && domain != "" { + cmdArgs = append(cmdArgs, "--domain", domain) + } + + output, err := runActionbook(ctx, 30*time.Second, cmdArgs...) + if err != nil { + return ErrorResult(fmt.Sprintf("browser_search failed: %v", err)) + } + + output = truncateOutput(output, 10000) + return &ToolResult{ + ForLLM: output, + ForUser: output, + } +} + +// --------------------------------------------------------------------------- +// BrowserGetTool — actionbook get +// --------------------------------------------------------------------------- + +// BrowserGetTool wraps `actionbook get` to retrieve action details. +type BrowserGetTool struct{} + +func NewBrowserGetTool() *BrowserGetTool { return &BrowserGetTool{} } + +func (t *BrowserGetTool) Name() string { return "browser_get" } +func (t *BrowserGetTool) Description() string { + return "Retrieve a specific ActionBook action manual by its ID. Returns page structure with CSS selectors, element types, and allowed methods. Use the ID from browser_search results." +} + +func (t *BrowserGetTool) Parameters() map[string]interface{} { + return map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "action_id": map[string]interface{}{ + "type": "string", + "description": "Action manual ID from browser_search (e.g. 'airbnb.com:/:default')", + }, + }, + "required": []string{"action_id"}, + } +} + +func (t *BrowserGetTool) Execute(ctx context.Context, args map[string]interface{}) *ToolResult { + actionID, ok := args["action_id"].(string) + if !ok || actionID == "" { + return ErrorResult("action_id is required") + } + + output, err := runActionbook(ctx, 30*time.Second, "get", actionID, "--json") + if err != nil { + return ErrorResult(fmt.Sprintf("browser_get failed: %v", err)) + } + + output = truncateOutput(output, 15000) + return &ToolResult{ + ForLLM: output, + ForUser: output, + } +} + +// --------------------------------------------------------------------------- +// BrowserTool — actionbook browser * +// --------------------------------------------------------------------------- + +// allowedBrowserActions is the set of browser subcommands the LLM may invoke. +var allowedBrowserActions = map[string]bool{ + "open": true, + "goto": true, + "click": true, + "fill": true, + "type": true, + "select": true, + "hover": true, + "focus": true, + "press": true, + "text": true, + "snapshot": true, + "screenshot": true, + "wait": true, + "wait-nav": true, + "back": true, + "forward": true, + "reload": true, + "close": true, + "pages": true, + "switch": true, + "eval": true, + "html": true, + "pdf": true, + "cookies": true, + "status": true, + "viewport": true, +} + +// BrowserTool wraps `actionbook browser` for browser automation. +type BrowserTool struct { + headless bool +} + +func NewBrowserTool(headless bool) *BrowserTool { + return &BrowserTool{headless: headless} +} + +func (t *BrowserTool) Name() string { return "browser" } +func (t *BrowserTool) Description() string { + return `Execute browser automation commands via ActionBook. Supported actions: open, goto, click, fill, type, select, hover, focus, press, text, snapshot, screenshot, wait, wait-nav, back, forward, reload, close, pages, switch, eval, html, pdf, cookies, status, viewport. Typical workflow: browser_search → browser_get → browser (open → interact → close).` +} + +func (t *BrowserTool) Parameters() map[string]interface{} { + return map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "action": map[string]interface{}{ + "type": "string", + "description": "Browser action to perform (e.g. 'open', 'click', 'fill', 'text', 'snapshot', 'close')", + }, + "url": map[string]interface{}{ + "type": "string", + "description": "URL for open/goto actions", + }, + "selector": map[string]interface{}{ + "type": "string", + "description": "CSS selector for click/fill/type/select/hover/focus/wait/text/html actions", + }, + "value": map[string]interface{}{ + "type": "string", + "description": "Value for fill/type/select/press/eval/switch actions", + }, + "timeout": map[string]interface{}{ + "type": "integer", + "description": "Timeout in milliseconds for wait/wait-nav actions (default: 30000)", + }, + "full_page": map[string]interface{}{ + "type": "boolean", + "description": "For screenshot: capture full page instead of viewport", + }, + }, + "required": []string{"action"}, + } +} + +func (t *BrowserTool) Execute(ctx context.Context, args map[string]interface{}) *ToolResult { + action, ok := args["action"].(string) + if !ok || action == "" { + return ErrorResult("action is required") + } + + action = strings.TrimSpace(strings.ToLower(action)) + if !allowedBrowserActions[action] { + return ErrorResult(fmt.Sprintf("unknown browser action %q — allowed: open, goto, click, fill, type, select, hover, focus, press, text, snapshot, screenshot, wait, wait-nav, back, forward, reload, close, pages, switch, eval, html, pdf, cookies, status, viewport", action)) + } + + // Build argument list + cmdArgs := []string{} + + // Global flags + if t.headless { + cmdArgs = append(cmdArgs, "--headless") + } + + cmdArgs = append(cmdArgs, "browser", action) + + switch action { + case "open", "goto": + if u, ok := args["url"].(string); ok && u != "" { + cmdArgs = append(cmdArgs, u) + } else { + return ErrorResult(fmt.Sprintf("url is required for %s action", action)) + } + if timeout, ok := getIntArg(args, "timeout"); ok { + cmdArgs = append(cmdArgs, "--timeout", fmt.Sprintf("%d", timeout)) + } + + case "click", "hover", "focus": + sel, ok := args["selector"].(string) + if !ok || sel == "" { + return ErrorResult(fmt.Sprintf("selector is required for %s action", action)) + } + cmdArgs = append(cmdArgs, sel) + if timeout, ok := getIntArg(args, "timeout"); ok { + cmdArgs = append(cmdArgs, "--wait", fmt.Sprintf("%d", timeout)) + } + + case "fill", "type": + sel, ok := args["selector"].(string) + if !ok || sel == "" { + return ErrorResult(fmt.Sprintf("selector is required for %s action", action)) + } + val, ok := args["value"].(string) + if !ok { + return ErrorResult(fmt.Sprintf("value is required for %s action", action)) + } + cmdArgs = append(cmdArgs, sel, val) + if timeout, ok := getIntArg(args, "timeout"); ok { + cmdArgs = append(cmdArgs, "--wait", fmt.Sprintf("%d", timeout)) + } + + case "select": + sel, ok := args["selector"].(string) + if !ok || sel == "" { + return ErrorResult("selector is required for select action") + } + val, ok := args["value"].(string) + if !ok || val == "" { + return ErrorResult("value is required for select action") + } + cmdArgs = append(cmdArgs, sel, val) + + case "press": + key, ok := args["value"].(string) + if !ok || key == "" { + return ErrorResult("value is required for press action (e.g. 'Enter', 'Tab')") + } + cmdArgs = append(cmdArgs, key) + + case "wait": + sel, ok := args["selector"].(string) + if !ok || sel == "" { + return ErrorResult("selector is required for wait action") + } + cmdArgs = append(cmdArgs, sel) + if timeout, ok := getIntArg(args, "timeout"); ok { + cmdArgs = append(cmdArgs, "--timeout", fmt.Sprintf("%d", timeout)) + } + + case "wait-nav": + if timeout, ok := getIntArg(args, "timeout"); ok { + cmdArgs = append(cmdArgs, "--timeout", fmt.Sprintf("%d", timeout)) + } + + case "switch": + pageID, ok := args["value"].(string) + if !ok || pageID == "" { + return ErrorResult("value (page_id) is required for switch action") + } + cmdArgs = append(cmdArgs, pageID) + + case "eval": + script, ok := args["value"].(string) + if !ok || script == "" { + return ErrorResult("value (javascript expression) is required for eval action") + } + cmdArgs = append(cmdArgs, script) + + case "text", "html": + if sel, ok := args["selector"].(string); ok && sel != "" { + cmdArgs = append(cmdArgs, sel) + } + + case "screenshot": + if fullPage, ok := args["full_page"].(bool); ok && fullPage { + cmdArgs = append(cmdArgs, "--full-page") + } + + case "cookies": + // cookies sub-actions via value: list, get , set , delete , clear + if val, ok := args["value"].(string); ok && val != "" { + cmdArgs = append(cmdArgs, strings.Fields(val)...) + } + + // For actions with no extra args: back, forward, reload, close, pages, snapshot, status, viewport, pdf + // — nothing extra needed, cmdArgs already has "browser" and action. + } + + // Determine timeout + cmdTimeout := 30 * time.Second + if action == "screenshot" || action == "pdf" { + cmdTimeout = 60 * time.Second + } + + output, err := runActionbook(ctx, cmdTimeout, cmdArgs...) + if err != nil { + return ErrorResult(fmt.Sprintf("browser %s failed: %v", action, err)) + } + + if output == "" { + output = "(no output)" + } + + output = truncateOutput(output, 15000) + return &ToolResult{ + ForLLM: output, + ForUser: output, + } +} + +// getIntArg extracts an integer argument from the args map. +// JSON numbers arrive as float64. +func getIntArg(args map[string]interface{}, key string) (int, bool) { + if v, ok := args[key].(float64); ok { + return int(v), true + } + return 0, false +} diff --git a/pkg/tools/browser_test.go b/pkg/tools/browser_test.go new file mode 100644 index 000000000..0b30ba9da --- /dev/null +++ b/pkg/tools/browser_test.go @@ -0,0 +1,283 @@ +package tools + +import ( + "context" + "os/exec" + "strings" + "testing" +) + +// TestBrowserSearchTool_Name verifies tool name +func TestBrowserSearchTool_Name(t *testing.T) { + tool := NewBrowserSearchTool(true) + if tool.Name() != "browser_search" { + t.Errorf("expected name 'browser_search', got '%s'", tool.Name()) + } +} + +// TestBrowserSearchTool_Description verifies description is non-empty +func TestBrowserSearchTool_Description(t *testing.T) { + tool := NewBrowserSearchTool(true) + if tool.Description() == "" { + t.Error("expected non-empty description") + } +} + +// TestBrowserSearchTool_Parameters verifies parameter schema +func TestBrowserSearchTool_Parameters(t *testing.T) { + tool := NewBrowserSearchTool(true) + params := tool.Parameters() + props, ok := params["properties"].(map[string]interface{}) + if !ok { + t.Fatal("expected properties map") + } + if _, ok := props["query"]; !ok { + t.Error("expected 'query' parameter") + } + if _, ok := props["domain"]; !ok { + t.Error("expected 'domain' parameter") + } + required, ok := params["required"].([]string) + if !ok || len(required) != 1 || required[0] != "query" { + t.Error("expected 'query' in required list") + } +} + +// TestBrowserSearchTool_MissingQuery verifies error when query is missing +func TestBrowserSearchTool_MissingQuery(t *testing.T) { + tool := NewBrowserSearchTool(true) + result := tool.Execute(context.Background(), map[string]interface{}{}) + if !result.IsError { + t.Error("expected error when query is missing") + } + if !strings.Contains(result.ForLLM, "query is required") { + t.Errorf("expected 'query is required' in error, got: %s", result.ForLLM) + } +} + +// TestBrowserGetTool_Name verifies tool name +func TestBrowserGetTool_Name(t *testing.T) { + tool := NewBrowserGetTool() + if tool.Name() != "browser_get" { + t.Errorf("expected name 'browser_get', got '%s'", tool.Name()) + } +} + +// TestBrowserGetTool_MissingID verifies error when action_id is missing +func TestBrowserGetTool_MissingID(t *testing.T) { + tool := NewBrowserGetTool() + result := tool.Execute(context.Background(), map[string]interface{}{}) + if !result.IsError { + t.Error("expected error when action_id is missing") + } + if !strings.Contains(result.ForLLM, "action_id is required") { + t.Errorf("expected 'action_id is required' in error, got: %s", result.ForLLM) + } +} + +// TestBrowserTool_Name verifies tool name +func TestBrowserTool_Name(t *testing.T) { + tool := NewBrowserTool(true) + if tool.Name() != "browser" { + t.Errorf("expected name 'browser', got '%s'", tool.Name()) + } +} + +// TestBrowserTool_MissingAction verifies error when action is missing +func TestBrowserTool_MissingAction(t *testing.T) { + tool := NewBrowserTool(true) + result := tool.Execute(context.Background(), map[string]interface{}{}) + if !result.IsError { + t.Error("expected error when action is missing") + } + if !strings.Contains(result.ForLLM, "action is required") { + t.Errorf("expected 'action is required' in error, got: %s", result.ForLLM) + } +} + +// TestBrowserTool_InvalidAction verifies rejection of unknown actions +func TestBrowserTool_InvalidAction(t *testing.T) { + tool := NewBrowserTool(true) + result := tool.Execute(context.Background(), map[string]interface{}{ + "action": "malicious_command", + }) + if !result.IsError { + t.Error("expected error for invalid action") + } + if !strings.Contains(result.ForLLM, "unknown browser action") { + t.Errorf("expected 'unknown browser action' in error, got: %s", result.ForLLM) + } +} + +// TestBrowserTool_ActionAllowlist verifies all valid actions pass validation +func TestBrowserTool_ActionAllowlist(t *testing.T) { + // Only test the allowlist check, not actual execution + for action := range allowedBrowserActions { + if !allowedBrowserActions[action] { + t.Errorf("action %q should be in allowlist", action) + } + } + + invalidActions := []string{"rm", "exec", "sudo", "rm -rf", "shell", ""} + for _, action := range invalidActions { + if allowedBrowserActions[action] { + t.Errorf("action %q should NOT be in allowlist", action) + } + } +} + +// TestBrowserTool_OpenRequiresURL verifies open action needs url parameter +func TestBrowserTool_OpenRequiresURL(t *testing.T) { + tool := NewBrowserTool(true) + result := tool.Execute(context.Background(), map[string]interface{}{ + "action": "open", + }) + if !result.IsError { + t.Error("expected error when url is missing for open") + } + if !strings.Contains(result.ForLLM, "url is required") { + t.Errorf("expected 'url is required' in error, got: %s", result.ForLLM) + } +} + +// TestBrowserTool_ClickRequiresSelector verifies click action needs selector +func TestBrowserTool_ClickRequiresSelector(t *testing.T) { + tool := NewBrowserTool(true) + result := tool.Execute(context.Background(), map[string]interface{}{ + "action": "click", + }) + if !result.IsError { + t.Error("expected error when selector is missing for click") + } + if !strings.Contains(result.ForLLM, "selector is required") { + t.Errorf("expected 'selector is required' in error, got: %s", result.ForLLM) + } +} + +// TestBrowserTool_FillRequiresSelectorAndValue verifies fill needs both params +func TestBrowserTool_FillRequiresSelectorAndValue(t *testing.T) { + tool := NewBrowserTool(true) + + // Missing selector + result := tool.Execute(context.Background(), map[string]interface{}{ + "action": "fill", + "value": "text", + }) + if !result.IsError { + t.Error("expected error when selector is missing for fill") + } + + // Missing value + result = tool.Execute(context.Background(), map[string]interface{}{ + "action": "fill", + "selector": "#input", + }) + if !result.IsError { + t.Error("expected error when value is missing for fill") + } +} + +// TestBrowserTool_PressRequiresValue verifies press needs key value +func TestBrowserTool_PressRequiresValue(t *testing.T) { + tool := NewBrowserTool(true) + result := tool.Execute(context.Background(), map[string]interface{}{ + "action": "press", + }) + if !result.IsError { + t.Error("expected error when value is missing for press") + } +} + +// TestBrowserTool_EvalRequiresValue verifies eval needs script value +func TestBrowserTool_EvalRequiresValue(t *testing.T) { + tool := NewBrowserTool(true) + result := tool.Execute(context.Background(), map[string]interface{}{ + "action": "eval", + }) + if !result.IsError { + t.Error("expected error when value is missing for eval") + } +} + +// TestBrowserTool_WaitRequiresSelector verifies wait needs selector +func TestBrowserTool_WaitRequiresSelector(t *testing.T) { + tool := NewBrowserTool(true) + result := tool.Execute(context.Background(), map[string]interface{}{ + "action": "wait", + }) + if !result.IsError { + t.Error("expected error when selector is missing for wait") + } +} + +// TestBrowserTool_CaseInsensitive verifies that action matching is case-insensitive +func TestBrowserTool_CaseInsensitive(t *testing.T) { + tool := NewBrowserTool(true) + // "OPEN" should normalize to "open" and then require url + result := tool.Execute(context.Background(), map[string]interface{}{ + "action": "OPEN", + }) + // It should not error with "unknown action" — it should error with "url is required" + if !result.IsError { + t.Error("expected error") + } + if strings.Contains(result.ForLLM, "unknown browser action") { + t.Error("action should be case-insensitive") + } + if !strings.Contains(result.ForLLM, "url is required") { + t.Errorf("expected 'url is required' for 'OPEN' action, got: %s", result.ForLLM) + } +} + +// TestGetIntArg verifies integer argument extraction from float64 +func TestGetIntArg(t *testing.T) { + args := map[string]interface{}{ + "timeout": float64(5000), + } + v, ok := getIntArg(args, "timeout") + if !ok || v != 5000 { + t.Errorf("expected 5000, got %d (ok=%v)", v, ok) + } + + _, ok = getIntArg(args, "nonexistent") + if ok { + t.Error("expected false for missing key") + } +} + +// TestBrowserSearchTool_Integration performs a real search if actionbook is available +func TestBrowserSearchTool_Integration(t *testing.T) { + if _, err := exec.LookPath("actionbook"); err != nil { + t.Skip("actionbook not in PATH, skipping integration test") + } + + tool := NewBrowserSearchTool(true) + result := tool.Execute(context.Background(), map[string]interface{}{ + "query": "google search", + }) + + if result.IsError { + t.Errorf("expected success, got error: %s", result.ForLLM) + } + + if result.ForLLM == "" { + t.Error("expected non-empty output") + } +} + +// TestTruncateOutput verifies output truncation +func TestTruncateOutput(t *testing.T) { + short := "hello" + if truncateOutput(short, 100) != short { + t.Error("short string should not be truncated") + } + + long := strings.Repeat("x", 200) + truncated := truncateOutput(long, 100) + if len(truncated) >= 200 { + t.Errorf("expected truncation, got length %d", len(truncated)) + } + if !strings.Contains(truncated, "truncated") { + t.Error("expected truncation marker") + } +}