diff --git a/go.mod b/go.mod index d056cc817..7413646a4 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/mymmrac/telego v1.6.0 github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1 github.com/openai/openai-go/v3 v3.22.0 + github.com/playwright-community/playwright-go v0.5700.1 github.com/rivo/tview v0.42.0 github.com/rs/zerolog v1.34.0 github.com/slack-go/slack v0.17.3 @@ -41,9 +42,12 @@ require ( github.com/beeper/argo-go v1.1.2 // indirect github.com/coder/websocket v1.8.14 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/deckarep/golang-set/v2 v2.8.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/elliotchance/orderedmap/v3 v3.1.0 // indirect github.com/gdamore/encoding v1.0.1 // indirect + github.com/go-jose/go-jose/v3 v3.0.4 // indirect + github.com/go-stack/stack v1.8.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect diff --git a/pkg/agent/loop_init.go b/pkg/agent/loop_init.go index 5df39adb2..283ead868 100644 --- a/pkg/agent/loop_init.go +++ b/pkg/agent/loop_init.go @@ -115,6 +115,16 @@ func registerSharedTools( } } + if cfg.Tools.IsToolEnabled("browser_action") { + browserActionTool := tools.NewBrowserActionTool() + agent.Tools.Register(browserActionTool) + } + + if cfg.Tools.IsToolEnabled("go_eval") { + goEvalTool := tools.NewGoEvalTool(agent.Workspace) + agent.Tools.Register(goEvalTool) + } + // Hardware tools (I2C, SPI) - Linux only, returns error on other platforms if cfg.Tools.IsToolEnabled("i2c") { agent.Tools.Register(tools.NewI2CTool()) diff --git a/pkg/config/tools.go b/pkg/config/tools.go index 8566a3937..c47ff755f 100644 --- a/pkg/config/tools.go +++ b/pkg/config/tools.go @@ -125,6 +125,8 @@ type ToolsConfig struct { Subagent ToolConfig `json:"subagent" envPrefix:"PICOCLAW_TOOLS_SUBAGENT_"` WebFetch ToolConfig `json:"web_fetch" envPrefix:"PICOCLAW_TOOLS_WEB_FETCH_"` WriteFile ToolConfig `json:"write_file" envPrefix:"PICOCLAW_TOOLS_WRITE_FILE_"` + BrowserAction ToolConfig `json:"browser_action" envPrefix:"PICOCLAW_TOOLS_BROWSER_ACTION_"` + GoEval ToolConfig `json:"go_eval" envPrefix:"PICOCLAW_TOOLS_GO_EVAL_"` } type SearchCacheConfig struct { @@ -212,6 +214,10 @@ func (t *ToolsConfig) IsToolEnabled(name string) bool { return t.Subagent.Enabled case "web_fetch": return t.WebFetch.Enabled + case "browser_action": + return t.BrowserAction.Enabled + case "go_eval": + return t.GoEval.Enabled case "send_file": return t.SendFile.Enabled case "write_file": diff --git a/pkg/tools/browser.go b/pkg/tools/browser.go new file mode 100644 index 000000000..c7783f86c --- /dev/null +++ b/pkg/tools/browser.go @@ -0,0 +1,252 @@ +package tools + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/playwright-community/playwright-go" + "jane/pkg/logger" +) + +type BrowserActionTool struct { + mu sync.Mutex + pw *playwright.Playwright + browser playwright.Browser + context playwright.BrowserContext + page playwright.Page +} + +func NewBrowserActionTool() *BrowserActionTool { + return &BrowserActionTool{} +} + +func (t *BrowserActionTool) Name() string { + return "browser_action" +} + +func (t *BrowserActionTool) Description() string { + return "Interact with a web browser. Actions: 'navigate' (requires url), 'click' (requires selector), 'type' (requires selector, text), 'extract' (returns page text), 'screenshot' (returns base64 image), 'wait' (requires selector). Use this for complex web tasks that require JavaScript rendering or interaction." +} + +func (t *BrowserActionTool) Parameters() map[string]any { + return map[string]any{ + "type": "object", + "properties": map[string]any{ + "action": map[string]any{ + "type": "string", + "description": "The action to perform: 'navigate', 'click', 'type', 'extract', 'screenshot', 'wait'", + "enum": []string{"navigate", "click", "type", "extract", "screenshot", "wait"}, + }, + "url": map[string]any{ + "type": "string", + "description": "URL to navigate to (used with 'navigate' action)", + }, + "selector": map[string]any{ + "type": "string", + "description": "CSS selector or XPath for the element to interact with (used with 'click', 'type', 'wait' actions)", + }, + "text": map[string]any{ + "type": "string", + "description": "Text to type into the element (used with 'type' action)", + }, + }, + "required": []string{"action"}, + } +} + +func (t *BrowserActionTool) ensureBrowser() error { + t.mu.Lock() + defer t.mu.Unlock() + + if t.pw != nil && t.browser != nil && t.context != nil && t.page != nil { + return nil + } + + err := playwright.Install() + if err != nil { + logger.WarnCF("tool", "Playwright install warning/error", map[string]any{"error": err.Error()}) + // Continue even if install returns an error, as it might already be installed + } + + pw, err := playwright.Run() + if err != nil { + return fmt.Errorf("could not start playwright: %w", err) + } + t.pw = pw + + browser, err := pw.Chromium.Launch(playwright.BrowserTypeLaunchOptions{ + Headless: playwright.Bool(true), + }) + if err != nil { + return fmt.Errorf("could not launch browser: %w", err) + } + t.browser = browser + + context, err := browser.NewContext(playwright.BrowserNewContextOptions{ + UserAgent: playwright.String("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"), + }) + if err != nil { + return fmt.Errorf("could not create context: %w", err) + } + t.context = context + + page, err := context.NewPage() + if err != nil { + return fmt.Errorf("could not create page: %w", err) + } + t.page = page + + return nil +} + +func (t *BrowserActionTool) Execute(ctx context.Context, args map[string]any) *ToolResult { + action, ok := args["action"].(string) + if !ok { + return ErrorResult("action is required") + } + + if err := t.ensureBrowser(); err != nil { + return ErrorResult(fmt.Sprintf("Failed to initialize browser: %v", err)) + } + + // We lock around the actual playwright interactions to avoid concurrent page mutations + t.mu.Lock() + defer t.mu.Unlock() + + switch action { + case "navigate": + urlStr, ok := args["url"].(string) + if !ok || urlStr == "" { + return ErrorResult("url is required for navigate action") + } + + // If the URL doesn't have a scheme, prepend https:// + if !strings.HasPrefix(urlStr, "http://") && !strings.HasPrefix(urlStr, "https://") { + urlStr = "https://" + urlStr + } + + if _, err := t.page.Goto(urlStr, playwright.PageGotoOptions{ + WaitUntil: playwright.WaitUntilStateDomcontentloaded, + Timeout: playwright.Float(30000), // 30s timeout + }); err != nil { + return ErrorResult(fmt.Sprintf("Failed to navigate to %s: %v", urlStr, err)) + } + + title, _ := t.page.Title() + return SilentResult(fmt.Sprintf("Navigated to %s. Page title: %s", urlStr, title)) + + case "click": + selector, ok := args["selector"].(string) + if !ok || selector == "" { + return ErrorResult("selector is required for click action") + } + if err := t.page.Click(selector, playwright.PageClickOptions{ + Timeout: playwright.Float(10000), + }); err != nil { + return ErrorResult(fmt.Sprintf("Failed to click %s: %v", selector, err)) + } + + // Wait a bit for navigation or changes after click + time.Sleep(1 * time.Second) + url := t.page.URL() + return SilentResult(fmt.Sprintf("Clicked element %s. Current URL: %s", selector, url)) + + case "type": + selector, ok := args["selector"].(string) + if !ok || selector == "" { + return ErrorResult("selector is required for type action") + } + text, ok := args["text"].(string) + if !ok { + return ErrorResult("text is required for type action") + } + if err := t.page.Fill(selector, text, playwright.PageFillOptions{ + Timeout: playwright.Float(10000), + }); err != nil { + return ErrorResult(fmt.Sprintf("Failed to type into %s: %v", selector, err)) + } + return SilentResult(fmt.Sprintf("Typed text into %s", selector)) + + case "wait": + selector, ok := args["selector"].(string) + if !ok || selector == "" { + return ErrorResult("selector is required for wait action") + } + _, err := t.page.WaitForSelector(selector, playwright.PageWaitForSelectorOptions{ + State: playwright.WaitForSelectorStateVisible, + Timeout: playwright.Float(15000), + }) + if err != nil { + return ErrorResult(fmt.Sprintf("Timed out waiting for selector %s: %v", selector, err)) + } + return SilentResult(fmt.Sprintf("Selector %s is now visible", selector)) + + case "extract": + text, err := t.page.Evaluate(`() => { + // Basic text extraction, preferring readable content over scripts/styles + const extractText = (node) => { + if (node.nodeType === Node.TEXT_NODE) { + return node.textContent; + } + if (node.nodeType !== Node.ELEMENT_NODE) { + return ''; + } + const tag = node.tagName.toLowerCase(); + if (tag === 'script' || tag === 'style' || tag === 'noscript') { + return ''; + } + let text = ''; + for (const child of node.childNodes) { + text += extractText(child); + } + return text; + }; + return extractText(document.body).replace(/\s+/g, ' ').trim(); + }`) + if err != nil { + return ErrorResult(fmt.Sprintf("Failed to extract text: %v", err)) + } + + extractedText, ok := text.(string) + if !ok { + return ErrorResult("Failed to parse extracted text") + } + + // Truncate if too long (similar to web_fetch) + maxChars := 10000 + if len(extractedText) > maxChars { + extractedText = extractedText[:maxChars] + fmt.Sprintf("\n... (truncated, %d more chars)", len(extractedText)-maxChars) + } + + url := t.page.URL() + title, _ := t.page.Title() + return &ToolResult{ + ForLLM: fmt.Sprintf("URL: %s\nTitle: %s\n\nContent:\n%s", url, title, extractedText), + ForUser: fmt.Sprintf("Extracted %d chars from %s", len(extractedText), url), + } + + case "screenshot": + return ErrorResult("Screenshot action is not fully implemented for this environment yet (requires media handling).") + + default: + return ErrorResult(fmt.Sprintf("Unknown action: %s", action)) + } +} + +// Close gracefully shuts down the browser instance +func (t *BrowserActionTool) Close() { + t.mu.Lock() + defer t.mu.Unlock() + + if t.browser != nil { + t.browser.Close() + t.browser = nil + } + if t.pw != nil { + t.pw.Stop() + t.pw = nil + } +} diff --git a/pkg/tools/go_eval.go b/pkg/tools/go_eval.go new file mode 100644 index 000000000..5a8547d02 --- /dev/null +++ b/pkg/tools/go_eval.go @@ -0,0 +1,131 @@ +package tools + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" +) + +type GoEvalTool struct { + workspace string + timeout time.Duration +} + +func NewGoEvalTool(workspace string) *GoEvalTool { + return &GoEvalTool{ + workspace: workspace, + timeout: 60 * time.Second, // Default timeout + } +} + +func (t *GoEvalTool) Name() string { + return "go_eval" +} + +func (t *GoEvalTool) Description() string { + return "Executes Go code dynamically. Provide valid Go source code containing a 'main' function. The code will be saved to a temporary file, compiled, and executed. Useful for complex logic or tasks that require writing a Go script." +} + +func (t *GoEvalTool) Parameters() map[string]any { + return map[string]any{ + "type": "object", + "properties": map[string]any{ + "code": map[string]any{ + "type": "string", + "description": "Valid Go source code to execute (must include 'package main' and 'func main()').", + }, + }, + "required": []string{"code"}, + } +} + +func (t *GoEvalTool) Execute(ctx context.Context, args map[string]any) *ToolResult { + code, ok := args["code"].(string) + if !ok || code == "" { + return ErrorResult("code is required") + } + + // Create a temporary directory for the Go code + tmpDir, err := os.MkdirTemp(t.workspace, "go_eval_*") + if err != nil { + return ErrorResult(fmt.Sprintf("failed to create temp dir: %v", err)) + } + defer os.RemoveAll(tmpDir) + + // Write code to main.go + mainFile := filepath.Join(tmpDir, "main.go") + if err := os.WriteFile(mainFile, []byte(code), 0600); err != nil { + return ErrorResult(fmt.Sprintf("failed to write code to file: %v", err)) + } + + // Initialize a go module to allow imports from standard library + cmdMod := exec.Command("go", "mod", "init", "goeval") + cmdMod.Dir = tmpDir + if out, err := cmdMod.CombinedOutput(); err != nil { + return ErrorResult(fmt.Sprintf("failed to init go mod: %v\nOutput: %s", err, string(out))) + } + + // Tidy the module to fetch any dependencies + cmdTidy := exec.Command("go", "mod", "tidy") + cmdTidy.Dir = tmpDir + if out, err := cmdTidy.CombinedOutput(); err != nil { + // Log the error but don't fail immediately, it might just be standard library + _ = out + } + + // Run the code + cmdCtx, cancel := context.WithTimeout(ctx, t.timeout) + defer cancel() + + cmdRun := exec.CommandContext(cmdCtx, "go", "run", "main.go") + cmdRun.Dir = tmpDir + + var stdout, stderr bytes.Buffer + cmdRun.Stdout = &stdout + cmdRun.Stderr = &stderr + + err = cmdRun.Run() + + output := stdout.String() + if stderr.Len() > 0 { + if output != "" { + output += "\nSTDERR:\n" + } + output += stderr.String() + } + + if err != nil { + if cmdCtx.Err() == context.DeadlineExceeded { + return &ToolResult{ + ForLLM: fmt.Sprintf("Execution timed out after %v.\nOutput so far:\n%s", t.timeout, output), + ForUser: "Execution timed out.", + IsError: true, + } + } + + return &ToolResult{ + ForLLM: fmt.Sprintf("Execution failed: %v\nOutput:\n%s", err, output), + ForUser: "Execution failed.", + IsError: true, + } + } + + if output == "" { + output = "(no output)" + } + + // Truncate output if necessary + maxLen := 10000 + if len(output) > maxLen { + output = output[:maxLen] + fmt.Sprintf("\n... (truncated, %d more chars)", len(output)-maxLen) + } + + return &ToolResult{ + ForLLM: output, + ForUser: output, + } +}