Merge pull request #25 from hobbyistlabs-coder/feat/agent-autonomous-tools-12968736795531090015

✨ Feature: Autonomous Execution Tools (Browser & Go Eval)

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/mymmrac/telego v1.6.0
 	github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1
 	github.com/openai/openai-go/v3 v3.22.0
+	github.com/playwright-community/playwright-go v0.5700.1
 	github.com/rivo/tview v0.42.0
 	github.com/rs/zerolog v1.34.0
 	github.com/slack-go/slack v0.17.3
@@ -41,9 +42,12 @@ require (
 	github.com/beeper/argo-go v1.1.2 // indirect
 	github.com/coder/websocket v1.8.14 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/deckarep/golang-set/v2 v2.8.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/elliotchance/orderedmap/v3 v3.1.0 // indirect
 	github.com/gdamore/encoding v1.0.1 // indirect
+	github.com/go-jose/go-jose/v3 v3.0.4 // indirect
+	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.3.0 // indirect

pkg/agent/loop_init.go

@@ -115,6 +115,16 @@ func registerSharedTools(
 			}
 		}
 
+		if cfg.Tools.IsToolEnabled("browser_action") {
+			browserActionTool := tools.NewBrowserActionTool()
+			agent.Tools.Register(browserActionTool)
+		}
+
+		if cfg.Tools.IsToolEnabled("go_eval") {
+			goEvalTool := tools.NewGoEvalTool(agent.Workspace)
+			agent.Tools.Register(goEvalTool)
+		}
+
 		// Hardware tools (I2C, SPI) - Linux only, returns error on other platforms
 		if cfg.Tools.IsToolEnabled("i2c") {
 			agent.Tools.Register(tools.NewI2CTool())

pkg/config/tools.go

@@ -125,6 +125,8 @@ type ToolsConfig struct {
 	Subagent        ToolConfig         `json:"subagent"                                                 envPrefix:"PICOCLAW_TOOLS_SUBAGENT_"`
 	WebFetch        ToolConfig         `json:"web_fetch"                                                envPrefix:"PICOCLAW_TOOLS_WEB_FETCH_"`
 	WriteFile       ToolConfig         `json:"write_file"                                               envPrefix:"PICOCLAW_TOOLS_WRITE_FILE_"`
+	BrowserAction   ToolConfig         `json:"browser_action"                                           envPrefix:"PICOCLAW_TOOLS_BROWSER_ACTION_"`
+	GoEval          ToolConfig         `json:"go_eval"                                                  envPrefix:"PICOCLAW_TOOLS_GO_EVAL_"`
 }
 
 type SearchCacheConfig struct {
@@ -212,6 +214,10 @@ func (t *ToolsConfig) IsToolEnabled(name string) bool {
 		return t.Subagent.Enabled
 	case "web_fetch":
 		return t.WebFetch.Enabled
+	case "browser_action":
+		return t.BrowserAction.Enabled
+	case "go_eval":
+		return t.GoEval.Enabled
 	case "send_file":
 		return t.SendFile.Enabled
 	case "write_file":

pkg/tools/browser.go

@@ -0,0 +1,252 @@
+package tools
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/playwright-community/playwright-go"
+	"jane/pkg/logger"
+)
+
+type BrowserActionTool struct {
+	mu      sync.Mutex
+	pw      *playwright.Playwright
+	browser playwright.Browser
+	context playwright.BrowserContext
+	page    playwright.Page
+}
+
+func NewBrowserActionTool() *BrowserActionTool {
+	return &BrowserActionTool{}
+}
+
+func (t *BrowserActionTool) Name() string {
+	return "browser_action"
+}
+
+func (t *BrowserActionTool) Description() string {
+	return "Interact with a web browser. Actions: 'navigate' (requires url), 'click' (requires selector), 'type' (requires selector, text), 'extract' (returns page text), 'screenshot' (returns base64 image), 'wait' (requires selector). Use this for complex web tasks that require JavaScript rendering or interaction."
+}
+
+func (t *BrowserActionTool) Parameters() map[string]any {
+	return map[string]any{
+		"type": "object",
+		"properties": map[string]any{
+			"action": map[string]any{
+				"type":        "string",
+				"description": "The action to perform: 'navigate', 'click', 'type', 'extract', 'screenshot', 'wait'",
+				"enum":        []string{"navigate", "click", "type", "extract", "screenshot", "wait"},
+			},
+			"url": map[string]any{
+				"type":        "string",
+				"description": "URL to navigate to (used with 'navigate' action)",
+			},
+			"selector": map[string]any{
+				"type":        "string",
+				"description": "CSS selector or XPath for the element to interact with (used with 'click', 'type', 'wait' actions)",
+			},
+			"text": map[string]any{
+				"type":        "string",
+				"description": "Text to type into the element (used with 'type' action)",
+			},
+		},
+		"required": []string{"action"},
+	}
+}
+
+func (t *BrowserActionTool) ensureBrowser() error {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	if t.pw != nil && t.browser != nil && t.context != nil && t.page != nil {
+		return nil
+	}
+
+	err := playwright.Install()
+	if err != nil {
+		logger.WarnCF("tool", "Playwright install warning/error", map[string]any{"error": err.Error()})
+		// Continue even if install returns an error, as it might already be installed
+	}
+
+	pw, err := playwright.Run()
+	if err != nil {
+		return fmt.Errorf("could not start playwright: %w", err)
+	}
+	t.pw = pw
+
+	browser, err := pw.Chromium.Launch(playwright.BrowserTypeLaunchOptions{
+		Headless: playwright.Bool(true),
+	})
+	if err != nil {
+		return fmt.Errorf("could not launch browser: %w", err)
+	}
+	t.browser = browser
+
+	context, err := browser.NewContext(playwright.BrowserNewContextOptions{
+		UserAgent: playwright.String("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
+	})
+	if err != nil {
+		return fmt.Errorf("could not create context: %w", err)
+	}
+	t.context = context
+
+	page, err := context.NewPage()
+	if err != nil {
+		return fmt.Errorf("could not create page: %w", err)
+	}
+	t.page = page
+
+	return nil
+}
+
+func (t *BrowserActionTool) Execute(ctx context.Context, args map[string]any) *ToolResult {
+	action, ok := args["action"].(string)
+	if !ok {
+		return ErrorResult("action is required")
+	}
+
+	if err := t.ensureBrowser(); err != nil {
+		return ErrorResult(fmt.Sprintf("Failed to initialize browser: %v", err))
+	}
+
+	// We lock around the actual playwright interactions to avoid concurrent page mutations
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	switch action {
+	case "navigate":
+		urlStr, ok := args["url"].(string)
+		if !ok || urlStr == "" {
+			return ErrorResult("url is required for navigate action")
+		}
+
+		// If the URL doesn't have a scheme, prepend https://
+		if !strings.HasPrefix(urlStr, "http://") && !strings.HasPrefix(urlStr, "https://") {
+			urlStr = "https://" + urlStr
+		}
+
+		if _, err := t.page.Goto(urlStr, playwright.PageGotoOptions{
+			WaitUntil: playwright.WaitUntilStateDomcontentloaded,
+			Timeout:   playwright.Float(30000), // 30s timeout
+		}); err != nil {
+			return ErrorResult(fmt.Sprintf("Failed to navigate to %s: %v", urlStr, err))
+		}
+
+		title, _ := t.page.Title()
+		return SilentResult(fmt.Sprintf("Navigated to %s. Page title: %s", urlStr, title))
+
+	case "click":
+		selector, ok := args["selector"].(string)
+		if !ok || selector == "" {
+			return ErrorResult("selector is required for click action")
+		}
+		if err := t.page.Click(selector, playwright.PageClickOptions{
+			Timeout: playwright.Float(10000),
+		}); err != nil {
+			return ErrorResult(fmt.Sprintf("Failed to click %s: %v", selector, err))
+		}
+
+		// Wait a bit for navigation or changes after click
+		time.Sleep(1 * time.Second)
+		url := t.page.URL()
+		return SilentResult(fmt.Sprintf("Clicked element %s. Current URL: %s", selector, url))
+
+	case "type":
+		selector, ok := args["selector"].(string)
+		if !ok || selector == "" {
+			return ErrorResult("selector is required for type action")
+		}
+		text, ok := args["text"].(string)
+		if !ok {
+			return ErrorResult("text is required for type action")
+		}
+		if err := t.page.Fill(selector, text, playwright.PageFillOptions{
+			Timeout: playwright.Float(10000),
+		}); err != nil {
+			return ErrorResult(fmt.Sprintf("Failed to type into %s: %v", selector, err))
+		}
+		return SilentResult(fmt.Sprintf("Typed text into %s", selector))
+
+	case "wait":
+		selector, ok := args["selector"].(string)
+		if !ok || selector == "" {
+			return ErrorResult("selector is required for wait action")
+		}
+		_, err := t.page.WaitForSelector(selector, playwright.PageWaitForSelectorOptions{
+			State:   playwright.WaitForSelectorStateVisible,
+			Timeout: playwright.Float(15000),
+		})
+		if err != nil {
+			return ErrorResult(fmt.Sprintf("Timed out waiting for selector %s: %v", selector, err))
+		}
+		return SilentResult(fmt.Sprintf("Selector %s is now visible", selector))
+
+	case "extract":
+		text, err := t.page.Evaluate(`() => {
+			// Basic text extraction, preferring readable content over scripts/styles
+			const extractText = (node) => {
+				if (node.nodeType === Node.TEXT_NODE) {
+					return node.textContent;
+				}
+				if (node.nodeType !== Node.ELEMENT_NODE) {
+					return '';
+				}
+				const tag = node.tagName.toLowerCase();
+				if (tag === 'script' || tag === 'style' || tag === 'noscript') {
+					return '';
+				}
+				let text = '';
+				for (const child of node.childNodes) {
+					text += extractText(child);
+				}
+				return text;
+			};
+			return extractText(document.body).replace(/\s+/g, ' ').trim();
+		}`)
+		if err != nil {
+			return ErrorResult(fmt.Sprintf("Failed to extract text: %v", err))
+		}
+
+		extractedText, ok := text.(string)
+		if !ok {
+			return ErrorResult("Failed to parse extracted text")
+		}
+
+		// Truncate if too long (similar to web_fetch)
+		maxChars := 10000
+		if len(extractedText) > maxChars {
+			extractedText = extractedText[:maxChars] + fmt.Sprintf("\n... (truncated, %d more chars)", len(extractedText)-maxChars)
+		}
+
+		url := t.page.URL()
+		title, _ := t.page.Title()
+		return &ToolResult{
+			ForLLM:  fmt.Sprintf("URL: %s\nTitle: %s\n\nContent:\n%s", url, title, extractedText),
+			ForUser: fmt.Sprintf("Extracted %d chars from %s", len(extractedText), url),
+		}
+
+	case "screenshot":
+		return ErrorResult("Screenshot action is not fully implemented for this environment yet (requires media handling).")
+
+	default:
+		return ErrorResult(fmt.Sprintf("Unknown action: %s", action))
+	}
+}
+
+// Close gracefully shuts down the browser instance
+func (t *BrowserActionTool) Close() {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	if t.browser != nil {
+		t.browser.Close()
+		t.browser = nil
+	}
+	if t.pw != nil {
+		t.pw.Stop()
+		t.pw = nil
+	}
+}

pkg/tools/go_eval.go

@@ -0,0 +1,131 @@
+package tools
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+)
+
+type GoEvalTool struct {
+	workspace string
+	timeout   time.Duration
+}
+
+func NewGoEvalTool(workspace string) *GoEvalTool {
+	return &GoEvalTool{
+		workspace: workspace,
+		timeout:   60 * time.Second, // Default timeout
+	}
+}
+
+func (t *GoEvalTool) Name() string {
+	return "go_eval"
+}
+
+func (t *GoEvalTool) Description() string {
+	return "Executes Go code dynamically. Provide valid Go source code containing a 'main' function. The code will be saved to a temporary file, compiled, and executed. Useful for complex logic or tasks that require writing a Go script."
+}
+
+func (t *GoEvalTool) Parameters() map[string]any {
+	return map[string]any{
+		"type": "object",
+		"properties": map[string]any{
+			"code": map[string]any{
+				"type":        "string",
+				"description": "Valid Go source code to execute (must include 'package main' and 'func main()').",
+			},
+		},
+		"required": []string{"code"},
+	}
+}
+
+func (t *GoEvalTool) Execute(ctx context.Context, args map[string]any) *ToolResult {
+	code, ok := args["code"].(string)
+	if !ok || code == "" {
+		return ErrorResult("code is required")
+	}
+
+	// Create a temporary directory for the Go code
+	tmpDir, err := os.MkdirTemp(t.workspace, "go_eval_*")
+	if err != nil {
+		return ErrorResult(fmt.Sprintf("failed to create temp dir: %v", err))
+	}
+	defer os.RemoveAll(tmpDir)
+
+	// Write code to main.go
+	mainFile := filepath.Join(tmpDir, "main.go")
+	if err := os.WriteFile(mainFile, []byte(code), 0600); err != nil {
+		return ErrorResult(fmt.Sprintf("failed to write code to file: %v", err))
+	}
+
+	// Initialize a go module to allow imports from standard library
+	cmdMod := exec.Command("go", "mod", "init", "goeval")
+	cmdMod.Dir = tmpDir
+	if out, err := cmdMod.CombinedOutput(); err != nil {
+		return ErrorResult(fmt.Sprintf("failed to init go mod: %v\nOutput: %s", err, string(out)))
+	}
+
+	// Tidy the module to fetch any dependencies
+	cmdTidy := exec.Command("go", "mod", "tidy")
+	cmdTidy.Dir = tmpDir
+	if out, err := cmdTidy.CombinedOutput(); err != nil {
+		// Log the error but don't fail immediately, it might just be standard library
+		_ = out
+	}
+
+	// Run the code
+	cmdCtx, cancel := context.WithTimeout(ctx, t.timeout)
+	defer cancel()
+
+	cmdRun := exec.CommandContext(cmdCtx, "go", "run", "main.go")
+	cmdRun.Dir = tmpDir
+
+	var stdout, stderr bytes.Buffer
+	cmdRun.Stdout = &stdout
+	cmdRun.Stderr = &stderr
+
+	err = cmdRun.Run()
+
+	output := stdout.String()
+	if stderr.Len() > 0 {
+		if output != "" {
+			output += "\nSTDERR:\n"
+		}
+		output += stderr.String()
+	}
+
+	if err != nil {
+		if cmdCtx.Err() == context.DeadlineExceeded {
+			return &ToolResult{
+				ForLLM:  fmt.Sprintf("Execution timed out after %v.\nOutput so far:\n%s", t.timeout, output),
+				ForUser: "Execution timed out.",
+				IsError: true,
+			}
+		}
+
+		return &ToolResult{
+			ForLLM:  fmt.Sprintf("Execution failed: %v\nOutput:\n%s", err, output),
+			ForUser: "Execution failed.",
+			IsError: true,
+		}
+	}
+
+	if output == "" {
+		output = "(no output)"
+	}
+
+	// Truncate output if necessary
+	maxLen := 10000
+	if len(output) > maxLen {
+		output = output[:maxLen] + fmt.Sprintf("\n... (truncated, %d more chars)", len(output)-maxLen)
+	}
+
+	return &ToolResult{
+		ForLLM:  output,
+		ForUser: output,
+	}
+}