From 45adabf82c9aa8dcb3913ebcd7b99df6ab287157 Mon Sep 17 00:00:00 2001 From: Kohei Date: Sun, 22 Feb 2026 13:36:46 +0900 Subject: [PATCH] feat: replace current_activity with screenshot and get_ui_tree actions Remove the current_activity action and add screenshot (returns base64 JPEG as multimodal tool result) and get_ui_tree (accessibility tree with optional resource_id/bounds/index targeting and depth/node limits). - Add Media field to ToolResult and propagate through message pipeline - Extend HTTP provider to support multimodal content on tool messages - Add input validation (mutual exclusivity, bounds, negative index) - Fix existing bug: status.go used "list_apps" instead of "search_apps" Co-Authored-By: Claude Opus 4.6 --- .../android/assistant/AssistantService.kt | 4 + .../android/assistant/DeviceController.kt | 10 +- .../android/assistant/ToolRequestHandler.kt | 154 +++++++++++++++++- pkg/agent/loop.go | 1 + pkg/agent/status.go | 10 +- pkg/providers/http_provider.go | 4 +- pkg/tools/android.go | 76 ++++++++- pkg/tools/result.go | 4 + 8 files changed, 241 insertions(+), 22 deletions(-) diff --git a/android/app/src/main/java/io/picoclaw/android/assistant/AssistantService.kt b/android/app/src/main/java/io/picoclaw/android/assistant/AssistantService.kt index fca09fe76..c83192526 100644 --- a/android/app/src/main/java/io/picoclaw/android/assistant/AssistantService.kt +++ b/android/app/src/main/java/io/picoclaw/android/assistant/AssistantService.kt @@ -117,6 +117,10 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner { toolRequestHandler = ToolRequestHandler( context = applicationContext, deviceController = deviceController, + screenshotSource = screenshotSource, + setOverlayVisibility = { visible -> + overlayView?.visibility = if (visible) View.VISIBLE else View.INVISIBLE + }, onAccessibilityNeeded = { showAccessibilityGuide = true } ) (connection as AssistantConnectionImpl).onToolRequest = { request -> diff --git a/android/app/src/main/java/io/picoclaw/android/assistant/DeviceController.kt b/android/app/src/main/java/io/picoclaw/android/assistant/DeviceController.kt index d0dad59cc..23efe872b 100644 --- a/android/app/src/main/java/io/picoclaw/android/assistant/DeviceController.kt +++ b/android/app/src/main/java/io/picoclaw/android/assistant/DeviceController.kt @@ -54,14 +54,8 @@ class DeviceController { return service?.performGlobalAction(AccessibilityService.GLOBAL_ACTION_RECENTS) ?: false } - fun getCurrentPackage(): String? { - val svc = service ?: return null - val ownPackage = svc.packageName - // Skip our own overlay window and find the actual foreground app - return svc.windows - .filter { it.type == android.view.accessibility.AccessibilityWindowInfo.TYPE_APPLICATION } - .firstOrNull { it.root?.packageName?.toString() != ownPackage } - ?.root?.packageName?.toString() + fun getRootNode(): AccessibilityNodeInfo? { + return service?.rootInActiveWindow } suspend fun inputText(text: String): Boolean { diff --git a/android/app/src/main/java/io/picoclaw/android/assistant/ToolRequestHandler.kt b/android/app/src/main/java/io/picoclaw/android/assistant/ToolRequestHandler.kt index 21bb77daf..e98bbe347 100644 --- a/android/app/src/main/java/io/picoclaw/android/assistant/ToolRequestHandler.kt +++ b/android/app/src/main/java/io/picoclaw/android/assistant/ToolRequestHandler.kt @@ -4,21 +4,33 @@ import android.content.Context import android.content.Intent import android.content.pm.ApplicationInfo import android.content.pm.PackageManager +import android.graphics.Bitmap +import android.graphics.Rect import android.net.Uri import android.os.Build +import android.util.Base64 import android.util.Log +import android.view.accessibility.AccessibilityNodeInfo import io.picoclaw.android.core.data.remote.dto.ToolRequest import io.picoclaw.android.core.data.remote.dto.ToolResponse +import io.picoclaw.android.feature.chat.voice.ScreenshotSource +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.withContext import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.contentOrNull import kotlinx.serialization.json.doubleOrNull import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.intOrNull import kotlinx.serialization.json.longOrNull +import java.io.ByteArrayOutputStream class ToolRequestHandler( private val context: Context, private val deviceController: DeviceController, + private val screenshotSource: ScreenshotSource, + private val setOverlayVisibility: (Boolean) -> Unit, private val onAccessibilityNeeded: () -> Unit ) { @@ -28,7 +40,8 @@ class ToolRequestHandler( "search_apps" -> handleSearchApps(request) "app_info" -> handleAppInfo(request) "launch_app" -> handleLaunchApp(request) - "current_activity" -> handleCurrentActivity(request) + "screenshot" -> handleScreenshot(request) + "get_ui_tree" -> handleGetUiTree(request) "tap" -> handleTap(request) "swipe" -> handleSwipe(request) "text" -> handleText(request) @@ -140,13 +153,144 @@ class ToolRequestHandler( return ToolResponse(request.requestId, true, result = "Launched $packageName") } - private suspend fun handleCurrentActivity(request: ToolRequest): ToolResponse { + private suspend fun handleScreenshot(request: ToolRequest): ToolResponse { requireAccessibility(request)?.let { return it } - val pkg = deviceController.getCurrentPackage() - ?: return ToolResponse(request.requestId, false, error = "Could not get current activity") + return try { + withContext(Dispatchers.Main) { setOverlayVisibility(false) } + delay(150) + val bitmap = screenshotSource.takeScreenshot() + ?: return ToolResponse(request.requestId, false, error = "Screenshot capture failed") + try { + val base64 = withContext(Dispatchers.IO) { + val stream = ByteArrayOutputStream() + bitmap.compress(Bitmap.CompressFormat.JPEG, 80, stream) + Base64.encodeToString(stream.toByteArray(), Base64.NO_WRAP) + } + ToolResponse(request.requestId, true, result = base64) + } finally { + bitmap.recycle() + } + } finally { + withContext(Dispatchers.Main) { setOverlayVisibility(true) } + } + } - return ToolResponse(request.requestId, true, result = "Current package: $pkg") + private suspend fun handleGetUiTree(request: ToolRequest): ToolResponse { + requireAccessibility(request)?.let { return it } + + val resourceId = request.params?.get("resource_id")?.jsonPrimitive?.contentOrNull + val index = request.params?.get("index")?.jsonPrimitive?.intOrNull ?: 0 + val boundsX = request.params?.get("bounds_x")?.jsonPrimitive?.doubleOrNull + val boundsY = request.params?.get("bounds_y")?.jsonPrimitive?.doubleOrNull + val maxDepth = request.params?.get("max_depth")?.jsonPrimitive?.intOrNull ?: 30 + val maxNodes = request.params?.get("max_nodes")?.jsonPrimitive?.intOrNull ?: 2000 + + return try { + withContext(Dispatchers.Main) { setOverlayVisibility(false) } + delay(150) + val root = deviceController.getRootNode() + ?: return ToolResponse(request.requestId, false, error = "Could not get UI tree") + try { + val startNode = resolveStartNode(root, resourceId, index, boundsX, boundsY) + ?: return ToolResponse(request.requestId, false, error = buildString { + if (resourceId != null) append("No node found with resource_id=$resourceId (index=$index)") + else append("No node found at bounds ($boundsX, $boundsY)") + }) + try { + val sb = StringBuilder() + val nodeCount = intArrayOf(0) + dumpNode(startNode, sb, 0, maxDepth, maxNodes, nodeCount) + if (nodeCount[0] >= maxNodes) { + sb.appendLine("[truncated: max_nodes=$maxNodes reached]") + } + ToolResponse(request.requestId, true, result = sb.toString()) + } finally { + if (startNode !== root) startNode.recycle() + } + } finally { + root.recycle() + } + } finally { + withContext(Dispatchers.Main) { setOverlayVisibility(true) } + } + } + + private fun resolveStartNode( + root: AccessibilityNodeInfo, + resourceId: String?, + index: Int, + boundsX: Double?, + boundsY: Double? + ): AccessibilityNodeInfo? { + if (resourceId != null) { + val matches = root.findAccessibilityNodeInfosByViewId(resourceId) + if (matches.isNullOrEmpty()) return null + val target = matches.getOrNull(index) + // Recycle unused matches + for ((i, node) in matches.withIndex()) { + if (i != index) node.recycle() + } + return target + } + if (boundsX != null && boundsY != null) { + return findNodeAtPoint(root, boundsX.toInt(), boundsY.toInt()) + } + return root + } + + private fun findNodeAtPoint(node: AccessibilityNodeInfo, x: Int, y: Int): AccessibilityNodeInfo? { + val bounds = Rect() + node.getBoundsInScreen(bounds) + if (!bounds.contains(x, y)) return null + // Find the deepest (smallest) child that contains the point + for (i in 0 until node.childCount) { + val child = node.getChild(i) ?: continue + val found = findNodeAtPoint(child, x, y) + if (found != null) return found + child.recycle() + } + return node + } + + private fun dumpNode( + node: AccessibilityNodeInfo, + sb: StringBuilder, + depth: Int, + maxDepth: Int, + maxNodes: Int, + nodeCount: IntArray + ) { + if (nodeCount[0] >= maxNodes) return + nodeCount[0]++ + val indent = " ".repeat(depth) + val bounds = Rect() + node.getBoundsInScreen(bounds) + sb.appendLine( + "${indent}[${node.className}] " + + "text=${node.text ?: ""} " + + "desc=${node.contentDescription ?: ""} " + + "bounds=${bounds} " + + "clickable=${node.isClickable} " + + "enabled=${node.isEnabled} " + + "visible=${node.isVisibleToUser} " + + "id=${node.viewIdResourceName ?: ""}" + ) + if (depth >= maxDepth) { + if (node.childCount > 0) { + sb.appendLine("${indent} [truncated: ${node.childCount} children at depth $depth]") + } + return + } + for (i in 0 until node.childCount) { + if (nodeCount[0] >= maxNodes) return + val child = node.getChild(i) ?: continue + try { + dumpNode(child, sb, depth + 1, maxDepth, maxNodes, nodeCount) + } finally { + child.recycle() + } + } } private suspend fun handleTap(request: ToolRequest): ToolResponse { diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 9555c765b..136a7691d 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -957,6 +957,7 @@ func (al *AgentLoop) runLLMIteration(ctx context.Context, messages []providers.M toolResultMsg := providers.Message{ Role: "tool", Content: contentForLLM, + Media: toolResult.Media, ToolCallID: tc.ID, } messages = append(messages, toolResultMsg) diff --git a/pkg/agent/status.go b/pkg/agent/status.go index 86a628093..6ec47e7c2 100644 --- a/pkg/agent/status.go +++ b/pkg/agent/status.go @@ -120,8 +120,8 @@ func cronStatusLabel(args map[string]interface{}) string { func androidStatusLabel(args map[string]interface{}) string { switch strArg(args, "action") { - case "list_apps": - return "アプリ一覧取得中..." + case "search_apps": + return "アプリ検索中..." case "app_info": if p := strArg(args, "package_name"); p != "" { return fmt.Sprintf("アプリ情報取得中...（%s）", truncLabel(p, 25)) @@ -132,8 +132,10 @@ func androidStatusLabel(args map[string]interface{}) string { return fmt.Sprintf("アプリ起動中...（%s）", truncLabel(p, 25)) } return "アプリ起動中..." - case "current_activity": - return "画面情報取得中..." + case "screenshot": + return "スクリーンショット撮影中..." + case "get_ui_tree": + return "UI要素取得中..." case "tap": return "タップ中..." case "swipe": diff --git a/pkg/providers/http_provider.go b/pkg/providers/http_provider.go index 866a099e5..d7c9c8b45 100644 --- a/pkg/providers/http_provider.go +++ b/pkg/providers/http_provider.go @@ -134,8 +134,8 @@ func (p *HTTPProvider) buildAPIMessages(messages []Message) []map[string]interfa "role": msg.Role, } - // Only user messages with media get the array-style content - if msg.Role == "user" && len(msg.Media) > 0 { + // Messages with media get the array-style content (user or tool) + if len(msg.Media) > 0 && (msg.Role == "user" || msg.Role == "tool") { parts := make([]map[string]interface{}, 0, 1+len(msg.Media)) if msg.Content != "" { parts = append(parts, map[string]interface{}{ diff --git a/pkg/tools/android.go b/pkg/tools/android.go index 0f5e2127c..f4344e8d6 100644 --- a/pkg/tools/android.go +++ b/pkg/tools/android.go @@ -45,7 +45,8 @@ func (t *AndroidTool) Description() string { - search_apps: Search installed apps by name or package name (requires query) - app_info: Get app details (requires package_name) - launch_app: Launch an app (requires package_name) -- current_activity: Get the currently active app/window +- screenshot: Capture a screenshot of the current screen (no params) +- get_ui_tree: Get the accessibility UI tree (optional: resource_id, index, bounds_x/bounds_y, max_depth, max_nodes) - tap: Tap a screen coordinate (requires x, y) - swipe: Swipe between coordinates (requires x, y, x2, y2; optional duration_ms) - text: Input text into the focused field (requires text) @@ -62,7 +63,8 @@ func (t *AndroidTool) Parameters() map[string]interface{} { "action": map[string]interface{}{ "type": "string", "enum": []string{ - "search_apps", "app_info", "launch_app", "current_activity", + "search_apps", "app_info", "launch_app", + "screenshot", "get_ui_tree", "tap", "swipe", "text", "keyevent", "broadcast", "intent", }, @@ -125,6 +127,30 @@ func (t *AndroidTool) Parameters() map[string]interface{} { "type": "object", "description": "Extra key-value pairs for broadcast/intent", }, + "resource_id": map[string]interface{}{ + "type": "string", + "description": "View resource ID to start UI tree from (for get_ui_tree, e.g. com.example:id/button)", + }, + "index": map[string]interface{}{ + "type": "integer", + "description": "Which match to use when resource_id has multiple hits (for get_ui_tree, default 0)", + }, + "bounds_x": map[string]interface{}{ + "type": "number", + "description": "X coordinate to find the containing node (for get_ui_tree, alternative to resource_id)", + }, + "bounds_y": map[string]interface{}{ + "type": "number", + "description": "Y coordinate to find the containing node (for get_ui_tree, alternative to resource_id)", + }, + "max_depth": map[string]interface{}{ + "type": "integer", + "description": "Maximum traversal depth (for get_ui_tree, default 30)", + }, + "max_nodes": map[string]interface{}{ + "type": "integer", + "description": "Maximum number of nodes to output (for get_ui_tree, default 2000)", + }, }, "required": []string{"action"}, } @@ -181,9 +207,45 @@ func (t *AndroidTool) validateAndBuildParams(action string, args map[string]inte } params["package_name"] = pkg - case "current_activity": + case "screenshot": // No params needed + case "get_ui_tree": + // Start node selection: resource_id or bounds (mutually exclusive) + hasResourceID := false + hasBounds := false + if rid, ok := args["resource_id"].(string); ok && rid != "" { + params["resource_id"] = rid + hasResourceID = true + if idx, ok := toFloat64(args["index"]); ok { + idxInt := int(idx) + if idxInt < 0 { + return nil, fmt.Errorf("get_ui_tree: index must be non-negative, got %d", idxInt) + } + params["index"] = idxInt + } + } + if bx, bxOk := toFloat64(args["bounds_x"]); bxOk { + if by, byOk := toFloat64(args["bounds_y"]); byOk { + params["bounds_x"] = bx + params["bounds_y"] = by + hasBounds = true + } + } + if hasResourceID && hasBounds { + return nil, fmt.Errorf("get_ui_tree: cannot specify both resource_id and bounds_x/bounds_y") + } + if md, ok := toFloat64(args["max_depth"]); ok { + params["max_depth"] = int(md) + } + if mn, ok := toFloat64(args["max_nodes"]); ok { + mnInt := int(mn) + if mnInt < 1 { + return nil, fmt.Errorf("get_ui_tree: max_nodes must be at least 1, got %d", mnInt) + } + params["max_nodes"] = mnInt + } + case "tap": x, xOk := toFloat64(args["x"]) y, yOk := toFloat64(args["y"]) @@ -306,6 +368,14 @@ func (t *AndroidTool) sendAndWait(ctx context.Context, action string, params map ForLLM: "accessibility_required: The accessibility service is not enabled. The settings dialog has been shown to the user. Do not retry automatically - wait for the user to enable the service and try again.", } } + // Screenshot returns base64 JPEG data — wrap as multimodal result + if action == "screenshot" { + return &ToolResult{ + ForLLM: "Screenshot captured.", + Media: []string{"data:image/jpeg;base64," + content}, + Silent: true, + } + } return SilentResult(content) case <-time.After(androidToolTimeout): DeviceResponseWaiter.Cleanup(requestID) diff --git a/pkg/tools/result.go b/pkg/tools/result.go index b13055b1c..a3afe0159 100644 --- a/pkg/tools/result.go +++ b/pkg/tools/result.go @@ -27,6 +27,10 @@ type ToolResult struct { // When true, the tool will complete later and notify via callback. Async bool `json:"async"` + // Media contains base64 data URLs (e.g. "data:image/jpeg;base64,...") to + // pass multimodal content (screenshots, etc.) to the LLM. + Media []string `json:"media,omitempty"` + // Err is the underlying error (not JSON serialized). // Used for internal error handling and logging. Err error `json:"-"`