feat: replace current_activity with screenshot and get_ui_tree actions

Remove the current_activity action and add screenshot (returns base64
JPEG as multimodal tool result) and get_ui_tree (accessibility tree
with optional resource_id/bounds/index targeting and depth/node limits).

- Add Media field to ToolResult and propagate through message pipeline
- Extend HTTP provider to support multimodal content on tool messages
- Add input validation (mutual exclusivity, bounds, negative index)
- Fix existing bug: status.go used "list_apps" instead of "search_apps"

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

android/app/src/main/java/io/picoclaw/android/assistant/AssistantService.kt

@@ -117,6 +117,10 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner {
         toolRequestHandler = ToolRequestHandler(
             context = applicationContext,
             deviceController = deviceController,
+            screenshotSource = screenshotSource,
+            setOverlayVisibility = { visible ->
+                overlayView?.visibility = if (visible) View.VISIBLE else View.INVISIBLE
+            },
             onAccessibilityNeeded = { showAccessibilityGuide = true }
         )
         (connection as AssistantConnectionImpl).onToolRequest = { request ->

android/app/src/main/java/io/picoclaw/android/assistant/DeviceController.kt

@@ -54,14 +54,8 @@ class DeviceController {
         return service?.performGlobalAction(AccessibilityService.GLOBAL_ACTION_RECENTS) ?: false
     }
 
-    fun getCurrentPackage(): String? {
+    fun getRootNode(): AccessibilityNodeInfo? {
-        val svc = service ?: return null
+        return service?.rootInActiveWindow
-        val ownPackage = svc.packageName
-        // Skip our own overlay window and find the actual foreground app
-        return svc.windows
-            .filter { it.type == android.view.accessibility.AccessibilityWindowInfo.TYPE_APPLICATION }
-            .firstOrNull { it.root?.packageName?.toString() != ownPackage }
-            ?.root?.packageName?.toString()
     }
 
     suspend fun inputText(text: String): Boolean {

android/app/src/main/java/io/picoclaw/android/assistant/ToolRequestHandler.kt

@@ -4,21 +4,33 @@ import android.content.Context
 import android.content.Intent
 import android.content.pm.ApplicationInfo
 import android.content.pm.PackageManager
+import android.graphics.Bitmap
+import android.graphics.Rect
 import android.net.Uri
 import android.os.Build
+import android.util.Base64
 import android.util.Log
+import android.view.accessibility.AccessibilityNodeInfo
 import io.picoclaw.android.core.data.remote.dto.ToolRequest
 import io.picoclaw.android.core.data.remote.dto.ToolResponse
+import io.picoclaw.android.feature.chat.voice.ScreenshotSource
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.withContext
 import kotlinx.serialization.json.JsonObject
 import kotlinx.serialization.json.JsonPrimitive
 import kotlinx.serialization.json.contentOrNull
 import kotlinx.serialization.json.doubleOrNull
 import kotlinx.serialization.json.jsonPrimitive
+import kotlinx.serialization.json.intOrNull
 import kotlinx.serialization.json.longOrNull
+import java.io.ByteArrayOutputStream
 
 class ToolRequestHandler(
     private val context: Context,
     private val deviceController: DeviceController,
+    private val screenshotSource: ScreenshotSource,
+    private val setOverlayVisibility: (Boolean) -> Unit,
     private val onAccessibilityNeeded: () -> Unit
 ) {
 
@@ -28,7 +40,8 @@ class ToolRequestHandler(
                 "search_apps" -> handleSearchApps(request)
                 "app_info" -> handleAppInfo(request)
                 "launch_app" -> handleLaunchApp(request)
-                "current_activity" -> handleCurrentActivity(request)
+                "screenshot" -> handleScreenshot(request)
+                "get_ui_tree" -> handleGetUiTree(request)
                 "tap" -> handleTap(request)
                 "swipe" -> handleSwipe(request)
                 "text" -> handleText(request)
@@ -140,13 +153,144 @@ class ToolRequestHandler(
         return ToolResponse(request.requestId, true, result = "Launched $packageName")
     }
 
-    private suspend fun handleCurrentActivity(request: ToolRequest): ToolResponse {
+    private suspend fun handleScreenshot(request: ToolRequest): ToolResponse {
         requireAccessibility(request)?.let { return it }
 
-        val pkg = deviceController.getCurrentPackage()
+        return try {
-            ?: return ToolResponse(request.requestId, false, error = "Could not get current activity")
+            withContext(Dispatchers.Main) { setOverlayVisibility(false) }
+            delay(150)
+            val bitmap = screenshotSource.takeScreenshot()
+                ?: return ToolResponse(request.requestId, false, error = "Screenshot capture failed")
+            try {
+                val base64 = withContext(Dispatchers.IO) {
+                    val stream = ByteArrayOutputStream()
+                    bitmap.compress(Bitmap.CompressFormat.JPEG, 80, stream)
+                    Base64.encodeToString(stream.toByteArray(), Base64.NO_WRAP)
+                }
+                ToolResponse(request.requestId, true, result = base64)
+            } finally {
+                bitmap.recycle()
+            }
+        } finally {
+            withContext(Dispatchers.Main) { setOverlayVisibility(true) }
+        }
+    }
 
-        return ToolResponse(request.requestId, true, result = "Current package: $pkg")
+    private suspend fun handleGetUiTree(request: ToolRequest): ToolResponse {
+        requireAccessibility(request)?.let { return it }
+
+        val resourceId = request.params?.get("resource_id")?.jsonPrimitive?.contentOrNull
+        val index = request.params?.get("index")?.jsonPrimitive?.intOrNull ?: 0
+        val boundsX = request.params?.get("bounds_x")?.jsonPrimitive?.doubleOrNull
+        val boundsY = request.params?.get("bounds_y")?.jsonPrimitive?.doubleOrNull
+        val maxDepth = request.params?.get("max_depth")?.jsonPrimitive?.intOrNull ?: 30
+        val maxNodes = request.params?.get("max_nodes")?.jsonPrimitive?.intOrNull ?: 2000
+
+        return try {
+            withContext(Dispatchers.Main) { setOverlayVisibility(false) }
+            delay(150)
+            val root = deviceController.getRootNode()
+                ?: return ToolResponse(request.requestId, false, error = "Could not get UI tree")
+            try {
+                val startNode = resolveStartNode(root, resourceId, index, boundsX, boundsY)
+                    ?: return ToolResponse(request.requestId, false, error = buildString {
+                        if (resourceId != null) append("No node found with resource_id=$resourceId (index=$index)")
+                        else append("No node found at bounds ($boundsX, $boundsY)")
+                    })
+                try {
+                    val sb = StringBuilder()
+                    val nodeCount = intArrayOf(0)
+                    dumpNode(startNode, sb, 0, maxDepth, maxNodes, nodeCount)
+                    if (nodeCount[0] >= maxNodes) {
+                        sb.appendLine("[truncated: max_nodes=$maxNodes reached]")
+                    }
+                    ToolResponse(request.requestId, true, result = sb.toString())
+                } finally {
+                    if (startNode !== root) startNode.recycle()
+                }
+            } finally {
+                root.recycle()
+            }
+        } finally {
+            withContext(Dispatchers.Main) { setOverlayVisibility(true) }
+        }
+    }
+
+    private fun resolveStartNode(
+        root: AccessibilityNodeInfo,
+        resourceId: String?,
+        index: Int,
+        boundsX: Double?,
+        boundsY: Double?
+    ): AccessibilityNodeInfo? {
+        if (resourceId != null) {
+            val matches = root.findAccessibilityNodeInfosByViewId(resourceId)
+            if (matches.isNullOrEmpty()) return null
+            val target = matches.getOrNull(index)
+            // Recycle unused matches
+            for ((i, node) in matches.withIndex()) {
+                if (i != index) node.recycle()
+            }
+            return target
+        }
+        if (boundsX != null && boundsY != null) {
+            return findNodeAtPoint(root, boundsX.toInt(), boundsY.toInt())
+        }
+        return root
+    }
+
+    private fun findNodeAtPoint(node: AccessibilityNodeInfo, x: Int, y: Int): AccessibilityNodeInfo? {
+        val bounds = Rect()
+        node.getBoundsInScreen(bounds)
+        if (!bounds.contains(x, y)) return null
+        // Find the deepest (smallest) child that contains the point
+        for (i in 0 until node.childCount) {
+            val child = node.getChild(i) ?: continue
+            val found = findNodeAtPoint(child, x, y)
+            if (found != null) return found
+            child.recycle()
+        }
+        return node
+    }
+
+    private fun dumpNode(
+        node: AccessibilityNodeInfo,
+        sb: StringBuilder,
+        depth: Int,
+        maxDepth: Int,
+        maxNodes: Int,
+        nodeCount: IntArray
+    ) {
+        if (nodeCount[0] >= maxNodes) return
+        nodeCount[0]++
+        val indent = "  ".repeat(depth)
+        val bounds = Rect()
+        node.getBoundsInScreen(bounds)
+        sb.appendLine(
+            "${indent}[${node.className}] " +
+                "text=${node.text ?: ""} " +
+                "desc=${node.contentDescription ?: ""} " +
+                "bounds=${bounds} " +
+                "clickable=${node.isClickable} " +
+                "enabled=${node.isEnabled} " +
+                "visible=${node.isVisibleToUser} " +
+                "id=${node.viewIdResourceName ?: ""}"
+        )
+        if (depth >= maxDepth) {
+            if (node.childCount > 0) {
+                sb.appendLine("${indent}  [truncated: ${node.childCount} children at depth $depth]")
+            }
+            return
+        }
+        for (i in 0 until node.childCount) {
+            if (nodeCount[0] >= maxNodes) return
+            val child = node.getChild(i) ?: continue
+            try {
+                dumpNode(child, sb, depth + 1, maxDepth, maxNodes, nodeCount)
+            } finally {
+                child.recycle()
+            }
+        }
     }
 
     private suspend fun handleTap(request: ToolRequest): ToolResponse {

pkg/agent/loop.go

@@ -957,6 +957,7 @@ func (al *AgentLoop) runLLMIteration(ctx context.Context, messages []providers.M
 			toolResultMsg := providers.Message{
 				Role:       "tool",
 				Content:    contentForLLM,
+				Media:      toolResult.Media,
 				ToolCallID: tc.ID,
 			}
 			messages = append(messages, toolResultMsg)

pkg/agent/status.go

@@ -120,8 +120,8 @@ func cronStatusLabel(args map[string]interface{}) string {
 
 func androidStatusLabel(args map[string]interface{}) string {
 	switch strArg(args, "action") {
-	case "list_apps":
+	case "search_apps":
-		return "アプリ一覧取得中..."
+		return "アプリ検索中..."
 	case "app_info":
 		if p := strArg(args, "package_name"); p != "" {
 			return fmt.Sprintf("アプリ情報取得中...（%s）", truncLabel(p, 25))
@@ -132,8 +132,10 @@ func androidStatusLabel(args map[string]interface{}) string {
 			return fmt.Sprintf("アプリ起動中...（%s）", truncLabel(p, 25))
 		}
 		return "アプリ起動中..."
-	case "current_activity":
+	case "screenshot":
-		return "画面情報取得中..."
+		return "スクリーンショット撮影中..."
+	case "get_ui_tree":
+		return "UI要素取得中..."
 	case "tap":
 		return "タップ中..."
 	case "swipe":

pkg/providers/http_provider.go

@@ -134,8 +134,8 @@ func (p *HTTPProvider) buildAPIMessages(messages []Message) []map[string]interfa
 			"role": msg.Role,
 		}
 
-		// Only user messages with media get the array-style content
+		// Messages with media get the array-style content (user or tool)
-		if msg.Role == "user" && len(msg.Media) > 0 {
+		if len(msg.Media) > 0 && (msg.Role == "user" || msg.Role == "tool") {
 			parts := make([]map[string]interface{}, 0, 1+len(msg.Media))
 			if msg.Content != "" {
 				parts = append(parts, map[string]interface{}{

pkg/tools/android.go

@@ -45,7 +45,8 @@ func (t *AndroidTool) Description() string {
 - search_apps: Search installed apps by name or package name (requires query)
 - app_info: Get app details (requires package_name)
 - launch_app: Launch an app (requires package_name)
-- current_activity: Get the currently active app/window
+- screenshot: Capture a screenshot of the current screen (no params)
+- get_ui_tree: Get the accessibility UI tree (optional: resource_id, index, bounds_x/bounds_y, max_depth, max_nodes)
 - tap: Tap a screen coordinate (requires x, y)
 - swipe: Swipe between coordinates (requires x, y, x2, y2; optional duration_ms)
 - text: Input text into the focused field (requires text)
@@ -62,7 +63,8 @@ func (t *AndroidTool) Parameters() map[string]interface{} {
 			"action": map[string]interface{}{
 				"type": "string",
 				"enum": []string{
-					"search_apps", "app_info", "launch_app", "current_activity",
+					"search_apps", "app_info", "launch_app",
+					"screenshot", "get_ui_tree",
 					"tap", "swipe", "text", "keyevent",
 					"broadcast", "intent",
 				},
@@ -125,6 +127,30 @@ func (t *AndroidTool) Parameters() map[string]interface{} {
 				"type":        "object",
 				"description": "Extra key-value pairs for broadcast/intent",
 			},
+			"resource_id": map[string]interface{}{
+				"type":        "string",
+				"description": "View resource ID to start UI tree from (for get_ui_tree, e.g. com.example:id/button)",
+			},
+			"index": map[string]interface{}{
+				"type":        "integer",
+				"description": "Which match to use when resource_id has multiple hits (for get_ui_tree, default 0)",
+			},
+			"bounds_x": map[string]interface{}{
+				"type":        "number",
+				"description": "X coordinate to find the containing node (for get_ui_tree, alternative to resource_id)",
+			},
+			"bounds_y": map[string]interface{}{
+				"type":        "number",
+				"description": "Y coordinate to find the containing node (for get_ui_tree, alternative to resource_id)",
+			},
+			"max_depth": map[string]interface{}{
+				"type":        "integer",
+				"description": "Maximum traversal depth (for get_ui_tree, default 30)",
+			},
+			"max_nodes": map[string]interface{}{
+				"type":        "integer",
+				"description": "Maximum number of nodes to output (for get_ui_tree, default 2000)",
+			},
 		},
 		"required": []string{"action"},
 	}
@@ -181,9 +207,45 @@ func (t *AndroidTool) validateAndBuildParams(action string, args map[string]inte
 		}
 		params["package_name"] = pkg
 
-	case "current_activity":
+	case "screenshot":
 		// No params needed
 
+	case "get_ui_tree":
+		// Start node selection: resource_id or bounds (mutually exclusive)
+		hasResourceID := false
+		hasBounds := false
+		if rid, ok := args["resource_id"].(string); ok && rid != "" {
+			params["resource_id"] = rid
+			hasResourceID = true
+			if idx, ok := toFloat64(args["index"]); ok {
+				idxInt := int(idx)
+				if idxInt < 0 {
+					return nil, fmt.Errorf("get_ui_tree: index must be non-negative, got %d", idxInt)
+				}
+				params["index"] = idxInt
+			}
+		}
+		if bx, bxOk := toFloat64(args["bounds_x"]); bxOk {
+			if by, byOk := toFloat64(args["bounds_y"]); byOk {
+				params["bounds_x"] = bx
+				params["bounds_y"] = by
+				hasBounds = true
+			}
+		}
+		if hasResourceID && hasBounds {
+			return nil, fmt.Errorf("get_ui_tree: cannot specify both resource_id and bounds_x/bounds_y")
+		}
+		if md, ok := toFloat64(args["max_depth"]); ok {
+			params["max_depth"] = int(md)
+		}
+		if mn, ok := toFloat64(args["max_nodes"]); ok {
+			mnInt := int(mn)
+			if mnInt < 1 {
+				return nil, fmt.Errorf("get_ui_tree: max_nodes must be at least 1, got %d", mnInt)
+			}
+			params["max_nodes"] = mnInt
+		}
+
 	case "tap":
 		x, xOk := toFloat64(args["x"])
 		y, yOk := toFloat64(args["y"])
@@ -306,6 +368,14 @@ func (t *AndroidTool) sendAndWait(ctx context.Context, action string, params map
 				ForLLM:  "accessibility_required: The accessibility service is not enabled. The settings dialog has been shown to the user. Do not retry automatically - wait for the user to enable the service and try again.",
 			}
 		}
+		// Screenshot returns base64 JPEG data — wrap as multimodal result
+		if action == "screenshot" {
+			return &ToolResult{
+				ForLLM: "Screenshot captured.",
+				Media:  []string{"data:image/jpeg;base64," + content},
+				Silent: true,
+			}
+		}
 		return SilentResult(content)
 	case <-time.After(androidToolTimeout):
 		DeviceResponseWaiter.Cleanup(requestID)

pkg/tools/result.go

@@ -27,6 +27,10 @@ type ToolResult struct {
 	// When true, the tool will complete later and notify via callback.
 	Async bool `json:"async"`
 
+	// Media contains base64 data URLs (e.g. "data:image/jpeg;base64,...") to
+	// pass multimodal content (screenshots, etc.) to the LLM.
+	Media []string `json:"media,omitempty"`
+
 	// Err is the underlying error (not JSON serialized).
 	// Used for internal error handling and logging.
 	Err error `json:"-"`