feat: replace current_activity with screenshot and get_ui_tree actions
Remove the current_activity action and add screenshot (returns base64 JPEG as multimodal tool result) and get_ui_tree (accessibility tree with optional resource_id/bounds/index targeting and depth/node limits). - Add Media field to ToolResult and propagate through message pipeline - Extend HTTP provider to support multimodal content on tool messages - Add input validation (mutual exclusivity, bounds, negative index) - Fix existing bug: status.go used "list_apps" instead of "search_apps" Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Kohei
parent
aa83f448f9
commit
45adabf82c
8 changed files with 241 additions and 22 deletions
|
|
@ -117,6 +117,10 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner {
|
|||
toolRequestHandler = ToolRequestHandler(
|
||||
context = applicationContext,
|
||||
deviceController = deviceController,
|
||||
screenshotSource = screenshotSource,
|
||||
setOverlayVisibility = { visible ->
|
||||
overlayView?.visibility = if (visible) View.VISIBLE else View.INVISIBLE
|
||||
},
|
||||
onAccessibilityNeeded = { showAccessibilityGuide = true }
|
||||
)
|
||||
(connection as AssistantConnectionImpl).onToolRequest = { request ->
|
||||
|
|
|
|||
|
|
@ -54,14 +54,8 @@ class DeviceController {
|
|||
return service?.performGlobalAction(AccessibilityService.GLOBAL_ACTION_RECENTS) ?: false
|
||||
}
|
||||
|
||||
fun getCurrentPackage(): String? {
|
||||
val svc = service ?: return null
|
||||
val ownPackage = svc.packageName
|
||||
// Skip our own overlay window and find the actual foreground app
|
||||
return svc.windows
|
||||
.filter { it.type == android.view.accessibility.AccessibilityWindowInfo.TYPE_APPLICATION }
|
||||
.firstOrNull { it.root?.packageName?.toString() != ownPackage }
|
||||
?.root?.packageName?.toString()
|
||||
fun getRootNode(): AccessibilityNodeInfo? {
|
||||
return service?.rootInActiveWindow
|
||||
}
|
||||
|
||||
suspend fun inputText(text: String): Boolean {
|
||||
|
|
|
|||
|
|
@ -4,21 +4,33 @@ import android.content.Context
|
|||
import android.content.Intent
|
||||
import android.content.pm.ApplicationInfo
|
||||
import android.content.pm.PackageManager
|
||||
import android.graphics.Bitmap
|
||||
import android.graphics.Rect
|
||||
import android.net.Uri
|
||||
import android.os.Build
|
||||
import android.util.Base64
|
||||
import android.util.Log
|
||||
import android.view.accessibility.AccessibilityNodeInfo
|
||||
import io.picoclaw.android.core.data.remote.dto.ToolRequest
|
||||
import io.picoclaw.android.core.data.remote.dto.ToolResponse
|
||||
import io.picoclaw.android.feature.chat.voice.ScreenshotSource
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.contentOrNull
|
||||
import kotlinx.serialization.json.doubleOrNull
|
||||
import kotlinx.serialization.json.jsonPrimitive
|
||||
import kotlinx.serialization.json.intOrNull
|
||||
import kotlinx.serialization.json.longOrNull
|
||||
import java.io.ByteArrayOutputStream
|
||||
|
||||
class ToolRequestHandler(
|
||||
private val context: Context,
|
||||
private val deviceController: DeviceController,
|
||||
private val screenshotSource: ScreenshotSource,
|
||||
private val setOverlayVisibility: (Boolean) -> Unit,
|
||||
private val onAccessibilityNeeded: () -> Unit
|
||||
) {
|
||||
|
||||
|
|
@ -28,7 +40,8 @@ class ToolRequestHandler(
|
|||
"search_apps" -> handleSearchApps(request)
|
||||
"app_info" -> handleAppInfo(request)
|
||||
"launch_app" -> handleLaunchApp(request)
|
||||
"current_activity" -> handleCurrentActivity(request)
|
||||
"screenshot" -> handleScreenshot(request)
|
||||
"get_ui_tree" -> handleGetUiTree(request)
|
||||
"tap" -> handleTap(request)
|
||||
"swipe" -> handleSwipe(request)
|
||||
"text" -> handleText(request)
|
||||
|
|
@ -140,13 +153,144 @@ class ToolRequestHandler(
|
|||
return ToolResponse(request.requestId, true, result = "Launched $packageName")
|
||||
}
|
||||
|
||||
private suspend fun handleCurrentActivity(request: ToolRequest): ToolResponse {
|
||||
private suspend fun handleScreenshot(request: ToolRequest): ToolResponse {
|
||||
requireAccessibility(request)?.let { return it }
|
||||
|
||||
val pkg = deviceController.getCurrentPackage()
|
||||
?: return ToolResponse(request.requestId, false, error = "Could not get current activity")
|
||||
return try {
|
||||
withContext(Dispatchers.Main) { setOverlayVisibility(false) }
|
||||
delay(150)
|
||||
val bitmap = screenshotSource.takeScreenshot()
|
||||
?: return ToolResponse(request.requestId, false, error = "Screenshot capture failed")
|
||||
try {
|
||||
val base64 = withContext(Dispatchers.IO) {
|
||||
val stream = ByteArrayOutputStream()
|
||||
bitmap.compress(Bitmap.CompressFormat.JPEG, 80, stream)
|
||||
Base64.encodeToString(stream.toByteArray(), Base64.NO_WRAP)
|
||||
}
|
||||
ToolResponse(request.requestId, true, result = base64)
|
||||
} finally {
|
||||
bitmap.recycle()
|
||||
}
|
||||
} finally {
|
||||
withContext(Dispatchers.Main) { setOverlayVisibility(true) }
|
||||
}
|
||||
}
|
||||
|
||||
return ToolResponse(request.requestId, true, result = "Current package: $pkg")
|
||||
private suspend fun handleGetUiTree(request: ToolRequest): ToolResponse {
|
||||
requireAccessibility(request)?.let { return it }
|
||||
|
||||
val resourceId = request.params?.get("resource_id")?.jsonPrimitive?.contentOrNull
|
||||
val index = request.params?.get("index")?.jsonPrimitive?.intOrNull ?: 0
|
||||
val boundsX = request.params?.get("bounds_x")?.jsonPrimitive?.doubleOrNull
|
||||
val boundsY = request.params?.get("bounds_y")?.jsonPrimitive?.doubleOrNull
|
||||
val maxDepth = request.params?.get("max_depth")?.jsonPrimitive?.intOrNull ?: 30
|
||||
val maxNodes = request.params?.get("max_nodes")?.jsonPrimitive?.intOrNull ?: 2000
|
||||
|
||||
return try {
|
||||
withContext(Dispatchers.Main) { setOverlayVisibility(false) }
|
||||
delay(150)
|
||||
val root = deviceController.getRootNode()
|
||||
?: return ToolResponse(request.requestId, false, error = "Could not get UI tree")
|
||||
try {
|
||||
val startNode = resolveStartNode(root, resourceId, index, boundsX, boundsY)
|
||||
?: return ToolResponse(request.requestId, false, error = buildString {
|
||||
if (resourceId != null) append("No node found with resource_id=$resourceId (index=$index)")
|
||||
else append("No node found at bounds ($boundsX, $boundsY)")
|
||||
})
|
||||
try {
|
||||
val sb = StringBuilder()
|
||||
val nodeCount = intArrayOf(0)
|
||||
dumpNode(startNode, sb, 0, maxDepth, maxNodes, nodeCount)
|
||||
if (nodeCount[0] >= maxNodes) {
|
||||
sb.appendLine("[truncated: max_nodes=$maxNodes reached]")
|
||||
}
|
||||
ToolResponse(request.requestId, true, result = sb.toString())
|
||||
} finally {
|
||||
if (startNode !== root) startNode.recycle()
|
||||
}
|
||||
} finally {
|
||||
root.recycle()
|
||||
}
|
||||
} finally {
|
||||
withContext(Dispatchers.Main) { setOverlayVisibility(true) }
|
||||
}
|
||||
}
|
||||
|
||||
private fun resolveStartNode(
|
||||
root: AccessibilityNodeInfo,
|
||||
resourceId: String?,
|
||||
index: Int,
|
||||
boundsX: Double?,
|
||||
boundsY: Double?
|
||||
): AccessibilityNodeInfo? {
|
||||
if (resourceId != null) {
|
||||
val matches = root.findAccessibilityNodeInfosByViewId(resourceId)
|
||||
if (matches.isNullOrEmpty()) return null
|
||||
val target = matches.getOrNull(index)
|
||||
// Recycle unused matches
|
||||
for ((i, node) in matches.withIndex()) {
|
||||
if (i != index) node.recycle()
|
||||
}
|
||||
return target
|
||||
}
|
||||
if (boundsX != null && boundsY != null) {
|
||||
return findNodeAtPoint(root, boundsX.toInt(), boundsY.toInt())
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
private fun findNodeAtPoint(node: AccessibilityNodeInfo, x: Int, y: Int): AccessibilityNodeInfo? {
|
||||
val bounds = Rect()
|
||||
node.getBoundsInScreen(bounds)
|
||||
if (!bounds.contains(x, y)) return null
|
||||
// Find the deepest (smallest) child that contains the point
|
||||
for (i in 0 until node.childCount) {
|
||||
val child = node.getChild(i) ?: continue
|
||||
val found = findNodeAtPoint(child, x, y)
|
||||
if (found != null) return found
|
||||
child.recycle()
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
private fun dumpNode(
|
||||
node: AccessibilityNodeInfo,
|
||||
sb: StringBuilder,
|
||||
depth: Int,
|
||||
maxDepth: Int,
|
||||
maxNodes: Int,
|
||||
nodeCount: IntArray
|
||||
) {
|
||||
if (nodeCount[0] >= maxNodes) return
|
||||
nodeCount[0]++
|
||||
val indent = " ".repeat(depth)
|
||||
val bounds = Rect()
|
||||
node.getBoundsInScreen(bounds)
|
||||
sb.appendLine(
|
||||
"${indent}[${node.className}] " +
|
||||
"text=${node.text ?: ""} " +
|
||||
"desc=${node.contentDescription ?: ""} " +
|
||||
"bounds=${bounds} " +
|
||||
"clickable=${node.isClickable} " +
|
||||
"enabled=${node.isEnabled} " +
|
||||
"visible=${node.isVisibleToUser} " +
|
||||
"id=${node.viewIdResourceName ?: ""}"
|
||||
)
|
||||
if (depth >= maxDepth) {
|
||||
if (node.childCount > 0) {
|
||||
sb.appendLine("${indent} [truncated: ${node.childCount} children at depth $depth]")
|
||||
}
|
||||
return
|
||||
}
|
||||
for (i in 0 until node.childCount) {
|
||||
if (nodeCount[0] >= maxNodes) return
|
||||
val child = node.getChild(i) ?: continue
|
||||
try {
|
||||
dumpNode(child, sb, depth + 1, maxDepth, maxNodes, nodeCount)
|
||||
} finally {
|
||||
child.recycle()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun handleTap(request: ToolRequest): ToolResponse {
|
||||
|
|
|
|||
|
|
@ -957,6 +957,7 @@ func (al *AgentLoop) runLLMIteration(ctx context.Context, messages []providers.M
|
|||
toolResultMsg := providers.Message{
|
||||
Role: "tool",
|
||||
Content: contentForLLM,
|
||||
Media: toolResult.Media,
|
||||
ToolCallID: tc.ID,
|
||||
}
|
||||
messages = append(messages, toolResultMsg)
|
||||
|
|
|
|||
|
|
@ -120,8 +120,8 @@ func cronStatusLabel(args map[string]interface{}) string {
|
|||
|
||||
func androidStatusLabel(args map[string]interface{}) string {
|
||||
switch strArg(args, "action") {
|
||||
case "list_apps":
|
||||
return "アプリ一覧取得中..."
|
||||
case "search_apps":
|
||||
return "アプリ検索中..."
|
||||
case "app_info":
|
||||
if p := strArg(args, "package_name"); p != "" {
|
||||
return fmt.Sprintf("アプリ情報取得中...（%s）", truncLabel(p, 25))
|
||||
|
|
@ -132,8 +132,10 @@ func androidStatusLabel(args map[string]interface{}) string {
|
|||
return fmt.Sprintf("アプリ起動中...（%s）", truncLabel(p, 25))
|
||||
}
|
||||
return "アプリ起動中..."
|
||||
case "current_activity":
|
||||
return "画面情報取得中..."
|
||||
case "screenshot":
|
||||
return "スクリーンショット撮影中..."
|
||||
case "get_ui_tree":
|
||||
return "UI要素取得中..."
|
||||
case "tap":
|
||||
return "タップ中..."
|
||||
case "swipe":
|
||||
|
|
|
|||
|
|
@ -134,8 +134,8 @@ func (p *HTTPProvider) buildAPIMessages(messages []Message) []map[string]interfa
|
|||
"role": msg.Role,
|
||||
}
|
||||
|
||||
// Only user messages with media get the array-style content
|
||||
if msg.Role == "user" && len(msg.Media) > 0 {
|
||||
// Messages with media get the array-style content (user or tool)
|
||||
if len(msg.Media) > 0 && (msg.Role == "user" || msg.Role == "tool") {
|
||||
parts := make([]map[string]interface{}, 0, 1+len(msg.Media))
|
||||
if msg.Content != "" {
|
||||
parts = append(parts, map[string]interface{}{
|
||||
|
|
|
|||
|
|
@ -45,7 +45,8 @@ func (t *AndroidTool) Description() string {
|
|||
- search_apps: Search installed apps by name or package name (requires query)
|
||||
- app_info: Get app details (requires package_name)
|
||||
- launch_app: Launch an app (requires package_name)
|
||||
- current_activity: Get the currently active app/window
|
||||
- screenshot: Capture a screenshot of the current screen (no params)
|
||||
- get_ui_tree: Get the accessibility UI tree (optional: resource_id, index, bounds_x/bounds_y, max_depth, max_nodes)
|
||||
- tap: Tap a screen coordinate (requires x, y)
|
||||
- swipe: Swipe between coordinates (requires x, y, x2, y2; optional duration_ms)
|
||||
- text: Input text into the focused field (requires text)
|
||||
|
|
@ -62,7 +63,8 @@ func (t *AndroidTool) Parameters() map[string]interface{} {
|
|||
"action": map[string]interface{}{
|
||||
"type": "string",
|
||||
"enum": []string{
|
||||
"search_apps", "app_info", "launch_app", "current_activity",
|
||||
"search_apps", "app_info", "launch_app",
|
||||
"screenshot", "get_ui_tree",
|
||||
"tap", "swipe", "text", "keyevent",
|
||||
"broadcast", "intent",
|
||||
},
|
||||
|
|
@ -125,6 +127,30 @@ func (t *AndroidTool) Parameters() map[string]interface{} {
|
|||
"type": "object",
|
||||
"description": "Extra key-value pairs for broadcast/intent",
|
||||
},
|
||||
"resource_id": map[string]interface{}{
|
||||
"type": "string",
|
||||
"description": "View resource ID to start UI tree from (for get_ui_tree, e.g. com.example:id/button)",
|
||||
},
|
||||
"index": map[string]interface{}{
|
||||
"type": "integer",
|
||||
"description": "Which match to use when resource_id has multiple hits (for get_ui_tree, default 0)",
|
||||
},
|
||||
"bounds_x": map[string]interface{}{
|
||||
"type": "number",
|
||||
"description": "X coordinate to find the containing node (for get_ui_tree, alternative to resource_id)",
|
||||
},
|
||||
"bounds_y": map[string]interface{}{
|
||||
"type": "number",
|
||||
"description": "Y coordinate to find the containing node (for get_ui_tree, alternative to resource_id)",
|
||||
},
|
||||
"max_depth": map[string]interface{}{
|
||||
"type": "integer",
|
||||
"description": "Maximum traversal depth (for get_ui_tree, default 30)",
|
||||
},
|
||||
"max_nodes": map[string]interface{}{
|
||||
"type": "integer",
|
||||
"description": "Maximum number of nodes to output (for get_ui_tree, default 2000)",
|
||||
},
|
||||
},
|
||||
"required": []string{"action"},
|
||||
}
|
||||
|
|
@ -181,9 +207,45 @@ func (t *AndroidTool) validateAndBuildParams(action string, args map[string]inte
|
|||
}
|
||||
params["package_name"] = pkg
|
||||
|
||||
case "current_activity":
|
||||
case "screenshot":
|
||||
// No params needed
|
||||
|
||||
case "get_ui_tree":
|
||||
// Start node selection: resource_id or bounds (mutually exclusive)
|
||||
hasResourceID := false
|
||||
hasBounds := false
|
||||
if rid, ok := args["resource_id"].(string); ok && rid != "" {
|
||||
params["resource_id"] = rid
|
||||
hasResourceID = true
|
||||
if idx, ok := toFloat64(args["index"]); ok {
|
||||
idxInt := int(idx)
|
||||
if idxInt < 0 {
|
||||
return nil, fmt.Errorf("get_ui_tree: index must be non-negative, got %d", idxInt)
|
||||
}
|
||||
params["index"] = idxInt
|
||||
}
|
||||
}
|
||||
if bx, bxOk := toFloat64(args["bounds_x"]); bxOk {
|
||||
if by, byOk := toFloat64(args["bounds_y"]); byOk {
|
||||
params["bounds_x"] = bx
|
||||
params["bounds_y"] = by
|
||||
hasBounds = true
|
||||
}
|
||||
}
|
||||
if hasResourceID && hasBounds {
|
||||
return nil, fmt.Errorf("get_ui_tree: cannot specify both resource_id and bounds_x/bounds_y")
|
||||
}
|
||||
if md, ok := toFloat64(args["max_depth"]); ok {
|
||||
params["max_depth"] = int(md)
|
||||
}
|
||||
if mn, ok := toFloat64(args["max_nodes"]); ok {
|
||||
mnInt := int(mn)
|
||||
if mnInt < 1 {
|
||||
return nil, fmt.Errorf("get_ui_tree: max_nodes must be at least 1, got %d", mnInt)
|
||||
}
|
||||
params["max_nodes"] = mnInt
|
||||
}
|
||||
|
||||
case "tap":
|
||||
x, xOk := toFloat64(args["x"])
|
||||
y, yOk := toFloat64(args["y"])
|
||||
|
|
@ -306,6 +368,14 @@ func (t *AndroidTool) sendAndWait(ctx context.Context, action string, params map
|
|||
ForLLM: "accessibility_required: The accessibility service is not enabled. The settings dialog has been shown to the user. Do not retry automatically - wait for the user to enable the service and try again.",
|
||||
}
|
||||
}
|
||||
// Screenshot returns base64 JPEG data — wrap as multimodal result
|
||||
if action == "screenshot" {
|
||||
return &ToolResult{
|
||||
ForLLM: "Screenshot captured.",
|
||||
Media: []string{"data:image/jpeg;base64," + content},
|
||||
Silent: true,
|
||||
}
|
||||
}
|
||||
return SilentResult(content)
|
||||
case <-time.After(androidToolTimeout):
|
||||
DeviceResponseWaiter.Cleanup(requestID)
|
||||
|
|
|
|||
|
|
@ -27,6 +27,10 @@ type ToolResult struct {
|
|||
// When true, the tool will complete later and notify via callback.
|
||||
Async bool `json:"async"`
|
||||
|
||||
// Media contains base64 data URLs (e.g. "data:image/jpeg;base64,...") to
|
||||
// pass multimodal content (screenshots, etc.) to the LLM.
|
||||
Media []string `json:"media,omitempty"`
|
||||
|
||||
// Err is the underlying error (not JSON serialized).
|
||||
// Used for internal error handling and logging.
|
||||
Err error `json:"-"`
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue