From 2c5d8fbf5dc9ad06bc5608893536d5123623a283 Mon Sep 17 00:00:00 2001 From: KoheiYamashita Date: Sun, 22 Feb 2026 19:43:25 +0900 Subject: [PATCH] fix: hide overlay before tap/swipe/text to prevent touch interception The overlay window could intercept dispatchGesture touches in its 350dp bottom area. Now all UI actions (tap, swipe, text, screenshot, get_ui_tree) hide the overlay with both View.INVISIBLE and FLAG_NOT_TOUCHABLE via a shared withOverlayHidden helper. Co-Authored-By: Claude Opus 4.6 --- .../android/assistant/AssistantService.kt | 19 +++-- .../android/assistant/ToolRequestHandler.kt | 70 +++++++++++-------- 2 files changed, 54 insertions(+), 35 deletions(-) diff --git a/android/app/src/main/java/io/picoclaw/android/assistant/AssistantService.kt b/android/app/src/main/java/io/picoclaw/android/assistant/AssistantService.kt index f2a12dedf..2b31e67f7 100644 --- a/android/app/src/main/java/io/picoclaw/android/assistant/AssistantService.kt +++ b/android/app/src/main/java/io/picoclaw/android/assistant/AssistantService.kt @@ -118,9 +118,7 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner { context = applicationContext, deviceController = deviceController, screenshotSource = screenshotSource, - setOverlayVisibility = { visible -> - overlayView?.visibility = if (visible) View.VISIBLE else View.INVISIBLE - }, + setOverlayVisibility = { visible -> setOverlayVisible(visible) }, onAccessibilityNeeded = { showAccessibilityGuide = true } ) (connection as AssistantConnectionImpl).onToolRequest = { request -> @@ -136,7 +134,7 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner { ttsWrapper = TextToSpeechWrapper(this, ttsSettingsRepo.ttsConfig) cameraCaptureManager = CameraCaptureManager(this) screenCaptureManager = ScreenCaptureManager(screenshotSource, applicationContext) { visible -> - overlayView?.visibility = if (visible) View.VISIBLE else View.INVISIBLE + setOverlayVisible(visible) } assistantManager = AssistantManager( @@ -226,6 +224,19 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner { stopSelf() } + private fun setOverlayVisible(visible: Boolean) { + val view = overlayView ?: return + val lp = view.layoutParams as? WindowManager.LayoutParams ?: return + if (visible) { + lp.flags = lp.flags and WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE.inv() + view.visibility = View.VISIBLE + } else { + lp.flags = lp.flags or WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE + view.visibility = View.INVISIBLE + } + windowManager.updateViewLayout(view, lp) + } + private fun addOverlay() { if (overlayView != null) return diff --git a/android/app/src/main/java/io/picoclaw/android/assistant/ToolRequestHandler.kt b/android/app/src/main/java/io/picoclaw/android/assistant/ToolRequestHandler.kt index cfde0f0bf..a9a578842 100644 --- a/android/app/src/main/java/io/picoclaw/android/assistant/ToolRequestHandler.kt +++ b/android/app/src/main/java/io/picoclaw/android/assistant/ToolRequestHandler.kt @@ -76,6 +76,16 @@ class ToolRequestHandler( return null } + private suspend fun withOverlayHidden(block: suspend () -> T): T { + return try { + withContext(Dispatchers.Main) { setOverlayVisibility(false) } + delay(150) + block() + } finally { + withContext(Dispatchers.Main) { setOverlayVisibility(true) } + } + } + private fun handleSearchApps(request: ToolRequest): ToolResponse { val query = request.params?.get("query")?.jsonPrimitive?.contentOrNull ?: return ToolResponse(request.requestId, false, error = "query required") @@ -156,11 +166,9 @@ class ToolRequestHandler( private suspend fun handleScreenshot(request: ToolRequest): ToolResponse { requireAccessibility(request)?.let { return it } - return try { - withContext(Dispatchers.Main) { setOverlayVisibility(false) } - delay(150) + return withOverlayHidden { val bitmap = screenshotSource.takeScreenshot() - ?: return ToolResponse(request.requestId, false, error = "Screenshot capture failed") + ?: return@withOverlayHidden ToolResponse(request.requestId, false, error = "Screenshot capture failed") try { val base64 = withContext(Dispatchers.IO) { val stream = ByteArrayOutputStream() @@ -171,8 +179,6 @@ class ToolRequestHandler( } finally { bitmap.recycle() } - } finally { - withContext(Dispatchers.Main) { setOverlayVisibility(true) } } } @@ -186,14 +192,12 @@ class ToolRequestHandler( val maxDepth = request.params?.get("max_depth")?.jsonPrimitive?.intOrNull ?: 15 val maxNodes = request.params?.get("max_nodes")?.jsonPrimitive?.intOrNull ?: 300 - return try { - withContext(Dispatchers.Main) { setOverlayVisibility(false) } - delay(150) + return withOverlayHidden { val root = deviceController.getRootNode() - ?: return ToolResponse(request.requestId, false, error = "Could not get UI tree") + ?: return@withOverlayHidden ToolResponse(request.requestId, false, error = "Could not get UI tree") try { val startNode = resolveStartNode(root, resourceId, index, boundsX, boundsY) - ?: return ToolResponse(request.requestId, false, error = buildString { + ?: return@withOverlayHidden ToolResponse(request.requestId, false, error = buildString { if (resourceId != null) append("No node found with resource_id=$resourceId (index=$index)") else append("No node found at bounds ($boundsX, $boundsY)") }) @@ -211,8 +215,6 @@ class ToolRequestHandler( } finally { root.recycle() } - } finally { - withContext(Dispatchers.Main) { setOverlayVisibility(true) } } } @@ -313,12 +315,14 @@ class ToolRequestHandler( val y = request.params?.get("y")?.jsonPrimitive?.doubleOrNull?.toFloat() ?: return ToolResponse(request.requestId, false, error = "y coordinate required") - val success = deviceController.tap(x, y) - return ToolResponse( - request.requestId, success, - result = if (success) "Tapped at ($x, $y)" else null, - error = if (!success) "Tap failed" else null - ) + return withOverlayHidden { + val success = deviceController.tap(x, y) + ToolResponse( + request.requestId, success, + result = if (success) "Tapped at ($x, $y)" else null, + error = if (!success) "Tap failed" else null + ) + } } private suspend fun handleSwipe(request: ToolRequest): ToolResponse { @@ -334,12 +338,14 @@ class ToolRequestHandler( ?: return ToolResponse(request.requestId, false, error = "y2 coordinate required") val durationMs = request.params?.get("duration_ms")?.jsonPrimitive?.longOrNull ?: 300L - val success = deviceController.swipe(x, y, x2, y2, durationMs) - return ToolResponse( - request.requestId, success, - result = if (success) "Swiped from ($x,$y) to ($x2,$y2)" else null, - error = if (!success) "Swipe failed" else null - ) + return withOverlayHidden { + val success = deviceController.swipe(x, y, x2, y2, durationMs) + ToolResponse( + request.requestId, success, + result = if (success) "Swiped from ($x,$y) to ($x2,$y2)" else null, + error = if (!success) "Swipe failed" else null + ) + } } private suspend fun handleText(request: ToolRequest): ToolResponse { @@ -348,12 +354,14 @@ class ToolRequestHandler( val text = request.params?.get("text")?.jsonPrimitive?.contentOrNull ?: return ToolResponse(request.requestId, false, error = "text required") - val success = deviceController.inputText(text) - return ToolResponse( - request.requestId, success, - result = if (success) "Text input: $text" else null, - error = if (!success) "Text input failed (no focused input field?)" else null - ) + return withOverlayHidden { + val success = deviceController.inputText(text) + ToolResponse( + request.requestId, success, + result = if (success) "Text input: $text" else null, + error = if (!success) "Text input failed (no focused input field?)" else null + ) + } } private fun handleKeyEvent(request: ToolRequest): ToolResponse {