fix: hide overlay before tap/swipe/text to prevent touch interception

The overlay window could intercept dispatchGesture touches in its
350dp bottom area. Now all UI actions (tap, swipe, text, screenshot,
get_ui_tree) hide the overlay with both View.INVISIBLE and
FLAG_NOT_TOUCHABLE via a shared withOverlayHidden helper.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
KoheiYamashita 2026-02-22 19:43:25 +09:00
parent fac018d539
commit 2c5d8fbf5d
2 changed files with 54 additions and 35 deletions

View file

@ -118,9 +118,7 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner {
context = applicationContext,
deviceController = deviceController,
screenshotSource = screenshotSource,
setOverlayVisibility = { visible ->
overlayView?.visibility = if (visible) View.VISIBLE else View.INVISIBLE
},
setOverlayVisibility = { visible -> setOverlayVisible(visible) },
onAccessibilityNeeded = { showAccessibilityGuide = true }
)
(connection as AssistantConnectionImpl).onToolRequest = { request ->
@ -136,7 +134,7 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner {
ttsWrapper = TextToSpeechWrapper(this, ttsSettingsRepo.ttsConfig)
cameraCaptureManager = CameraCaptureManager(this)
screenCaptureManager = ScreenCaptureManager(screenshotSource, applicationContext) { visible ->
overlayView?.visibility = if (visible) View.VISIBLE else View.INVISIBLE
setOverlayVisible(visible)
}
assistantManager = AssistantManager(
@ -226,6 +224,19 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner {
stopSelf()
}
private fun setOverlayVisible(visible: Boolean) {
val view = overlayView ?: return
val lp = view.layoutParams as? WindowManager.LayoutParams ?: return
if (visible) {
lp.flags = lp.flags and WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE.inv()
view.visibility = View.VISIBLE
} else {
lp.flags = lp.flags or WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE
view.visibility = View.INVISIBLE
}
windowManager.updateViewLayout(view, lp)
}
private fun addOverlay() {
if (overlayView != null) return

View file

@ -76,6 +76,16 @@ class ToolRequestHandler(
return null
}
private suspend fun <T> withOverlayHidden(block: suspend () -> T): T {
return try {
withContext(Dispatchers.Main) { setOverlayVisibility(false) }
delay(150)
block()
} finally {
withContext(Dispatchers.Main) { setOverlayVisibility(true) }
}
}
private fun handleSearchApps(request: ToolRequest): ToolResponse {
val query = request.params?.get("query")?.jsonPrimitive?.contentOrNull
?: return ToolResponse(request.requestId, false, error = "query required")
@ -156,11 +166,9 @@ class ToolRequestHandler(
private suspend fun handleScreenshot(request: ToolRequest): ToolResponse {
requireAccessibility(request)?.let { return it }
return try {
withContext(Dispatchers.Main) { setOverlayVisibility(false) }
delay(150)
return withOverlayHidden {
val bitmap = screenshotSource.takeScreenshot()
?: return ToolResponse(request.requestId, false, error = "Screenshot capture failed")
?: return@withOverlayHidden ToolResponse(request.requestId, false, error = "Screenshot capture failed")
try {
val base64 = withContext(Dispatchers.IO) {
val stream = ByteArrayOutputStream()
@ -171,8 +179,6 @@ class ToolRequestHandler(
} finally {
bitmap.recycle()
}
} finally {
withContext(Dispatchers.Main) { setOverlayVisibility(true) }
}
}
@ -186,14 +192,12 @@ class ToolRequestHandler(
val maxDepth = request.params?.get("max_depth")?.jsonPrimitive?.intOrNull ?: 15
val maxNodes = request.params?.get("max_nodes")?.jsonPrimitive?.intOrNull ?: 300
return try {
withContext(Dispatchers.Main) { setOverlayVisibility(false) }
delay(150)
return withOverlayHidden {
val root = deviceController.getRootNode()
?: return ToolResponse(request.requestId, false, error = "Could not get UI tree")
?: return@withOverlayHidden ToolResponse(request.requestId, false, error = "Could not get UI tree")
try {
val startNode = resolveStartNode(root, resourceId, index, boundsX, boundsY)
?: return ToolResponse(request.requestId, false, error = buildString {
?: return@withOverlayHidden ToolResponse(request.requestId, false, error = buildString {
if (resourceId != null) append("No node found with resource_id=$resourceId (index=$index)")
else append("No node found at bounds ($boundsX, $boundsY)")
})
@ -211,8 +215,6 @@ class ToolRequestHandler(
} finally {
root.recycle()
}
} finally {
withContext(Dispatchers.Main) { setOverlayVisibility(true) }
}
}
@ -313,12 +315,14 @@ class ToolRequestHandler(
val y = request.params?.get("y")?.jsonPrimitive?.doubleOrNull?.toFloat()
?: return ToolResponse(request.requestId, false, error = "y coordinate required")
val success = deviceController.tap(x, y)
return ToolResponse(
request.requestId, success,
result = if (success) "Tapped at ($x, $y)" else null,
error = if (!success) "Tap failed" else null
)
return withOverlayHidden {
val success = deviceController.tap(x, y)
ToolResponse(
request.requestId, success,
result = if (success) "Tapped at ($x, $y)" else null,
error = if (!success) "Tap failed" else null
)
}
}
private suspend fun handleSwipe(request: ToolRequest): ToolResponse {
@ -334,12 +338,14 @@ class ToolRequestHandler(
?: return ToolResponse(request.requestId, false, error = "y2 coordinate required")
val durationMs = request.params?.get("duration_ms")?.jsonPrimitive?.longOrNull ?: 300L
val success = deviceController.swipe(x, y, x2, y2, durationMs)
return ToolResponse(
request.requestId, success,
result = if (success) "Swiped from ($x,$y) to ($x2,$y2)" else null,
error = if (!success) "Swipe failed" else null
)
return withOverlayHidden {
val success = deviceController.swipe(x, y, x2, y2, durationMs)
ToolResponse(
request.requestId, success,
result = if (success) "Swiped from ($x,$y) to ($x2,$y2)" else null,
error = if (!success) "Swipe failed" else null
)
}
}
private suspend fun handleText(request: ToolRequest): ToolResponse {
@ -348,12 +354,14 @@ class ToolRequestHandler(
val text = request.params?.get("text")?.jsonPrimitive?.contentOrNull
?: return ToolResponse(request.requestId, false, error = "text required")
val success = deviceController.inputText(text)
return ToolResponse(
request.requestId, success,
result = if (success) "Text input: $text" else null,
error = if (!success) "Text input failed (no focused input field?)" else null
)
return withOverlayHidden {
val success = deviceController.inputText(text)
ToolResponse(
request.requestId, success,
result = if (success) "Text input: $text" else null,
error = if (!success) "Text input failed (no focused input field?)" else null
)
}
}
private fun handleKeyEvent(request: ToolRequest): ToolResponse {