opencode/picoclaw
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: hide overlay before tap/swipe/text to prevent touch interception

Browse source 
The overlay window could intercept dispatchGesture touches in its
350dp bottom area. Now all UI actions (tap, swipe, text, screenshot,
get_ui_tree) hide the overlay with both View.INVISIBLE and
FLAG_NOT_TOUCHABLE via a shared withOverlayHidden helper.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
KoheiYamashita 2026-02-22 19:43:25 +09:00
parent fac018d539
commit 2c5d8fbf5d
2 changed files with 54 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
android/app/src/main/java/io/picoclaw/android/assistant/AssistantService.kt
View file

@ -118,9 +118,7 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner {
context = applicationContext, context = applicationContext,
deviceController = deviceController, deviceController = deviceController,
screenshotSource = screenshotSource, screenshotSource = screenshotSource,
setOverlayVisibility = { visible -> setOverlayVisibility = { visible -> setOverlayVisible(visible) },
overlayView?.visibility = if (visible) View.VISIBLE else View.INVISIBLE
},
onAccessibilityNeeded = { showAccessibilityGuide = true } onAccessibilityNeeded = { showAccessibilityGuide = true }
) )
(connection as AssistantConnectionImpl).onToolRequest = { request -> (connection as AssistantConnectionImpl).onToolRequest = { request ->
@ -136,7 +134,7 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner {
ttsWrapper = TextToSpeechWrapper(this, ttsSettingsRepo.ttsConfig) ttsWrapper = TextToSpeechWrapper(this, ttsSettingsRepo.ttsConfig)
cameraCaptureManager = CameraCaptureManager(this) cameraCaptureManager = CameraCaptureManager(this)
screenCaptureManager = ScreenCaptureManager(screenshotSource, applicationContext) { visible -> screenCaptureManager = ScreenCaptureManager(screenshotSource, applicationContext) { visible ->
overlayView?.visibility = if (visible) View.VISIBLE else View.INVISIBLE setOverlayVisible(visible)
} }
assistantManager = AssistantManager( assistantManager = AssistantManager(
@ -226,6 +224,19 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner {
stopSelf() stopSelf()
} }
private fun setOverlayVisible(visible: Boolean) {
val view = overlayView ?: return
val lp = view.layoutParams as? WindowManager.LayoutParams ?: return
if (visible) {
lp.flags = lp.flags and WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE.inv()
view.visibility = View.VISIBLE
} else {
lp.flags = lp.flags or WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE
view.visibility = View.INVISIBLE
}
windowManager.updateViewLayout(view, lp)
}
private fun addOverlay() { private fun addOverlay() {
if (overlayView != null) return if (overlayView != null) return

70
android/app/src/main/java/io/picoclaw/android/assistant/ToolRequestHandler.kt
View file

@ -76,6 +76,16 @@ class ToolRequestHandler(
return null return null
} }
private suspend fun <T> withOverlayHidden(block: suspend () -> T): T {
return try {
withContext(Dispatchers.Main) { setOverlayVisibility(false) }
delay(150)
block()
} finally {
withContext(Dispatchers.Main) { setOverlayVisibility(true) }
}
}
private fun handleSearchApps(request: ToolRequest): ToolResponse { private fun handleSearchApps(request: ToolRequest): ToolResponse {
val query = request.params?.get("query")?.jsonPrimitive?.contentOrNull val query = request.params?.get("query")?.jsonPrimitive?.contentOrNull
?: return ToolResponse(request.requestId, false, error = "query required") ?: return ToolResponse(request.requestId, false, error = "query required")
@ -156,11 +166,9 @@ class ToolRequestHandler(
private suspend fun handleScreenshot(request: ToolRequest): ToolResponse { private suspend fun handleScreenshot(request: ToolRequest): ToolResponse {
requireAccessibility(request)?.let { return it } requireAccessibility(request)?.let { return it }
return try { return withOverlayHidden {
withContext(Dispatchers.Main) { setOverlayVisibility(false) }
delay(150)
val bitmap = screenshotSource.takeScreenshot() val bitmap = screenshotSource.takeScreenshot()
?: return ToolResponse(request.requestId, false, error = "Screenshot capture failed") ?: return@withOverlayHidden ToolResponse(request.requestId, false, error = "Screenshot capture failed")
try { try {
val base64 = withContext(Dispatchers.IO) { val base64 = withContext(Dispatchers.IO) {
val stream = ByteArrayOutputStream() val stream = ByteArrayOutputStream()
@ -171,8 +179,6 @@ class ToolRequestHandler(
} finally { } finally {
bitmap.recycle() bitmap.recycle()
} }
} finally {
withContext(Dispatchers.Main) { setOverlayVisibility(true) }
} }
} }
@ -186,14 +192,12 @@ class ToolRequestHandler(
val maxDepth = request.params?.get("max_depth")?.jsonPrimitive?.intOrNull ?: 15 val maxDepth = request.params?.get("max_depth")?.jsonPrimitive?.intOrNull ?: 15
val maxNodes = request.params?.get("max_nodes")?.jsonPrimitive?.intOrNull ?: 300 val maxNodes = request.params?.get("max_nodes")?.jsonPrimitive?.intOrNull ?: 300
return try { return withOverlayHidden {
withContext(Dispatchers.Main) { setOverlayVisibility(false) }
delay(150)
val root = deviceController.getRootNode() val root = deviceController.getRootNode()
?: return ToolResponse(request.requestId, false, error = "Could not get UI tree") ?: return@withOverlayHidden ToolResponse(request.requestId, false, error = "Could not get UI tree")
try { try {
val startNode = resolveStartNode(root, resourceId, index, boundsX, boundsY) val startNode = resolveStartNode(root, resourceId, index, boundsX, boundsY)
?: return ToolResponse(request.requestId, false, error = buildString { ?: return@withOverlayHidden ToolResponse(request.requestId, false, error = buildString {
if (resourceId != null) append("No node found with resource_id=$resourceId (index=$index)") if (resourceId != null) append("No node found with resource_id=$resourceId (index=$index)")
else append("No node found at bounds ($boundsX, $boundsY)") else append("No node found at bounds ($boundsX, $boundsY)")
}) })
@ -211,8 +215,6 @@ class ToolRequestHandler(
} finally { } finally {
root.recycle() root.recycle()
} }
} finally {
withContext(Dispatchers.Main) { setOverlayVisibility(true) }
} }
} }
@ -313,12 +315,14 @@ class ToolRequestHandler(
val y = request.params?.get("y")?.jsonPrimitive?.doubleOrNull?.toFloat() val y = request.params?.get("y")?.jsonPrimitive?.doubleOrNull?.toFloat()
?: return ToolResponse(request.requestId, false, error = "y coordinate required") ?: return ToolResponse(request.requestId, false, error = "y coordinate required")
val success = deviceController.tap(x, y) return withOverlayHidden {
return ToolResponse( val success = deviceController.tap(x, y)
request.requestId, success, ToolResponse(
result = if (success) "Tapped at ($x, $y)" else null, request.requestId, success,
error = if (!success) "Tap failed" else null result = if (success) "Tapped at ($x, $y)" else null,
) error = if (!success) "Tap failed" else null
)
}
} }
private suspend fun handleSwipe(request: ToolRequest): ToolResponse { private suspend fun handleSwipe(request: ToolRequest): ToolResponse {
@ -334,12 +338,14 @@ class ToolRequestHandler(
?: return ToolResponse(request.requestId, false, error = "y2 coordinate required") ?: return ToolResponse(request.requestId, false, error = "y2 coordinate required")
val durationMs = request.params?.get("duration_ms")?.jsonPrimitive?.longOrNull ?: 300L val durationMs = request.params?.get("duration_ms")?.jsonPrimitive?.longOrNull ?: 300L
val success = deviceController.swipe(x, y, x2, y2, durationMs) return withOverlayHidden {
return ToolResponse( val success = deviceController.swipe(x, y, x2, y2, durationMs)
request.requestId, success, ToolResponse(
result = if (success) "Swiped from ($x,$y) to ($x2,$y2)" else null, request.requestId, success,
error = if (!success) "Swipe failed" else null result = if (success) "Swiped from ($x,$y) to ($x2,$y2)" else null,
) error = if (!success) "Swipe failed" else null
)
}
} }
private suspend fun handleText(request: ToolRequest): ToolResponse { private suspend fun handleText(request: ToolRequest): ToolResponse {
@ -348,12 +354,14 @@ class ToolRequestHandler(
val text = request.params?.get("text")?.jsonPrimitive?.contentOrNull val text = request.params?.get("text")?.jsonPrimitive?.contentOrNull
?: return ToolResponse(request.requestId, false, error = "text required") ?: return ToolResponse(request.requestId, false, error = "text required")
val success = deviceController.inputText(text) return withOverlayHidden {
return ToolResponse( val success = deviceController.inputText(text)
request.requestId, success, ToolResponse(
result = if (success) "Text input: $text" else null, request.requestId, success,
error = if (!success) "Text input failed (no focused input field?)" else null result = if (success) "Text input: $text" else null,
) error = if (!success) "Text input failed (no focused input field?)" else null
)
}
} }
private fun handleKeyEvent(request: ToolRequest): ToolResponse { private fun handleKeyEvent(request: ToolRequest): ToolResponse {