opencode/picoclaw
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: switch screen capture to AccessibilityService with clean architecture

Browse source 
Replace MediaProjection-based screen capture with AccessibilityService.takeScreenshot()
(API 30+) to eliminate per-session consent dialogs. Extract ScreenshotSource interface
in feature/chat module with implementation in app module via Koin DI, maintaining
unidirectional dependency flow.

- Raise minSdk 29 → 30
- Add ScreenshotSource interface and AccessibilityScreenshotSource implementation
- Add PicoClawAccessibilityService for screenshot capability
- Remove MediaProjection code (ScreenCaptureConsentActivity, broadcast receivers)
- Add screen capture toggle with camera mutual exclusion

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Kohei 2026-02-22 02:09:24 +09:00
parent e3131cfe1a
commit c91a09f0c1
13 changed files with 251 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
android/app/src/main/AndroidManifest.xml
View file

@ -82,6 +82,17 @@
android:name="android.app.PROPERTY_SPECIAL_USE_FGS_SUBTYPE"
android:value="AI voice assistant overlay for real-time conversation" />
</service>
<service
android:name=".assistant.PicoClawAccessibilityService"
android:exported="false"
android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE">
<intent-filter>
<action android:name="android.accessibilityservice.AccessibilityService" />
</intent-filter>
<meta-data
android:name="android.accessibilityservice"
android:resource="@xml/accessibility_service_config" />
</service>
</application>
</manifest>

55
android/app/src/main/java/io/picoclaw/android/assistant/AccessibilityScreenshotSource.kt Normal file
View file

@ -0,0 +1,55 @@
package io.picoclaw.android.assistant
import android.accessibilityservice.AccessibilityService
import android.graphics.Bitmap
import android.util.Log
import android.view.Display
import io.picoclaw.android.feature.chat.voice.ScreenshotSource
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlin.coroutines.resume
class AccessibilityScreenshotSource : ScreenshotSource {
@Volatile
private var service: AccessibilityService? = null
override val isAvailable: Boolean get() = service != null
fun setService(s: AccessibilityService) {
service = s
}
fun clearService() {
service = null
}
override suspend fun takeScreenshot(): Bitmap? {
val svc = service ?: return null
return suspendCancellableCoroutine { cont ->
svc.takeScreenshot(
Display.DEFAULT_DISPLAY,
svc.mainExecutor,
object : AccessibilityService.TakeScreenshotCallback {
override fun onSuccess(result: AccessibilityService.ScreenshotResult) {
val hwBitmap = Bitmap.wrapHardwareBuffer(
result.hardwareBuffer, result.colorSpace
)
result.hardwareBuffer.close()
val swBitmap = hwBitmap?.copy(Bitmap.Config.ARGB_8888, false)
hwBitmap?.recycle()
cont.resume(swBitmap)
}
override fun onFailure(errorCode: Int) {
Log.w(TAG, "takeScreenshot failed with errorCode=$errorCode")
cont.resume(null)
}
}
)
}
}
companion object {
private const val TAG = "A11yScreenshotSource"
}
}

29
android/app/src/main/java/io/picoclaw/android/assistant/AssistantService.kt
View file

@ -10,6 +10,7 @@ import android.content.pm.PackageManager
import android.content.pm.ServiceInfo
import android.graphics.PixelFormat
import android.os.IBinder
import android.provider.Settings
import android.view.Gravity
import android.view.MotionEvent
import android.view.View
@ -42,6 +43,8 @@ import io.picoclaw.android.core.ui.theme.PicoClawTheme
import io.picoclaw.android.feature.chat.assistant.AssistantManager
import io.picoclaw.android.feature.chat.assistant.AssistantPillBar
import io.picoclaw.android.feature.chat.voice.CameraCaptureManager
import io.picoclaw.android.feature.chat.voice.ScreenCaptureManager
import io.picoclaw.android.feature.chat.voice.ScreenshotSource
import io.picoclaw.android.feature.chat.voice.SpeechRecognizerWrapper
import io.picoclaw.android.feature.chat.voice.TextToSpeechWrapper
import io.picoclaw.android.receiver.NotificationHelper
@ -55,6 +58,7 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner {
private val httpClient: HttpClient by inject()
private val ttsSettingsRepo: TtsSettingsRepository by inject()
private val screenshotSource: ScreenshotSource by inject()
private lateinit var serviceScope: CoroutineScope
private lateinit var connection: AssistantConnection
@ -62,6 +66,7 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner {
private lateinit var ttsWrapper: TextToSpeechWrapper
private lateinit var sttWrapper: SpeechRecognizerWrapper
private lateinit var cameraCaptureManager: CameraCaptureManager
private lateinit var screenCaptureManager: ScreenCaptureManager
private var overlayView: View? = null
private val windowManager by lazy { getSystemService(WINDOW_SERVICE) as WindowManager }
@ -93,12 +98,16 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner {
sttWrapper = SpeechRecognizerWrapper(this)
ttsWrapper = TextToSpeechWrapper(this, ttsSettingsRepo.ttsConfig)
cameraCaptureManager = CameraCaptureManager(this)
screenCaptureManager = ScreenCaptureManager(screenshotSource, applicationContext) { visible ->
overlayView?.visibility = if (visible) View.VISIBLE else View.INVISIBLE
}
assistantManager = AssistantManager(
sttWrapper = sttWrapper,
ttsWrapper = ttsWrapper,
connection = connection,
cameraCaptureManager = cameraCaptureManager,
screenCaptureManager = screenCaptureManager,
contentResolver = contentResolver
)
@ -159,6 +168,25 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner {
}
}
private fun handleScreenCaptureToggle() {
if (assistantManager.state.value.isScreenCaptureActive) {
assistantManager.toggleScreenCapture()
return
}
if (screenCaptureManager.isAvailable) {
// Turn off camera first if active
if (assistantManager.state.value.isCameraActive) {
assistantManager.toggleCamera()
}
assistantManager.toggleScreenCapture()
} else {
// Open accessibility settings so the user can enable the service
val intent = Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS)
.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
startActivity(intent)
}
}
private fun shutdown() {
stopForeground(STOP_FOREGROUND_REMOVE)
stopSelf()
@ -209,6 +237,7 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner {
onClose = { shutdown() },
onInterrupt = { assistantManager.interrupt() },
onCameraToggle = { handleCameraToggle() },
onScreenCaptureToggle = { handleScreenCaptureToggle() },
cameraCaptureManager = cameraCaptureManager,
modifier = Modifier.onGloballyPositioned { coordinates ->
wrapper.contentTop = coordinates.positionInWindow().y.toInt()

28
android/app/src/main/java/io/picoclaw/android/assistant/PicoClawAccessibilityService.kt Normal file
View file

@ -0,0 +1,28 @@
package io.picoclaw.android.assistant
import android.accessibilityservice.AccessibilityService
import android.view.accessibility.AccessibilityEvent
import org.koin.android.ext.android.inject
class PicoClawAccessibilityService : AccessibilityService() {
private val screenshotSource: AccessibilityScreenshotSource by inject()
override fun onServiceConnected() {
super.onServiceConnected()
screenshotSource.setService(this)
}
override fun onDestroy() {
screenshotSource.clearService()
super.onDestroy()
}
override fun onAccessibilityEvent(event: AccessibilityEvent?) {
// No-op: used only for screenshot capture
}
override fun onInterrupt() {
// No-op
}
}

6
android/app/src/main/java/io/picoclaw/android/di/AppModule.kt
View file

@ -5,7 +5,9 @@ import io.ktor.client.HttpClient
import io.ktor.client.engine.okhttp.OkHttp
import io.ktor.client.plugins.websocket.WebSockets
import io.picoclaw.android.core.data.local.AppDatabase
import io.picoclaw.android.assistant.AccessibilityScreenshotSource
import io.picoclaw.android.core.data.local.ImageFileStorage
import io.picoclaw.android.feature.chat.voice.ScreenshotSource
import io.picoclaw.android.core.data.remote.WebSocketClient
import io.picoclaw.android.core.data.repository.ChatRepositoryImpl
import io.picoclaw.android.core.data.repository.TtsCatalogRepositoryImpl
@ -90,6 +92,10 @@ val appModule = module {
factory { ConnectChatUseCase(get()) }
factory { DisconnectChatUseCase(get()) }
// Screenshot
single { AccessibilityScreenshotSource() }
single<ScreenshotSource> { get<AccessibilityScreenshotSource>() }
// Voice
factory { SpeechRecognizerWrapper(androidContext()) }
single { TextToSpeechWrapper(androidContext(), get<TtsSettingsRepository>().ttsConfig) }

1
android/app/src/main/res/values/strings.xml
View file

@ -1,3 +1,4 @@
<resources>
<string name="app_name">PicoClaw</string>
<string name="accessibility_service_description">Allows PicoClaw to capture screenshots during voice assistant conversations.</string>
</resources>

5
android/app/src/main/res/xml/accessibility_service_config.xml Normal file
View file

@ -0,0 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
android:canTakeScreenshot="true"
android:canRetrieveWindowContent="false"
android:description="@string/accessibility_service_description" />

35
android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/assistant/AssistantManager.kt
View file

@ -8,6 +8,7 @@ import io.picoclaw.android.core.domain.model.AssistantMessage
import io.picoclaw.android.core.domain.model.VoicePhase
import io.picoclaw.android.core.domain.repository.AssistantConnection
import io.picoclaw.android.feature.chat.voice.CameraCaptureManager
import io.picoclaw.android.feature.chat.voice.ScreenCaptureManager
import io.picoclaw.android.feature.chat.voice.SpeechRecognizerWrapper
import io.picoclaw.android.feature.chat.voice.SttResult
import io.picoclaw.android.feature.chat.voice.TextToSpeechWrapper
@ -34,6 +35,7 @@ class AssistantManager(
private val ttsWrapper: TextToSpeechWrapper,
private val connection: AssistantConnection,
private val cameraCaptureManager: CameraCaptureManager,
private val screenCaptureManager: ScreenCaptureManager,
private val contentResolver: ContentResolver
) {
@ -44,14 +46,33 @@ class AssistantManager(
private var parentScope: CoroutineScope? = null
fun toggleCamera() {
_state.update { it.copy(isCameraActive = !it.isCameraActive) }
_state.update {
it.copy(
isCameraActive = !it.isCameraActive,
isScreenCaptureActive = false
)
}
}
fun toggleScreenCapture() {
if (_state.value.isScreenCaptureActive) {
_state.update { it.copy(isScreenCaptureActive = false) }
} else {
_state.update {
it.copy(
isScreenCaptureActive = true,
isCameraActive = false
)
}
cameraCaptureManager.unbind()
}
}
fun start(scope: CoroutineScope) {
if (loopJob?.isActive == true) return
parentScope = scope
_state.update {
VoiceModeState(isActive = true, phase = VoicePhase.LISTENING, isCameraActive = it.isCameraActive, chatHistory = it.chatHistory)
VoiceModeState(isActive = true, phase = VoicePhase.LISTENING, isCameraActive = it.isCameraActive, isScreenCaptureActive = it.isScreenCaptureActive, chatHistory = it.chatHistory)
}
loopJob = scope.launch {
voiceLoop()
@ -76,7 +97,7 @@ class AssistantManager(
loopJob = null
_state.update {
VoiceModeState(isActive = true, phase = VoicePhase.LISTENING, isCameraActive = it.isCameraActive, chatHistory = it.chatHistory)
VoiceModeState(isActive = true, phase = VoicePhase.LISTENING, isCameraActive = it.isCameraActive, isScreenCaptureActive = it.isScreenCaptureActive, chatHistory = it.chatHistory)
}
loopJob = scope.launch { voiceLoop() }
}
@ -126,7 +147,7 @@ class AssistantManager(
if (!text.isNullOrBlank()) {
_state.update { it.copy(phase = VoicePhase.SENDING, recognizedText = text, chatHistory = it.chatHistory + ChatTurn("user", text)) }
try {
val base64Images = if (_state.value.isCameraActive) {
val base64Images = if (_state.value.isCameraActive || _state.value.isScreenCaptureActive) {
captureAndEncode()
} else emptyList()
connection.send(text, base64Images)
@ -265,7 +286,11 @@ class AssistantManager(
}
private suspend fun captureAndEncode(): List<String> {
val attachment = cameraCaptureManager.captureFrame() ?: return emptyList()
val attachment = when {
_state.value.isScreenCaptureActive -> screenCaptureManager.captureFrame()
_state.value.isCameraActive -> cameraCaptureManager.captureFrame()
else -> null
} ?: return emptyList()
val uri = android.net.Uri.parse(attachment.uri)
return try {
val bytes = contentResolver.openInputStream(uri)?.use { it.readBytes() }

17
android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/assistant/AssistantPillBar.kt
View file

@ -76,6 +76,7 @@ fun AssistantPillBar(
onClose: () -> Unit,
onInterrupt: () -> Unit,
onCameraToggle: () -> Unit,
onScreenCaptureToggle: () -> Unit,
cameraCaptureManager: CameraCaptureManager,
modifier: Modifier = Modifier
) {
@ -253,6 +254,22 @@ fun AssistantPillBar(
Spacer(modifier = Modifier.width(8.dp))
// Screen capture toggle
IconButton(
onClick = onScreenCaptureToggle,
modifier = Modifier.size(36.dp)
) {
Icon(
painter = painterResource(
if (state.isScreenCaptureActive) LucideR.drawable.lucide_ic_monitor_off
else LucideR.drawable.lucide_ic_monitor
),
contentDescription = if (state.isScreenCaptureActive) "Turn off screen capture" else "Turn on screen capture",
modifier = Modifier.size(18.dp),
tint = if (state.isScreenCaptureActive) GradientCyan else TextSecondary
)
}
// Camera toggle
IconButton(
onClick = onCameraToggle,

59
android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/ScreenCaptureManager.kt Normal file
View file

@ -0,0 +1,59 @@
package io.picoclaw.android.feature.chat.voice
import android.content.Context
import android.graphics.Bitmap
import android.util.Log
import io.picoclaw.android.core.domain.model.ImageAttachment
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.withContext
import java.io.File
class ScreenCaptureManager(
private val screenshotSource: ScreenshotSource,
private val context: Context,
private val setOverlayVisibility: (Boolean) -> Unit
) {
val isAvailable: Boolean get() = screenshotSource.isAvailable
suspend fun captureFrame(): ImageAttachment? {
if (!screenshotSource.isAvailable) return null
return try {
withContext(Dispatchers.Main) {
setOverlayVisibility(false)
}
delay(150)
val bitmap = screenshotSource.takeScreenshot() ?: return null
withContext(Dispatchers.IO) {
try {
val imagesDir = File(context.cacheDir, "images").apply { mkdirs() }
val file = File(imagesDir, "screen_cap_${System.currentTimeMillis()}.jpg")
file.outputStream().use { bitmap.compress(Bitmap.CompressFormat.JPEG, 80, it) }
bitmap.recycle()
val uri = androidx.core.content.FileProvider.getUriForFile(
context,
"${context.packageName}.fileprovider",
file
)
ImageAttachment(uri = uri.toString())
} catch (e: Exception) {
bitmap.recycle()
throw e
}
}
} catch (e: Exception) {
Log.w(TAG, "Failed to capture screen frame", e)
null
} finally {
withContext(Dispatchers.Main) {
setOverlayVisibility(true)
}
}
}
companion object {
private const val TAG = "ScreenCaptureManager"
}
}

8
android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/ScreenshotSource.kt Normal file
View file

@ -0,0 +1,8 @@
package io.picoclaw.android.feature.chat.voice
import android.graphics.Bitmap
interface ScreenshotSource {
val isAvailable: Boolean
suspend fun takeScreenshot(): Bitmap?
}

1
android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeState.kt
View file

@ -13,5 +13,6 @@ data class VoiceModeState(
val errorMessage: String? = null,
val amplitudeNormalized: Float = 0f,
val isCameraActive: Boolean = false,
val isScreenCaptureActive: Boolean = false,
val chatHistory: List<ChatTurn> = emptyList()
)

2
android/gradle/libs.versions.toml
View file

@ -1,6 +1,6 @@
[versions]
android-compileSdk = "36"
android-minSdk = "29"
android-minSdk = "30"
android-targetSdk = "36"
agp = "9.0.1"
kotlin = "2.3.0"