diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 60bb596bf..0eb31e3b6 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -82,6 +82,17 @@ android:name="android.app.PROPERTY_SPECIAL_USE_FGS_SUBTYPE" android:value="AI voice assistant overlay for real-time conversation" /> + + + + + + diff --git a/android/app/src/main/java/io/picoclaw/android/assistant/AccessibilityScreenshotSource.kt b/android/app/src/main/java/io/picoclaw/android/assistant/AccessibilityScreenshotSource.kt new file mode 100644 index 000000000..5beb9d329 --- /dev/null +++ b/android/app/src/main/java/io/picoclaw/android/assistant/AccessibilityScreenshotSource.kt @@ -0,0 +1,55 @@ +package io.picoclaw.android.assistant + +import android.accessibilityservice.AccessibilityService +import android.graphics.Bitmap +import android.util.Log +import android.view.Display +import io.picoclaw.android.feature.chat.voice.ScreenshotSource +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlin.coroutines.resume + +class AccessibilityScreenshotSource : ScreenshotSource { + + @Volatile + private var service: AccessibilityService? = null + + override val isAvailable: Boolean get() = service != null + + fun setService(s: AccessibilityService) { + service = s + } + + fun clearService() { + service = null + } + + override suspend fun takeScreenshot(): Bitmap? { + val svc = service ?: return null + return suspendCancellableCoroutine { cont -> + svc.takeScreenshot( + Display.DEFAULT_DISPLAY, + svc.mainExecutor, + object : AccessibilityService.TakeScreenshotCallback { + override fun onSuccess(result: AccessibilityService.ScreenshotResult) { + val hwBitmap = Bitmap.wrapHardwareBuffer( + result.hardwareBuffer, result.colorSpace + ) + result.hardwareBuffer.close() + val swBitmap = hwBitmap?.copy(Bitmap.Config.ARGB_8888, false) + hwBitmap?.recycle() + cont.resume(swBitmap) + } + + override fun onFailure(errorCode: Int) { + Log.w(TAG, "takeScreenshot failed with errorCode=$errorCode") + cont.resume(null) + } + } + ) + } + } + + companion object { + private const val TAG = "A11yScreenshotSource" + } +} diff --git a/android/app/src/main/java/io/picoclaw/android/assistant/AssistantService.kt b/android/app/src/main/java/io/picoclaw/android/assistant/AssistantService.kt index 18a0cb7c3..315bf0b1d 100644 --- a/android/app/src/main/java/io/picoclaw/android/assistant/AssistantService.kt +++ b/android/app/src/main/java/io/picoclaw/android/assistant/AssistantService.kt @@ -10,6 +10,7 @@ import android.content.pm.PackageManager import android.content.pm.ServiceInfo import android.graphics.PixelFormat import android.os.IBinder +import android.provider.Settings import android.view.Gravity import android.view.MotionEvent import android.view.View @@ -42,6 +43,8 @@ import io.picoclaw.android.core.ui.theme.PicoClawTheme import io.picoclaw.android.feature.chat.assistant.AssistantManager import io.picoclaw.android.feature.chat.assistant.AssistantPillBar import io.picoclaw.android.feature.chat.voice.CameraCaptureManager +import io.picoclaw.android.feature.chat.voice.ScreenCaptureManager +import io.picoclaw.android.feature.chat.voice.ScreenshotSource import io.picoclaw.android.feature.chat.voice.SpeechRecognizerWrapper import io.picoclaw.android.feature.chat.voice.TextToSpeechWrapper import io.picoclaw.android.receiver.NotificationHelper @@ -55,6 +58,7 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner { private val httpClient: HttpClient by inject() private val ttsSettingsRepo: TtsSettingsRepository by inject() + private val screenshotSource: ScreenshotSource by inject() private lateinit var serviceScope: CoroutineScope private lateinit var connection: AssistantConnection @@ -62,6 +66,7 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner { private lateinit var ttsWrapper: TextToSpeechWrapper private lateinit var sttWrapper: SpeechRecognizerWrapper private lateinit var cameraCaptureManager: CameraCaptureManager + private lateinit var screenCaptureManager: ScreenCaptureManager private var overlayView: View? = null private val windowManager by lazy { getSystemService(WINDOW_SERVICE) as WindowManager } @@ -93,12 +98,16 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner { sttWrapper = SpeechRecognizerWrapper(this) ttsWrapper = TextToSpeechWrapper(this, ttsSettingsRepo.ttsConfig) cameraCaptureManager = CameraCaptureManager(this) + screenCaptureManager = ScreenCaptureManager(screenshotSource, applicationContext) { visible -> + overlayView?.visibility = if (visible) View.VISIBLE else View.INVISIBLE + } assistantManager = AssistantManager( sttWrapper = sttWrapper, ttsWrapper = ttsWrapper, connection = connection, cameraCaptureManager = cameraCaptureManager, + screenCaptureManager = screenCaptureManager, contentResolver = contentResolver ) @@ -159,6 +168,25 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner { } } + private fun handleScreenCaptureToggle() { + if (assistantManager.state.value.isScreenCaptureActive) { + assistantManager.toggleScreenCapture() + return + } + if (screenCaptureManager.isAvailable) { + // Turn off camera first if active + if (assistantManager.state.value.isCameraActive) { + assistantManager.toggleCamera() + } + assistantManager.toggleScreenCapture() + } else { + // Open accessibility settings so the user can enable the service + val intent = Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + startActivity(intent) + } + } + private fun shutdown() { stopForeground(STOP_FOREGROUND_REMOVE) stopSelf() @@ -209,6 +237,7 @@ class AssistantService : LifecycleService(), SavedStateRegistryOwner { onClose = { shutdown() }, onInterrupt = { assistantManager.interrupt() }, onCameraToggle = { handleCameraToggle() }, + onScreenCaptureToggle = { handleScreenCaptureToggle() }, cameraCaptureManager = cameraCaptureManager, modifier = Modifier.onGloballyPositioned { coordinates -> wrapper.contentTop = coordinates.positionInWindow().y.toInt() diff --git a/android/app/src/main/java/io/picoclaw/android/assistant/PicoClawAccessibilityService.kt b/android/app/src/main/java/io/picoclaw/android/assistant/PicoClawAccessibilityService.kt new file mode 100644 index 000000000..b49c7473d --- /dev/null +++ b/android/app/src/main/java/io/picoclaw/android/assistant/PicoClawAccessibilityService.kt @@ -0,0 +1,28 @@ +package io.picoclaw.android.assistant + +import android.accessibilityservice.AccessibilityService +import android.view.accessibility.AccessibilityEvent +import org.koin.android.ext.android.inject + +class PicoClawAccessibilityService : AccessibilityService() { + + private val screenshotSource: AccessibilityScreenshotSource by inject() + + override fun onServiceConnected() { + super.onServiceConnected() + screenshotSource.setService(this) + } + + override fun onDestroy() { + screenshotSource.clearService() + super.onDestroy() + } + + override fun onAccessibilityEvent(event: AccessibilityEvent?) { + // No-op: used only for screenshot capture + } + + override fun onInterrupt() { + // No-op + } +} diff --git a/android/app/src/main/java/io/picoclaw/android/di/AppModule.kt b/android/app/src/main/java/io/picoclaw/android/di/AppModule.kt index 66dc94a59..a0bec7d64 100644 --- a/android/app/src/main/java/io/picoclaw/android/di/AppModule.kt +++ b/android/app/src/main/java/io/picoclaw/android/di/AppModule.kt @@ -5,7 +5,9 @@ import io.ktor.client.HttpClient import io.ktor.client.engine.okhttp.OkHttp import io.ktor.client.plugins.websocket.WebSockets import io.picoclaw.android.core.data.local.AppDatabase +import io.picoclaw.android.assistant.AccessibilityScreenshotSource import io.picoclaw.android.core.data.local.ImageFileStorage +import io.picoclaw.android.feature.chat.voice.ScreenshotSource import io.picoclaw.android.core.data.remote.WebSocketClient import io.picoclaw.android.core.data.repository.ChatRepositoryImpl import io.picoclaw.android.core.data.repository.TtsCatalogRepositoryImpl @@ -90,6 +92,10 @@ val appModule = module { factory { ConnectChatUseCase(get()) } factory { DisconnectChatUseCase(get()) } + // Screenshot + single { AccessibilityScreenshotSource() } + single { get() } + // Voice factory { SpeechRecognizerWrapper(androidContext()) } single { TextToSpeechWrapper(androidContext(), get().ttsConfig) } diff --git a/android/app/src/main/res/values/strings.xml b/android/app/src/main/res/values/strings.xml index 9652097f9..ea06a70b3 100644 --- a/android/app/src/main/res/values/strings.xml +++ b/android/app/src/main/res/values/strings.xml @@ -1,3 +1,4 @@ PicoClaw + Allows PicoClaw to capture screenshots during voice assistant conversations. diff --git a/android/app/src/main/res/xml/accessibility_service_config.xml b/android/app/src/main/res/xml/accessibility_service_config.xml new file mode 100644 index 000000000..429f45e65 --- /dev/null +++ b/android/app/src/main/res/xml/accessibility_service_config.xml @@ -0,0 +1,5 @@ + + diff --git a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/assistant/AssistantManager.kt b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/assistant/AssistantManager.kt index a0d321784..72a427499 100644 --- a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/assistant/AssistantManager.kt +++ b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/assistant/AssistantManager.kt @@ -8,6 +8,7 @@ import io.picoclaw.android.core.domain.model.AssistantMessage import io.picoclaw.android.core.domain.model.VoicePhase import io.picoclaw.android.core.domain.repository.AssistantConnection import io.picoclaw.android.feature.chat.voice.CameraCaptureManager +import io.picoclaw.android.feature.chat.voice.ScreenCaptureManager import io.picoclaw.android.feature.chat.voice.SpeechRecognizerWrapper import io.picoclaw.android.feature.chat.voice.SttResult import io.picoclaw.android.feature.chat.voice.TextToSpeechWrapper @@ -34,6 +35,7 @@ class AssistantManager( private val ttsWrapper: TextToSpeechWrapper, private val connection: AssistantConnection, private val cameraCaptureManager: CameraCaptureManager, + private val screenCaptureManager: ScreenCaptureManager, private val contentResolver: ContentResolver ) { @@ -44,14 +46,33 @@ class AssistantManager( private var parentScope: CoroutineScope? = null fun toggleCamera() { - _state.update { it.copy(isCameraActive = !it.isCameraActive) } + _state.update { + it.copy( + isCameraActive = !it.isCameraActive, + isScreenCaptureActive = false + ) + } + } + + fun toggleScreenCapture() { + if (_state.value.isScreenCaptureActive) { + _state.update { it.copy(isScreenCaptureActive = false) } + } else { + _state.update { + it.copy( + isScreenCaptureActive = true, + isCameraActive = false + ) + } + cameraCaptureManager.unbind() + } } fun start(scope: CoroutineScope) { if (loopJob?.isActive == true) return parentScope = scope _state.update { - VoiceModeState(isActive = true, phase = VoicePhase.LISTENING, isCameraActive = it.isCameraActive, chatHistory = it.chatHistory) + VoiceModeState(isActive = true, phase = VoicePhase.LISTENING, isCameraActive = it.isCameraActive, isScreenCaptureActive = it.isScreenCaptureActive, chatHistory = it.chatHistory) } loopJob = scope.launch { voiceLoop() @@ -76,7 +97,7 @@ class AssistantManager( loopJob = null _state.update { - VoiceModeState(isActive = true, phase = VoicePhase.LISTENING, isCameraActive = it.isCameraActive, chatHistory = it.chatHistory) + VoiceModeState(isActive = true, phase = VoicePhase.LISTENING, isCameraActive = it.isCameraActive, isScreenCaptureActive = it.isScreenCaptureActive, chatHistory = it.chatHistory) } loopJob = scope.launch { voiceLoop() } } @@ -126,7 +147,7 @@ class AssistantManager( if (!text.isNullOrBlank()) { _state.update { it.copy(phase = VoicePhase.SENDING, recognizedText = text, chatHistory = it.chatHistory + ChatTurn("user", text)) } try { - val base64Images = if (_state.value.isCameraActive) { + val base64Images = if (_state.value.isCameraActive || _state.value.isScreenCaptureActive) { captureAndEncode() } else emptyList() connection.send(text, base64Images) @@ -265,7 +286,11 @@ class AssistantManager( } private suspend fun captureAndEncode(): List { - val attachment = cameraCaptureManager.captureFrame() ?: return emptyList() + val attachment = when { + _state.value.isScreenCaptureActive -> screenCaptureManager.captureFrame() + _state.value.isCameraActive -> cameraCaptureManager.captureFrame() + else -> null + } ?: return emptyList() val uri = android.net.Uri.parse(attachment.uri) return try { val bytes = contentResolver.openInputStream(uri)?.use { it.readBytes() } diff --git a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/assistant/AssistantPillBar.kt b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/assistant/AssistantPillBar.kt index cfde64219..58b27a8dc 100644 --- a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/assistant/AssistantPillBar.kt +++ b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/assistant/AssistantPillBar.kt @@ -76,6 +76,7 @@ fun AssistantPillBar( onClose: () -> Unit, onInterrupt: () -> Unit, onCameraToggle: () -> Unit, + onScreenCaptureToggle: () -> Unit, cameraCaptureManager: CameraCaptureManager, modifier: Modifier = Modifier ) { @@ -253,6 +254,22 @@ fun AssistantPillBar( Spacer(modifier = Modifier.width(8.dp)) + // Screen capture toggle + IconButton( + onClick = onScreenCaptureToggle, + modifier = Modifier.size(36.dp) + ) { + Icon( + painter = painterResource( + if (state.isScreenCaptureActive) LucideR.drawable.lucide_ic_monitor_off + else LucideR.drawable.lucide_ic_monitor + ), + contentDescription = if (state.isScreenCaptureActive) "Turn off screen capture" else "Turn on screen capture", + modifier = Modifier.size(18.dp), + tint = if (state.isScreenCaptureActive) GradientCyan else TextSecondary + ) + } + // Camera toggle IconButton( onClick = onCameraToggle, diff --git a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/ScreenCaptureManager.kt b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/ScreenCaptureManager.kt new file mode 100644 index 000000000..895b33b3d --- /dev/null +++ b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/ScreenCaptureManager.kt @@ -0,0 +1,59 @@ +package io.picoclaw.android.feature.chat.voice + +import android.content.Context +import android.graphics.Bitmap +import android.util.Log +import io.picoclaw.android.core.domain.model.ImageAttachment +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.withContext +import java.io.File + +class ScreenCaptureManager( + private val screenshotSource: ScreenshotSource, + private val context: Context, + private val setOverlayVisibility: (Boolean) -> Unit +) { + + val isAvailable: Boolean get() = screenshotSource.isAvailable + + suspend fun captureFrame(): ImageAttachment? { + if (!screenshotSource.isAvailable) return null + return try { + withContext(Dispatchers.Main) { + setOverlayVisibility(false) + } + delay(150) + val bitmap = screenshotSource.takeScreenshot() ?: return null + withContext(Dispatchers.IO) { + try { + val imagesDir = File(context.cacheDir, "images").apply { mkdirs() } + val file = File(imagesDir, "screen_cap_${System.currentTimeMillis()}.jpg") + file.outputStream().use { bitmap.compress(Bitmap.CompressFormat.JPEG, 80, it) } + bitmap.recycle() + + val uri = androidx.core.content.FileProvider.getUriForFile( + context, + "${context.packageName}.fileprovider", + file + ) + ImageAttachment(uri = uri.toString()) + } catch (e: Exception) { + bitmap.recycle() + throw e + } + } + } catch (e: Exception) { + Log.w(TAG, "Failed to capture screen frame", e) + null + } finally { + withContext(Dispatchers.Main) { + setOverlayVisibility(true) + } + } + } + + companion object { + private const val TAG = "ScreenCaptureManager" + } +} diff --git a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/ScreenshotSource.kt b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/ScreenshotSource.kt new file mode 100644 index 000000000..668f729ea --- /dev/null +++ b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/ScreenshotSource.kt @@ -0,0 +1,8 @@ +package io.picoclaw.android.feature.chat.voice + +import android.graphics.Bitmap + +interface ScreenshotSource { + val isAvailable: Boolean + suspend fun takeScreenshot(): Bitmap? +} diff --git a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeState.kt b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeState.kt index 2b2aa34f9..d633ce746 100644 --- a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeState.kt +++ b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeState.kt @@ -13,5 +13,6 @@ data class VoiceModeState( val errorMessage: String? = null, val amplitudeNormalized: Float = 0f, val isCameraActive: Boolean = false, + val isScreenCaptureActive: Boolean = false, val chatHistory: List = emptyList() ) diff --git a/android/gradle/libs.versions.toml b/android/gradle/libs.versions.toml index f4577f8b7..58f91e597 100644 --- a/android/gradle/libs.versions.toml +++ b/android/gradle/libs.versions.toml @@ -1,6 +1,6 @@ [versions] android-compileSdk = "36" -android-minSdk = "29" +android-minSdk = "30" android-targetSdk = "36" agp = "9.0.1" kotlin = "2.3.0"