From 643aa0d47ccca111efce20419f37be69c375fa95 Mon Sep 17 00:00:00 2001 From: KoheiYamashita Date: Fri, 20 Feb 2026 10:35:02 +0900 Subject: [PATCH] feat: add camera capture during voice mode conversations Allow attaching camera images to messages sent via voice mode. A camera toggle button in VoiceModeOverlay shows/hides a live CameraX preview, and automatically captures a frame when speech recognition completes in the SENDING phase. Co-Authored-By: Claude Opus 4.6 --- .../java/io/picoclaw/android/di/AppModule.kt | 4 +- android/feature/chat/build.gradle.kts | 5 ++ .../android/feature/chat/ChatEvent.kt | 1 + .../android/feature/chat/ChatViewModel.kt | 3 + .../android/feature/chat/screen/ChatScreen.kt | 29 ++++++- .../chat/voice/CameraCaptureManager.kt | 80 +++++++++++++++++++ .../feature/chat/voice/VoiceModeManager.kt | 17 +++- .../feature/chat/voice/VoiceModeOverlay.kt | 51 ++++++++++++ .../feature/chat/voice/VoiceModeState.kt | 3 +- android/gradle/libs.versions.toml | 7 ++ 10 files changed, 193 insertions(+), 7 deletions(-) create mode 100644 android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/CameraCaptureManager.kt diff --git a/android/app/src/main/java/io/picoclaw/android/di/AppModule.kt b/android/app/src/main/java/io/picoclaw/android/di/AppModule.kt index c0c8162f4..66dc94a59 100644 --- a/android/app/src/main/java/io/picoclaw/android/di/AppModule.kt +++ b/android/app/src/main/java/io/picoclaw/android/di/AppModule.kt @@ -24,6 +24,7 @@ import io.picoclaw.android.feature.chat.ChatViewModel import io.picoclaw.android.feature.chat.SettingsViewModel import io.picoclaw.android.feature.chat.voice.SpeechRecognizerWrapper import io.picoclaw.android.feature.chat.voice.TextToSpeechWrapper +import io.picoclaw.android.feature.chat.voice.CameraCaptureManager import io.picoclaw.android.feature.chat.voice.VoiceModeManager import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -92,7 +93,8 @@ val appModule = module { // Voice factory { SpeechRecognizerWrapper(androidContext()) } single { TextToSpeechWrapper(androidContext(), get().ttsConfig) } - single { VoiceModeManager(get(), get(), get(), get(), get()) } + single { CameraCaptureManager(androidContext()) } + single { VoiceModeManager(get(), get(), get(), get(), get(), get()) } // ViewModel viewModel { ChatViewModel(get(), get(), get(), get(), get(), get(), get(), get()) } diff --git a/android/feature/chat/build.gradle.kts b/android/feature/chat/build.gradle.kts index 338f3b9c9..52bc820be 100644 --- a/android/feature/chat/build.gradle.kts +++ b/android/feature/chat/build.gradle.kts @@ -42,6 +42,11 @@ dependencies { implementation(libs.coil.compose) + implementation(libs.camerax.core) + implementation(libs.camerax.camera2) + implementation(libs.camerax.lifecycle) + implementation(libs.camerax.view) + implementation(libs.markdown.renderer.m3) debugImplementation(libs.compose.ui.tooling) diff --git a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/ChatEvent.kt b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/ChatEvent.kt index 809f1e8a3..897f71c24 100644 --- a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/ChatEvent.kt +++ b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/ChatEvent.kt @@ -13,4 +13,5 @@ sealed interface ChatEvent { data object OnVoiceModeStart : ChatEvent data object OnVoiceModeStop : ChatEvent data object OnVoiceModeInterrupt : ChatEvent + data object OnVoiceCameraToggle : ChatEvent } diff --git a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/ChatViewModel.kt b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/ChatViewModel.kt index 138432b96..13d85c83f 100644 --- a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/ChatViewModel.kt +++ b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/ChatViewModel.kt @@ -108,6 +108,9 @@ class ChatViewModel( is ChatEvent.OnVoiceModeInterrupt -> { voiceModeManager.interrupt() } + is ChatEvent.OnVoiceCameraToggle -> { + voiceModeManager.toggleCamera() + } } } diff --git a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/screen/ChatScreen.kt b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/screen/ChatScreen.kt index 171332ea3..48cf7b156 100644 --- a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/screen/ChatScreen.kt +++ b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/screen/ChatScreen.kt @@ -50,8 +50,10 @@ import io.picoclaw.android.feature.chat.component.ImagePreviewRow import io.picoclaw.android.feature.chat.component.MessageInput import io.picoclaw.android.feature.chat.component.MessageList import io.picoclaw.android.feature.chat.component.StatusIndicator +import io.picoclaw.android.feature.chat.voice.CameraCaptureManager import io.picoclaw.android.feature.chat.voice.VoiceModeOverlay import org.koin.androidx.compose.koinViewModel +import org.koin.compose.koinInject import java.io.File @OptIn(ExperimentalMaterial3Api::class) @@ -63,9 +65,32 @@ fun ChatScreen( val context = LocalContext.current val uiState by viewModel.uiState.collectAsState() val listState = rememberLazyListState() + val cameraCaptureManager: CameraCaptureManager = koinInject() var cameraImageUri by remember { mutableStateOf(null) } + val voiceCameraPermissionLauncher = rememberLauncherForActivityResult( + contract = ActivityResultContracts.RequestPermission() + ) { granted -> + if (granted) { + viewModel.onEvent(ChatEvent.OnVoiceCameraToggle) + } + } + + val onVoiceCameraToggle = { + if (uiState.voiceModeState.isCameraActive) { + viewModel.onEvent(ChatEvent.OnVoiceCameraToggle) + } else { + if (ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) + == PackageManager.PERMISSION_GRANTED + ) { + viewModel.onEvent(ChatEvent.OnVoiceCameraToggle) + } else { + voiceCameraPermissionLauncher.launch(Manifest.permission.CAMERA) + } + } + } + val cameraLauncher = rememberLauncherForActivityResult( contract = ActivityResultContracts.TakePicture() ) { success -> @@ -263,7 +288,9 @@ fun ChatScreen( VoiceModeOverlay( state = uiState.voiceModeState, onClose = { viewModel.onEvent(ChatEvent.OnVoiceModeStop) }, - onInterrupt = { viewModel.onEvent(ChatEvent.OnVoiceModeInterrupt) } + onInterrupt = { viewModel.onEvent(ChatEvent.OnVoiceModeInterrupt) }, + onCameraToggle = onVoiceCameraToggle, + cameraCaptureManager = cameraCaptureManager ) } } diff --git a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/CameraCaptureManager.kt b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/CameraCaptureManager.kt new file mode 100644 index 000000000..d0fbbe202 --- /dev/null +++ b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/CameraCaptureManager.kt @@ -0,0 +1,80 @@ +package io.picoclaw.android.feature.chat.voice + +import android.content.Context +import androidx.camera.core.CameraSelector +import androidx.camera.core.ImageCapture +import androidx.camera.core.ImageCaptureException +import androidx.camera.core.Preview +import androidx.camera.lifecycle.ProcessCameraProvider +import androidx.camera.view.PreviewView +import androidx.core.content.ContextCompat +import androidx.lifecycle.LifecycleOwner +import io.picoclaw.android.core.domain.model.ImageAttachment +import kotlinx.coroutines.suspendCancellableCoroutine +import java.io.File +import kotlin.coroutines.resume + +class CameraCaptureManager(private val context: Context) { + + private var imageCapture: ImageCapture? = null + private var cameraProvider: ProcessCameraProvider? = null + + fun bind(lifecycleOwner: LifecycleOwner, previewView: PreviewView) { + val providerFuture = ProcessCameraProvider.getInstance(context) + providerFuture.addListener({ + val provider = providerFuture.get() + cameraProvider = provider + + val preview = Preview.Builder().build().also { + it.surfaceProvider = previewView.surfaceProvider + } + + val capture = ImageCapture.Builder() + .setCaptureMode(ImageCapture.CAPTURE_MODE_MINIMIZE_LATENCY) + .build() + imageCapture = capture + + provider.unbindAll() + provider.bindToLifecycle( + lifecycleOwner, + CameraSelector.DEFAULT_BACK_CAMERA, + preview, + capture + ) + }, ContextCompat.getMainExecutor(context)) + } + + fun unbind() { + cameraProvider?.unbindAll() + cameraProvider = null + imageCapture = null + } + + suspend fun captureFrame(): ImageAttachment? { + val capture = imageCapture ?: return null + val imagesDir = File(context.cacheDir, "images").apply { mkdirs() } + val file = File(imagesDir, "voice_cam_${System.currentTimeMillis()}.jpg") + val outputOptions = ImageCapture.OutputFileOptions.Builder(file).build() + + return suspendCancellableCoroutine { cont -> + capture.takePicture( + outputOptions, + ContextCompat.getMainExecutor(context), + object : ImageCapture.OnImageSavedCallback { + override fun onImageSaved(output: ImageCapture.OutputFileResults) { + val uri = androidx.core.content.FileProvider.getUriForFile( + context, + "${context.packageName}.fileprovider", + file + ) + cont.resume(ImageAttachment(uri = uri.toString())) + } + + override fun onError(exception: ImageCaptureException) { + cont.resume(null) + } + } + ) + } + } +} diff --git a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeManager.kt b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeManager.kt index 50bb7ae77..55ec53306 100644 --- a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeManager.kt +++ b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeManager.kt @@ -28,7 +28,8 @@ class VoiceModeManager( private val ttsWrapper: TextToSpeechWrapper, private val sendMessage: SendMessageUseCase, private val observeMessages: ObserveMessagesUseCase, - private val observeStatus: ObserveStatusUseCase + private val observeStatus: ObserveStatusUseCase, + private val cameraCaptureManager: CameraCaptureManager ) { private val _state = MutableStateFlow(VoiceModeState()) @@ -37,11 +38,15 @@ class VoiceModeManager( private var loopJob: Job? = null private var parentScope: CoroutineScope? = null + fun toggleCamera() { + _state.update { it.copy(isCameraActive = !it.isCameraActive) } + } + fun start(scope: CoroutineScope) { if (loopJob?.isActive == true) return parentScope = scope _state.update { - VoiceModeState(isActive = true, phase = VoicePhase.LISTENING) + VoiceModeState(isActive = true, phase = VoicePhase.LISTENING, isCameraActive = it.isCameraActive) } loopJob = scope.launch { voiceLoop() @@ -53,6 +58,7 @@ class VoiceModeManager( loopJob = null parentScope = null ttsWrapper.stop() + cameraCaptureManager.unbind() _state.value = VoiceModeState() } @@ -65,7 +71,7 @@ class VoiceModeManager( loopJob = null _state.update { - VoiceModeState(isActive = true, phase = VoicePhase.LISTENING) + VoiceModeState(isActive = true, phase = VoicePhase.LISTENING, isCameraActive = it.isCameraActive) } loopJob = scope.launch { voiceLoop() } } @@ -119,7 +125,10 @@ class VoiceModeManager( if (!text.isNullOrBlank()) { _state.update { it.copy(phase = VoicePhase.SENDING, recognizedText = text) } try { - sendMessage(text, inputMode = "voice") + val images = if (_state.value.isCameraActive) { + listOfNotNull(cameraCaptureManager.captureFrame()) + } else emptyList() + sendMessage(text, images = images, inputMode = "voice") } catch (e: Exception) { _state.update { it.copy(phase = VoicePhase.ERROR, errorMessage = "送信に失敗しました") diff --git a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeOverlay.kt b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeOverlay.kt index 7faf796bc..8d5cb1f02 100644 --- a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeOverlay.kt +++ b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeOverlay.kt @@ -1,5 +1,6 @@ package io.picoclaw.android.feature.chat.voice +import androidx.camera.view.PreviewView import androidx.compose.animation.AnimatedVisibility import androidx.compose.animation.fadeIn import androidx.compose.animation.fadeOut @@ -14,6 +15,8 @@ import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.material3.Icon import androidx.compose.material3.IconButton import androidx.compose.material3.MaterialTheme @@ -21,16 +24,20 @@ import androidx.compose.material3.Text import androidx.compose.foundation.clickable import androidx.compose.foundation.interaction.MutableInteractionSource import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.remember import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip import androidx.compose.ui.draw.drawBehind import androidx.compose.ui.geometry.Offset import androidx.compose.ui.graphics.Brush import androidx.compose.ui.graphics.Color +import androidx.lifecycle.compose.LocalLifecycleOwner import androidx.compose.ui.res.painterResource import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp +import androidx.compose.ui.viewinterop.AndroidView import io.picoclaw.android.core.domain.model.VoicePhase import io.picoclaw.android.core.ui.theme.DeepBlack import io.picoclaw.android.core.ui.theme.GradientCyan @@ -44,6 +51,8 @@ fun VoiceModeOverlay( state: VoiceModeState, onClose: () -> Unit, onInterrupt: () -> Unit, + onCameraToggle: () -> Unit, + cameraCaptureManager: CameraCaptureManager, modifier: Modifier = Modifier ) { AnimatedVisibility( @@ -154,6 +163,48 @@ fun VoiceModeOverlay( textAlign = TextAlign.Center ) } + + Spacer(modifier = Modifier.height(24.dp)) + + AnimatedVisibility(visible = state.isCameraActive) { + val lifecycleOwner = LocalLifecycleOwner.current + + Box( + modifier = Modifier + .width(160.dp) + .height(120.dp) + .clip(RoundedCornerShape(12.dp)) + ) { + AndroidView( + factory = { ctx -> + PreviewView(ctx).also { preview -> + cameraCaptureManager.bind(lifecycleOwner, preview) + } + }, + modifier = Modifier.fillMaxSize() + ) + } + + DisposableEffect(Unit) { + onDispose { + cameraCaptureManager.unbind() + } + } + } + + Spacer(modifier = Modifier.height(16.dp)) + + IconButton(onClick = onCameraToggle) { + Icon( + painter = painterResource( + if (state.isCameraActive) LucideR.drawable.lucide_ic_camera_off + else LucideR.drawable.lucide_ic_camera + ), + contentDescription = if (state.isCameraActive) "Turn off camera" else "Turn on camera", + modifier = Modifier.size(28.dp), + tint = if (state.isCameraActive) GradientCyan else TextSecondary + ) + } } } } diff --git a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeState.kt b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeState.kt index 20e93d6e2..590e2592c 100644 --- a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeState.kt +++ b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeState.kt @@ -9,5 +9,6 @@ data class VoiceModeState( val responseText: String = "", val statusText: String? = null, val errorMessage: String? = null, - val amplitudeNormalized: Float = 0f + val amplitudeNormalized: Float = 0f, + val isCameraActive: Boolean = false ) diff --git a/android/gradle/libs.versions.toml b/android/gradle/libs.versions.toml index e507eb332..29357efb5 100644 --- a/android/gradle/libs.versions.toml +++ b/android/gradle/libs.versions.toml @@ -15,6 +15,7 @@ coil = "3.1.0" markdown-renderer = "0.39.0" navigation = "2.9.0" datastore = "1.1.7" +camerax = "1.5.1" [libraries] # Compose @@ -61,6 +62,12 @@ navigation-compose = { group = "androidx.navigation", name = "navigation-compose # DataStore datastore-preferences = { group = "androidx.datastore", name = "datastore-preferences", version.ref = "datastore" } +# CameraX +camerax-core = { group = "androidx.camera", name = "camera-core", version.ref = "camerax" } +camerax-camera2 = { group = "androidx.camera", name = "camera-camera2", version.ref = "camerax" } +camerax-lifecycle = { group = "androidx.camera", name = "camera-lifecycle", version.ref = "camerax" } +camerax-view = { group = "androidx.camera", name = "camera-view", version.ref = "camerax" } + # Markdown markdown-renderer-m3 = { group = "com.mikepenz", name = "multiplatform-markdown-renderer-m3", version.ref = "markdown-renderer" }