opencode/picoclaw
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add camera capture during voice mode conversations

Browse source 
Allow attaching camera images to messages sent via voice mode.
A camera toggle button in VoiceModeOverlay shows/hides a live
CameraX preview, and automatically captures a frame when speech
recognition completes in the SENDING phase.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
KoheiYamashita 2026-02-20 10:35:02 +09:00
parent 34df2490f5
commit 643aa0d47c
10 changed files with 193 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
android/app/src/main/java/io/picoclaw/android/di/AppModule.kt
View file

@ -24,6 +24,7 @@ import io.picoclaw.android.feature.chat.ChatViewModel
import io.picoclaw.android.feature.chat.SettingsViewModel
import io.picoclaw.android.feature.chat.voice.SpeechRecognizerWrapper
import io.picoclaw.android.feature.chat.voice.TextToSpeechWrapper
import io.picoclaw.android.feature.chat.voice.CameraCaptureManager
import io.picoclaw.android.feature.chat.voice.VoiceModeManager
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
@ -92,7 +93,8 @@ val appModule = module {
// Voice
factory { SpeechRecognizerWrapper(androidContext()) }
single { TextToSpeechWrapper(androidContext(), get<TtsSettingsRepository>().ttsConfig) }
single { VoiceModeManager(get(), get(), get(), get(), get()) }
single { CameraCaptureManager(androidContext()) }
single { VoiceModeManager(get(), get(), get(), get(), get(), get()) }
// ViewModel
viewModel { ChatViewModel(get(), get(), get(), get(), get(), get(), get(), get()) }

5
android/feature/chat/build.gradle.kts
View file

@ -42,6 +42,11 @@ dependencies {
implementation(libs.coil.compose)
implementation(libs.camerax.core)
implementation(libs.camerax.camera2)
implementation(libs.camerax.lifecycle)
implementation(libs.camerax.view)
implementation(libs.markdown.renderer.m3)
debugImplementation(libs.compose.ui.tooling)

1
android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/ChatEvent.kt
View file

@ -13,4 +13,5 @@ sealed interface ChatEvent {
data object OnVoiceModeStart : ChatEvent
data object OnVoiceModeStop : ChatEvent
data object OnVoiceModeInterrupt : ChatEvent
data object OnVoiceCameraToggle : ChatEvent
}

3
android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/ChatViewModel.kt
View file

@ -108,6 +108,9 @@ class ChatViewModel(
is ChatEvent.OnVoiceModeInterrupt -> {
voiceModeManager.interrupt()
}
is ChatEvent.OnVoiceCameraToggle -> {
voiceModeManager.toggleCamera()
}
}
}

29
android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/screen/ChatScreen.kt
View file

@ -50,8 +50,10 @@ import io.picoclaw.android.feature.chat.component.ImagePreviewRow
import io.picoclaw.android.feature.chat.component.MessageInput
import io.picoclaw.android.feature.chat.component.MessageList
import io.picoclaw.android.feature.chat.component.StatusIndicator
import io.picoclaw.android.feature.chat.voice.CameraCaptureManager
import io.picoclaw.android.feature.chat.voice.VoiceModeOverlay
import org.koin.androidx.compose.koinViewModel
import org.koin.compose.koinInject
import java.io.File
@OptIn(ExperimentalMaterial3Api::class)
@ -63,9 +65,32 @@ fun ChatScreen(
val context = LocalContext.current
val uiState by viewModel.uiState.collectAsState()
val listState = rememberLazyListState()
val cameraCaptureManager: CameraCaptureManager = koinInject()
var cameraImageUri by remember { mutableStateOf<Uri?>(null) }
val voiceCameraPermissionLauncher = rememberLauncherForActivityResult(
contract = ActivityResultContracts.RequestPermission()
) { granted ->
if (granted) {
viewModel.onEvent(ChatEvent.OnVoiceCameraToggle)
}
}
val onVoiceCameraToggle = {
if (uiState.voiceModeState.isCameraActive) {
viewModel.onEvent(ChatEvent.OnVoiceCameraToggle)
} else {
if (ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA)
== PackageManager.PERMISSION_GRANTED
) {
viewModel.onEvent(ChatEvent.OnVoiceCameraToggle)
} else {
voiceCameraPermissionLauncher.launch(Manifest.permission.CAMERA)
}
}
}
val cameraLauncher = rememberLauncherForActivityResult(
contract = ActivityResultContracts.TakePicture()
) { success ->
@ -263,7 +288,9 @@ fun ChatScreen(
VoiceModeOverlay(
state = uiState.voiceModeState,
onClose = { viewModel.onEvent(ChatEvent.OnVoiceModeStop) },
onInterrupt = { viewModel.onEvent(ChatEvent.OnVoiceModeInterrupt) }
onInterrupt = { viewModel.onEvent(ChatEvent.OnVoiceModeInterrupt) },
onCameraToggle = onVoiceCameraToggle,
cameraCaptureManager = cameraCaptureManager
)
}
}

80
android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/CameraCaptureManager.kt Normal file
View file

@ -0,0 +1,80 @@
package io.picoclaw.android.feature.chat.voice
import android.content.Context
import androidx.camera.core.CameraSelector
import androidx.camera.core.ImageCapture
import androidx.camera.core.ImageCaptureException
import androidx.camera.core.Preview
import androidx.camera.lifecycle.ProcessCameraProvider
import androidx.camera.view.PreviewView
import androidx.core.content.ContextCompat
import androidx.lifecycle.LifecycleOwner
import io.picoclaw.android.core.domain.model.ImageAttachment
import kotlinx.coroutines.suspendCancellableCoroutine
import java.io.File
import kotlin.coroutines.resume
class CameraCaptureManager(private val context: Context) {
private var imageCapture: ImageCapture? = null
private var cameraProvider: ProcessCameraProvider? = null
fun bind(lifecycleOwner: LifecycleOwner, previewView: PreviewView) {
val providerFuture = ProcessCameraProvider.getInstance(context)
providerFuture.addListener({
val provider = providerFuture.get()
cameraProvider = provider
val preview = Preview.Builder().build().also {
it.surfaceProvider = previewView.surfaceProvider
}
val capture = ImageCapture.Builder()
.setCaptureMode(ImageCapture.CAPTURE_MODE_MINIMIZE_LATENCY)
.build()
imageCapture = capture
provider.unbindAll()
provider.bindToLifecycle(
lifecycleOwner,
CameraSelector.DEFAULT_BACK_CAMERA,
preview,
capture
)
}, ContextCompat.getMainExecutor(context))
}
fun unbind() {
cameraProvider?.unbindAll()
cameraProvider = null
imageCapture = null
}
suspend fun captureFrame(): ImageAttachment? {
val capture = imageCapture ?: return null
val imagesDir = File(context.cacheDir, "images").apply { mkdirs() }
val file = File(imagesDir, "voice_cam_${System.currentTimeMillis()}.jpg")
val outputOptions = ImageCapture.OutputFileOptions.Builder(file).build()
return suspendCancellableCoroutine { cont ->
capture.takePicture(
outputOptions,
ContextCompat.getMainExecutor(context),
object : ImageCapture.OnImageSavedCallback {
override fun onImageSaved(output: ImageCapture.OutputFileResults) {
val uri = androidx.core.content.FileProvider.getUriForFile(
context,
"${context.packageName}.fileprovider",
file
)
cont.resume(ImageAttachment(uri = uri.toString()))
}
override fun onError(exception: ImageCaptureException) {
cont.resume(null)
}
}
)
}
}
}

17
android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeManager.kt
View file

@ -28,7 +28,8 @@ class VoiceModeManager(
private val ttsWrapper: TextToSpeechWrapper,
private val sendMessage: SendMessageUseCase,
private val observeMessages: ObserveMessagesUseCase,
private val observeStatus: ObserveStatusUseCase
private val observeStatus: ObserveStatusUseCase,
private val cameraCaptureManager: CameraCaptureManager
) {
private val _state = MutableStateFlow(VoiceModeState())
@ -37,11 +38,15 @@ class VoiceModeManager(
private var loopJob: Job? = null
private var parentScope: CoroutineScope? = null
fun toggleCamera() {
_state.update { it.copy(isCameraActive = !it.isCameraActive) }
}
fun start(scope: CoroutineScope) {
if (loopJob?.isActive == true) return
parentScope = scope
_state.update {
VoiceModeState(isActive = true, phase = VoicePhase.LISTENING)
VoiceModeState(isActive = true, phase = VoicePhase.LISTENING, isCameraActive = it.isCameraActive)
}
loopJob = scope.launch {
voiceLoop()
@ -53,6 +58,7 @@ class VoiceModeManager(
loopJob = null
parentScope = null
ttsWrapper.stop()
cameraCaptureManager.unbind()
_state.value = VoiceModeState()
}
@ -65,7 +71,7 @@ class VoiceModeManager(
loopJob = null
_state.update {
VoiceModeState(isActive = true, phase = VoicePhase.LISTENING)
VoiceModeState(isActive = true, phase = VoicePhase.LISTENING, isCameraActive = it.isCameraActive)
}
loopJob = scope.launch { voiceLoop() }
}
@ -119,7 +125,10 @@ class VoiceModeManager(
if (!text.isNullOrBlank()) {
_state.update { it.copy(phase = VoicePhase.SENDING, recognizedText = text) }
try {
sendMessage(text, inputMode = "voice")
val images = if (_state.value.isCameraActive) {
listOfNotNull(cameraCaptureManager.captureFrame())
} else emptyList()
sendMessage(text, images = images, inputMode = "voice")
} catch (e: Exception) {
_state.update {
it.copy(phase = VoicePhase.ERROR, errorMessage = "送信に失敗しました")

51
android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeOverlay.kt
View file

@ -1,5 +1,6 @@
package io.picoclaw.android.feature.chat.voice
import androidx.camera.view.PreviewView
import androidx.compose.animation.AnimatedVisibility
import androidx.compose.animation.fadeIn
import androidx.compose.animation.fadeOut
@ -14,6 +15,8 @@ import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButton
import androidx.compose.material3.MaterialTheme
@ -21,16 +24,20 @@ import androidx.compose.material3.Text
import androidx.compose.foundation.clickable
import androidx.compose.foundation.interaction.MutableInteractionSource
import androidx.compose.runtime.Composable
import androidx.compose.runtime.DisposableEffect
import androidx.compose.runtime.remember
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.draw.drawBehind
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.graphics.Brush
import androidx.compose.ui.graphics.Color
import androidx.lifecycle.compose.LocalLifecycleOwner
import androidx.compose.ui.res.painterResource
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.unit.dp
import androidx.compose.ui.viewinterop.AndroidView
import io.picoclaw.android.core.domain.model.VoicePhase
import io.picoclaw.android.core.ui.theme.DeepBlack
import io.picoclaw.android.core.ui.theme.GradientCyan
@ -44,6 +51,8 @@ fun VoiceModeOverlay(
state: VoiceModeState,
onClose: () -> Unit,
onInterrupt: () -> Unit,
onCameraToggle: () -> Unit,
cameraCaptureManager: CameraCaptureManager,
modifier: Modifier = Modifier
) {
AnimatedVisibility(
@ -154,6 +163,48 @@ fun VoiceModeOverlay(
textAlign = TextAlign.Center
)
}
Spacer(modifier = Modifier.height(24.dp))
AnimatedVisibility(visible = state.isCameraActive) {
val lifecycleOwner = LocalLifecycleOwner.current
Box(
modifier = Modifier
.width(160.dp)
.height(120.dp)
.clip(RoundedCornerShape(12.dp))
) {
AndroidView(
factory = { ctx ->
PreviewView(ctx).also { preview ->
cameraCaptureManager.bind(lifecycleOwner, preview)
}
},
modifier = Modifier.fillMaxSize()
)
}
DisposableEffect(Unit) {
onDispose {
cameraCaptureManager.unbind()
}
}
}
Spacer(modifier = Modifier.height(16.dp))
IconButton(onClick = onCameraToggle) {
Icon(
painter = painterResource(
if (state.isCameraActive) LucideR.drawable.lucide_ic_camera_off
else LucideR.drawable.lucide_ic_camera
),
contentDescription = if (state.isCameraActive) "Turn off camera" else "Turn on camera",
modifier = Modifier.size(28.dp),
tint = if (state.isCameraActive) GradientCyan else TextSecondary
)
}
}
}
}

3
android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeState.kt
View file

@ -9,5 +9,6 @@ data class VoiceModeState(
val responseText: String = "",
val statusText: String? = null,
val errorMessage: String? = null,
val amplitudeNormalized: Float = 0f
val amplitudeNormalized: Float = 0f,
val isCameraActive: Boolean = false
)

7
android/gradle/libs.versions.toml
View file

@ -15,6 +15,7 @@ coil = "3.1.0"
markdown-renderer = "0.39.0"
navigation = "2.9.0"
datastore = "1.1.7"
camerax = "1.5.1"
[libraries]
# Compose
@ -61,6 +62,12 @@ navigation-compose = { group = "androidx.navigation", name = "navigation-compose
# DataStore
datastore-preferences = { group = "androidx.datastore", name = "datastore-preferences", version.ref = "datastore" }
# CameraX
camerax-core = { group = "androidx.camera", name = "camera-core", version.ref = "camerax" }
camerax-camera2 = { group = "androidx.camera", name = "camera-camera2", version.ref = "camerax" }
camerax-lifecycle = { group = "androidx.camera", name = "camera-lifecycle", version.ref = "camerax" }
camerax-view = { group = "androidx.camera", name = "camera-view", version.ref = "camerax" }
# Markdown
markdown-renderer-m3 = { group = "com.mikepenz", name = "multiplatform-markdown-renderer-m3", version.ref = "markdown-renderer" }