diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index f170e9b1e..66bcede73 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -5,6 +5,9 @@ + + + @@ -47,6 +50,27 @@ + + + + + + + + + + + diff --git a/android/app/src/main/java/io/picoclaw/android/assistant/AssistantActivity.kt b/android/app/src/main/java/io/picoclaw/android/assistant/AssistantActivity.kt new file mode 100644 index 000000000..5d7cbd2d6 --- /dev/null +++ b/android/app/src/main/java/io/picoclaw/android/assistant/AssistantActivity.kt @@ -0,0 +1,65 @@ +package io.picoclaw.android.assistant + +import android.Manifest +import android.content.Intent +import android.content.pm.PackageManager +import android.net.Uri +import android.os.Bundle +import android.provider.Settings +import androidx.activity.ComponentActivity +import androidx.activity.result.contract.ActivityResultContracts +import androidx.core.content.ContextCompat + +class AssistantActivity : ComponentActivity() { + + private val audioPermissionLauncher = registerForActivityResult( + ActivityResultContracts.RequestPermission() + ) { granted -> + if (granted) { + launchServiceAndFinish() + } else { + finish() + } + } + + private val overlayPermissionLauncher = registerForActivityResult( + ActivityResultContracts.StartActivityForResult() + ) { + if (Settings.canDrawOverlays(this)) { + checkAudioPermission() + } else { + finish() + } + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + + if (!Settings.canDrawOverlays(this)) { + val intent = Intent( + Settings.ACTION_MANAGE_OVERLAY_PERMISSION, + Uri.parse("package:$packageName") + ) + overlayPermissionLauncher.launch(intent) + return + } + + checkAudioPermission() + } + + private fun checkAudioPermission() { + if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) + != PackageManager.PERMISSION_GRANTED + ) { + audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) + } else { + launchServiceAndFinish() + } + } + + private fun launchServiceAndFinish() { + val serviceIntent = Intent(this, AssistantService::class.java) + startForegroundService(serviceIntent) + finish() + } +} diff --git a/android/app/src/main/java/io/picoclaw/android/assistant/AssistantService.kt b/android/app/src/main/java/io/picoclaw/android/assistant/AssistantService.kt new file mode 100644 index 000000000..d784eb91a --- /dev/null +++ b/android/app/src/main/java/io/picoclaw/android/assistant/AssistantService.kt @@ -0,0 +1,168 @@ +package io.picoclaw.android.assistant + +import android.app.Notification +import android.content.Intent +import android.graphics.PixelFormat +import android.os.IBinder +import android.view.Gravity +import android.view.WindowManager +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.ui.platform.ComposeView +import androidx.core.app.NotificationCompat +import androidx.lifecycle.LifecycleService +import androidx.lifecycle.setViewTreeLifecycleOwner +import androidx.savedstate.SavedStateRegistry +import androidx.savedstate.SavedStateRegistryController +import androidx.savedstate.SavedStateRegistryOwner +import androidx.savedstate.setViewTreeSavedStateRegistryOwner +import io.ktor.client.HttpClient +import io.picoclaw.android.core.data.remote.WebSocketClient +import io.picoclaw.android.core.data.repository.AssistantConnectionImpl +import io.picoclaw.android.core.domain.repository.AssistantConnection +import io.picoclaw.android.core.domain.repository.TtsSettingsRepository +import io.picoclaw.android.core.ui.theme.PicoClawTheme +import io.picoclaw.android.feature.chat.assistant.AssistantManager +import io.picoclaw.android.feature.chat.assistant.AssistantPillBar +import io.picoclaw.android.feature.chat.voice.CameraCaptureManager +import io.picoclaw.android.feature.chat.voice.SpeechRecognizerWrapper +import io.picoclaw.android.feature.chat.voice.TextToSpeechWrapper +import io.picoclaw.android.receiver.NotificationHelper +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import org.koin.android.ext.android.inject + +class AssistantService : LifecycleService(), SavedStateRegistryOwner { + + private val httpClient: HttpClient by inject() + private val ttsSettingsRepo: TtsSettingsRepository by inject() + + private lateinit var serviceScope: CoroutineScope + private lateinit var connection: AssistantConnection + private lateinit var assistantManager: AssistantManager + private lateinit var ttsWrapper: TextToSpeechWrapper + private lateinit var sttWrapper: SpeechRecognizerWrapper + private lateinit var cameraCaptureManager: CameraCaptureManager + + private var overlayView: ComposeView? = null + private val windowManager by lazy { getSystemService(WINDOW_SERVICE) as WindowManager } + + private val savedStateRegistryController = SavedStateRegistryController.create(this) + override val savedStateRegistry: SavedStateRegistry + get() = savedStateRegistryController.savedStateRegistry + + override fun onCreate() { + savedStateRegistryController.performAttach() + savedStateRegistryController.performRestore(null) + super.onCreate() + + serviceScope = CoroutineScope(SupervisorJob() + Dispatchers.Main) + + connection = AssistantConnectionImpl(httpClient) + + sttWrapper = SpeechRecognizerWrapper(this) + ttsWrapper = TextToSpeechWrapper(this, ttsSettingsRepo.ttsConfig) + cameraCaptureManager = CameraCaptureManager(this) + + assistantManager = AssistantManager( + sttWrapper = sttWrapper, + ttsWrapper = ttsWrapper, + connection = connection, + cameraCaptureManager = cameraCaptureManager, + contentResolver = contentResolver + ) + } + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + super.onStartCommand(intent, flags, startId) + + startForeground(NOTIFICATION_ID, buildNotification()) + + // Resolve wsUrl from the main WebSocketClient + val mainWsClient: WebSocketClient by inject() + connection.connect(mainWsClient.wsUrl) + + addOverlay() + assistantManager.start(serviceScope) + + return START_NOT_STICKY + } + + override fun onDestroy() { + removeOverlay() + assistantManager.destroy() + ttsWrapper.destroy() + connection.disconnect() + serviceScope.cancel() + super.onDestroy() + } + + override fun onBind(intent: Intent): IBinder? { + super.onBind(intent) + return null + } + + private fun shutdown() { + stopForeground(STOP_FOREGROUND_REMOVE) + stopSelf() + } + + private fun addOverlay() { + if (overlayView != null) return + + val params = WindowManager.LayoutParams( + WindowManager.LayoutParams.MATCH_PARENT, + WindowManager.LayoutParams.WRAP_CONTENT, + WindowManager.LayoutParams.TYPE_APPLICATION_OVERLAY, + WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL or + WindowManager.LayoutParams.FLAG_LAYOUT_IN_SCREEN, + PixelFormat.TRANSLUCENT + ).apply { + gravity = Gravity.BOTTOM + } + + val view = ComposeView(this).apply { + setViewTreeLifecycleOwner(this@AssistantService) + setViewTreeSavedStateRegistryOwner(this@AssistantService) + + setContent { + PicoClawTheme { + val state by assistantManager.state.collectAsState() + AssistantPillBar( + state = state, + onClose = { shutdown() }, + onInterrupt = { assistantManager.interrupt() }, + onCameraToggle = { assistantManager.toggleCamera() }, + cameraCaptureManager = cameraCaptureManager + ) + } + } + } + + windowManager.addView(view, params) + overlayView = view + } + + private fun removeOverlay() { + overlayView?.let { + windowManager.removeView(it) + overlayView = null + } + } + + private fun buildNotification(): Notification { + return NotificationCompat.Builder(this, NotificationHelper.ASSISTANT_CHANNEL_ID) + .setSmallIcon(android.R.drawable.ic_btn_speak_now) + .setContentTitle("PicoClaw Assistant") + .setContentText("Listening...") + .setPriority(NotificationCompat.PRIORITY_LOW) + .setOngoing(true) + .build() + } + + companion object { + private const val NOTIFICATION_ID = 2001 + } +} diff --git a/android/app/src/main/java/io/picoclaw/android/receiver/NotificationHelper.kt b/android/app/src/main/java/io/picoclaw/android/receiver/NotificationHelper.kt index 9d0501507..8a3952455 100644 --- a/android/app/src/main/java/io/picoclaw/android/receiver/NotificationHelper.kt +++ b/android/app/src/main/java/io/picoclaw/android/receiver/NotificationHelper.kt @@ -12,16 +12,30 @@ object NotificationHelper { private const val CHANNEL_NAME = "Agent Messages" private const val NOTIFICATION_ID = 1001 + const val ASSISTANT_CHANNEL_ID = "picoclaw_assistant" + private const val ASSISTANT_CHANNEL_NAME = "Assistant" + fun createNotificationChannel(context: Context) { - val channel = NotificationChannel( + val manager = context.getSystemService(NotificationManager::class.java) + + val messageChannel = NotificationChannel( CHANNEL_ID, CHANNEL_NAME, NotificationManager.IMPORTANCE_DEFAULT ).apply { description = "Messages from PicoClaw agent" } - val manager = context.getSystemService(NotificationManager::class.java) - manager.createNotificationChannel(channel) + manager.createNotificationChannel(messageChannel) + + val assistantChannel = NotificationChannel( + ASSISTANT_CHANNEL_ID, + ASSISTANT_CHANNEL_NAME, + NotificationManager.IMPORTANCE_LOW + ).apply { + description = "Assistant overlay service" + setShowBadge(false) + } + manager.createNotificationChannel(assistantChannel) } fun showMessageNotification(context: Context, content: String) { diff --git a/android/core/data/src/main/java/io/picoclaw/android/core/data/remote/WebSocketClient.kt b/android/core/data/src/main/java/io/picoclaw/android/core/data/remote/WebSocketClient.kt index b560ad45c..693d428b2 100644 --- a/android/core/data/src/main/java/io/picoclaw/android/core/data/remote/WebSocketClient.kt +++ b/android/core/data/src/main/java/io/picoclaw/android/core/data/remote/WebSocketClient.kt @@ -26,7 +26,8 @@ import kotlinx.serialization.json.Json class WebSocketClient( private val client: HttpClient, private val scope: CoroutineScope, - private val clientId: String + private val clientId: String, + private val clientType: String = "main" ) { private val _connectionState = MutableStateFlow(ConnectionState.DISCONNECTED) @@ -48,7 +49,7 @@ class WebSocketClient( while (isActive) { try { _connectionState.value = ConnectionState.CONNECTING - val url = "$wsUrl?client_id=$clientId&client_type=main" + val url = "$wsUrl?client_id=$clientId&client_type=$clientType" client.webSocket(url) { session = this _connectionState.value = ConnectionState.CONNECTED diff --git a/android/core/data/src/main/java/io/picoclaw/android/core/data/repository/AssistantConnectionImpl.kt b/android/core/data/src/main/java/io/picoclaw/android/core/data/repository/AssistantConnectionImpl.kt new file mode 100644 index 000000000..289a0566b --- /dev/null +++ b/android/core/data/src/main/java/io/picoclaw/android/core/data/repository/AssistantConnectionImpl.kt @@ -0,0 +1,71 @@ +package io.picoclaw.android.core.data.repository + +import io.ktor.client.HttpClient +import io.picoclaw.android.core.data.remote.WebSocketClient +import io.picoclaw.android.core.data.remote.dto.WsIncoming +import io.picoclaw.android.core.domain.model.AssistantMessage +import io.picoclaw.android.core.domain.model.ConnectionState +import io.picoclaw.android.core.domain.repository.AssistantConnection +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharedFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asSharedFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import java.util.UUID + +class AssistantConnectionImpl( + private val httpClient: HttpClient +) : AssistantConnection { + + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val clientId = UUID.randomUUID().toString() + private val wsClient = WebSocketClient(httpClient, scope, clientId, "assistant") + + private val _messages = MutableSharedFlow(extraBufferCapacity = 64) + override val messages: SharedFlow = _messages.asSharedFlow() + + private val _statusText = MutableStateFlow(null) + override val statusText: StateFlow = _statusText.asStateFlow() + + override val connectionState: StateFlow = wsClient.connectionState + + init { + scope.launch { + wsClient.incomingMessages.collect { dto -> + when (dto.type) { + "status" -> _statusText.value = dto.content + "status_end" -> _statusText.value = null + else -> { + _statusText.value = null + _messages.emit(AssistantMessage(content = dto.content, type = dto.type)) + } + } + } + } + } + + override fun connect(wsUrl: String) { + wsClient.wsUrl = wsUrl + wsClient.connect() + } + + override fun disconnect() { + wsClient.disconnect() + scope.cancel() + } + + override suspend fun send(text: String, images: List, inputMode: String) { + val dto = WsIncoming( + content = text, + images = images.ifEmpty { null }, + inputMode = inputMode + ) + wsClient.send(dto) + } +} diff --git a/android/core/domain/src/main/java/io/picoclaw/android/core/domain/model/AssistantMessage.kt b/android/core/domain/src/main/java/io/picoclaw/android/core/domain/model/AssistantMessage.kt new file mode 100644 index 000000000..df1a8c473 --- /dev/null +++ b/android/core/domain/src/main/java/io/picoclaw/android/core/domain/model/AssistantMessage.kt @@ -0,0 +1,6 @@ +package io.picoclaw.android.core.domain.model + +data class AssistantMessage( + val content: String, + val type: String? = null +) diff --git a/android/core/domain/src/main/java/io/picoclaw/android/core/domain/repository/AssistantConnection.kt b/android/core/domain/src/main/java/io/picoclaw/android/core/domain/repository/AssistantConnection.kt new file mode 100644 index 000000000..db84d3164 --- /dev/null +++ b/android/core/domain/src/main/java/io/picoclaw/android/core/domain/repository/AssistantConnection.kt @@ -0,0 +1,16 @@ +package io.picoclaw.android.core.domain.repository + +import io.picoclaw.android.core.domain.model.AssistantMessage +import io.picoclaw.android.core.domain.model.ConnectionState +import kotlinx.coroutines.flow.SharedFlow +import kotlinx.coroutines.flow.StateFlow + +interface AssistantConnection { + val messages: SharedFlow + val statusText: StateFlow + val connectionState: StateFlow + + fun connect(wsUrl: String) + fun disconnect() + suspend fun send(text: String, images: List = emptyList(), inputMode: String = "assistant") +} diff --git a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/assistant/AssistantManager.kt b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/assistant/AssistantManager.kt new file mode 100644 index 000000000..28995ae11 --- /dev/null +++ b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/assistant/AssistantManager.kt @@ -0,0 +1,275 @@ +package io.picoclaw.android.feature.chat.assistant + +import android.content.ContentResolver +import android.speech.SpeechRecognizer +import android.util.Base64 +import android.util.Log +import io.picoclaw.android.core.domain.model.AssistantMessage +import io.picoclaw.android.core.domain.model.VoicePhase +import io.picoclaw.android.core.domain.repository.AssistantConnection +import io.picoclaw.android.feature.chat.voice.CameraCaptureManager +import io.picoclaw.android.feature.chat.voice.SpeechRecognizerWrapper +import io.picoclaw.android.feature.chat.voice.SttResult +import io.picoclaw.android.feature.chat.voice.TextToSpeechWrapper +import io.picoclaw.android.feature.chat.voice.VoiceModeState +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.Job +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.selects.onTimeout +import kotlinx.coroutines.selects.select + +@OptIn(ExperimentalCoroutinesApi::class) +class AssistantManager( + private val sttWrapper: SpeechRecognizerWrapper, + private val ttsWrapper: TextToSpeechWrapper, + private val connection: AssistantConnection, + private val cameraCaptureManager: CameraCaptureManager, + private val contentResolver: ContentResolver +) { + + private val _state = MutableStateFlow(VoiceModeState()) + val state: StateFlow = _state.asStateFlow() + + private var loopJob: Job? = null + private var parentScope: CoroutineScope? = null + + fun toggleCamera() { + _state.update { it.copy(isCameraActive = !it.isCameraActive) } + } + + fun start(scope: CoroutineScope) { + if (loopJob?.isActive == true) return + parentScope = scope + _state.update { + VoiceModeState(isActive = true, phase = VoicePhase.LISTENING, isCameraActive = it.isCameraActive) + } + loopJob = scope.launch { + voiceLoop() + } + } + + fun stop() { + loopJob?.cancel() + loopJob = null + parentScope = null + ttsWrapper.stop() + cameraCaptureManager.unbind() + _state.value = VoiceModeState() + } + + fun interrupt() { + val scope = parentScope ?: return + if (loopJob?.isActive != true) return + + ttsWrapper.stop() + loopJob?.cancel() + loopJob = null + + _state.update { + VoiceModeState(isActive = true, phase = VoicePhase.LISTENING, isCameraActive = it.isCameraActive) + } + loopJob = scope.launch { voiceLoop() } + } + + fun destroy() { + stop() + } + + private suspend fun voiceLoop() = coroutineScope { + val speechQueue = Channel(Channel.UNLIMITED) + + val messageCollectorJob = launch { + connection.messages.collect { msg: AssistantMessage -> + if (msg.type == "warning" || msg.type == "error") { + _state.update { it.copy(responseText = msg.content) } + } else { + speechQueue.send(msg.content) + } + } + } + + val statusCollectorJob = launch { + connection.statusText.collect { label -> + _state.update { it.copy(statusText = label) } + } + } + + try { + while (isActive) { + _state.update { + it.copy( + phase = VoicePhase.LISTENING, + recognizedText = "", responseText = "", + statusText = null, + errorMessage = null, amplitudeNormalized = 0f + ) + } + + val userTextChannel = Channel(1) + val listenJob = launch { + val text = listen() + userTextChannel.send(text) + } + + select { + userTextChannel.onReceive { text -> + if (!text.isNullOrBlank()) { + _state.update { it.copy(phase = VoicePhase.SENDING, recognizedText = text) } + try { + val base64Images = if (_state.value.isCameraActive) { + captureAndEncode() + } else emptyList() + connection.send(text, base64Images) + } catch (e: Exception) { + Log.w(TAG, "Failed to send message", e) + _state.update { + it.copy(phase = VoicePhase.ERROR, errorMessage = "Failed to send") + } + delay(2000) + return@onReceive + } + awaitAndSpeakResponse(speechQueue) + } else { + drainSpeechQueue(speechQueue) + } + } + speechQueue.onReceive { content -> + listenJob.cancel() + speakAndDrain(content, speechQueue) + } + } + } + } finally { + messageCollectorJob.cancel() + statusCollectorJob.cancel() + speechQueue.close() + } + } + + private sealed interface WaitResult { + data class Message(val content: String) : WaitResult + data object Heartbeat : WaitResult + data object Timeout : WaitResult + } + + private suspend fun awaitAndSpeakResponse( + speechQueue: Channel + ) = coroutineScope { + val heartbeat = Channel(Channel.CONFLATED) + + val statusJob = launch { + connection.statusText.collect { label -> + if (label != null) heartbeat.trySend(Unit) + } + } + + try { + _state.update { it.copy(phase = VoicePhase.THINKING, statusText = null) } + + while (true) { + val result = select { + speechQueue.onReceive { WaitResult.Message(it) } + heartbeat.onReceive { WaitResult.Heartbeat } + onTimeout(30_000) { WaitResult.Timeout } + } + + when (result) { + WaitResult.Timeout -> { + _state.update { + it.copy(phase = VoicePhase.ERROR, errorMessage = "Response timed out") + } + delay(2000) + return@coroutineScope + } + WaitResult.Heartbeat -> continue + is WaitResult.Message -> { + speakAndDrain(result.content, speechQueue) + val currentStatus = connection.statusText.value + if (currentStatus == null) return@coroutineScope + _state.update { + it.copy(phase = VoicePhase.THINKING, statusText = currentStatus) + } + } + } + } + } finally { + statusJob.cancel() + } + } + + private suspend fun speakAndDrain(firstContent: String, speechQueue: Channel) { + _state.update { it.copy(phase = VoicePhase.SPEAKING, responseText = firstContent) } + ttsWrapper.speak(firstContent) + while (true) { + val next = speechQueue.tryReceive().getOrNull() ?: break + _state.update { it.copy(responseText = next) } + ttsWrapper.speak(next) + } + } + + private suspend fun drainSpeechQueue(speechQueue: Channel) { + val first = speechQueue.tryReceive().getOrNull() ?: return + speakAndDrain(first, speechQueue) + } + + private suspend fun listen(): String? { + var finalText: String? = null + sttWrapper.startListening().collect { result -> + when (result) { + is SttResult.Partial -> { + _state.update { it.copy(recognizedText = result.text) } + } + is SttResult.Final -> { + finalText = result.text + } + is SttResult.RmsChanged -> { + val normalized = ((result.rms + 2f) / 12f).coerceIn(0f, 1f) + _state.update { it.copy(amplitudeNormalized = normalized) } + } + is SttResult.Error -> { + if (result.code == SpeechRecognizer.ERROR_NO_MATCH || + result.code == SpeechRecognizer.ERROR_SPEECH_TIMEOUT + ) { + finalText = "" + } else { + _state.update { + it.copy( + phase = VoicePhase.ERROR, + errorMessage = "Speech recognition error (code=${result.code})" + ) + } + delay(2000) + finalText = null + } + } + } + } + return finalText + } + + private suspend fun captureAndEncode(): List { + val attachment = cameraCaptureManager.captureFrame() ?: return emptyList() + val uri = android.net.Uri.parse(attachment.uri) + return try { + val bytes = contentResolver.openInputStream(uri)?.use { it.readBytes() } + ?: return emptyList() + listOf(Base64.encodeToString(bytes, Base64.NO_WRAP)) + } catch (e: Exception) { + Log.w(TAG, "Failed to encode captured image", e) + emptyList() + } + } + + companion object { + private const val TAG = "AssistantManager" + } +} diff --git a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/assistant/AssistantPillBar.kt b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/assistant/AssistantPillBar.kt new file mode 100644 index 000000000..a94b502b8 --- /dev/null +++ b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/assistant/AssistantPillBar.kt @@ -0,0 +1,312 @@ +package io.picoclaw.android.feature.chat.assistant + +import androidx.camera.view.PreviewView +import androidx.compose.animation.AnimatedVisibility +import androidx.compose.animation.animateContentSize +import androidx.compose.animation.core.LinearEasing +import androidx.compose.animation.core.RepeatMode +import androidx.compose.animation.core.animateFloat +import androidx.compose.animation.core.infiniteRepeatable +import androidx.compose.animation.core.rememberInfiniteTransition +import androidx.compose.animation.core.tween +import androidx.compose.animation.expandVertically +import androidx.compose.animation.shrinkVertically +import androidx.compose.foundation.Canvas +import androidx.compose.foundation.background +import androidx.compose.foundation.clickable +import androidx.compose.foundation.interaction.MutableInteractionSource +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.remember +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.graphics.Brush +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.res.painterResource +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import androidx.compose.ui.viewinterop.AndroidView +import androidx.lifecycle.compose.LocalLifecycleOwner +import io.picoclaw.android.core.domain.model.VoicePhase +import io.picoclaw.android.core.ui.theme.GradientCyan +import io.picoclaw.android.core.ui.theme.TextPrimary +import io.picoclaw.android.core.ui.theme.TextSecondary +import io.picoclaw.android.feature.chat.voice.CameraCaptureManager +import io.picoclaw.android.feature.chat.voice.VoiceModeState +import com.composables.icons.lucide.R as LucideR +import kotlin.math.PI +import kotlin.math.sin + +private val PillBackground = Color(0xE6141428) +private val ListeningColor = Color(0xFF00D4FF) +private val ThinkingColor = Color(0xFFA855F7) +private val SpeakingColor = Color(0xFF22C55E) +private val ErrorColor = Color(0xFFEF4444) + +@Composable +fun AssistantPillBar( + state: VoiceModeState, + onClose: () -> Unit, + onInterrupt: () -> Unit, + onCameraToggle: () -> Unit, + cameraCaptureManager: CameraCaptureManager, + modifier: Modifier = Modifier +) { + val isExpanded = state.phase == VoicePhase.THINKING || + state.phase == VoicePhase.SPEAKING || + state.phase == VoicePhase.ERROR + + val interruptable = state.phase != VoicePhase.LISTENING && + state.phase != VoicePhase.IDLE + + Column( + modifier = modifier + .fillMaxWidth() + .padding(horizontal = 8.dp, vertical = 8.dp) + ) { + // Camera preview above the pill bar + AnimatedVisibility( + visible = state.isCameraActive, + enter = expandVertically(expandFrom = Alignment.Bottom), + exit = shrinkVertically(shrinkTowards = Alignment.Bottom) + ) { + val lifecycleOwner = LocalLifecycleOwner.current + + Box( + modifier = Modifier + .fillMaxWidth() + .padding(bottom = 8.dp), + contentAlignment = Alignment.CenterEnd + ) { + Box( + modifier = Modifier + .width(120.dp) + .height(90.dp) + .clip(RoundedCornerShape(12.dp)) + ) { + AndroidView( + factory = { ctx -> + PreviewView(ctx).also { preview -> + cameraCaptureManager.bind(lifecycleOwner, preview) + } + }, + modifier = Modifier.fillMaxSize() + ) + } + } + + DisposableEffect(Unit) { + onDispose { + cameraCaptureManager.unbind() + } + } + } + + // Main pill bar + Box( + modifier = Modifier + .fillMaxWidth() + .clip(RoundedCornerShape(24.dp)) + .background( + Brush.verticalGradient( + colors = listOf( + PillBackground, + PillBackground.copy(alpha = 0.95f) + ) + ) + ) + .then( + if (interruptable) { + Modifier.clickable( + indication = null, + interactionSource = remember { MutableInteractionSource() } + ) { onInterrupt() } + } else Modifier + ) + .animateContentSize() + ) { + Column( + modifier = Modifier + .fillMaxWidth() + .padding(16.dp) + ) { + // Expanded content: response text area + AnimatedVisibility( + visible = isExpanded && (state.responseText.isNotEmpty() || state.errorMessage != null) + ) { + Column(modifier = Modifier.padding(bottom = 12.dp)) { + if (state.responseText.isNotEmpty()) { + Text( + text = state.responseText, + style = MaterialTheme.typography.bodyMedium, + color = TextPrimary, + maxLines = 6, + overflow = TextOverflow.Ellipsis + ) + } + if (state.errorMessage != null) { + Text( + text = state.errorMessage, + style = MaterialTheme.typography.bodySmall, + color = ErrorColor + ) + } + } + } + + // Status / recognized text row + AnimatedVisibility( + visible = isExpanded && (state.recognizedText.isNotEmpty() || state.statusText != null) + ) { + Text( + text = state.statusText ?: state.recognizedText, + style = MaterialTheme.typography.bodySmall, + color = TextSecondary, + maxLines = 1, + overflow = TextOverflow.Ellipsis, + modifier = Modifier.padding(bottom = 8.dp) + ) + } + + // Bottom row: waveform + controls + Row( + modifier = Modifier.fillMaxWidth(), + verticalAlignment = Alignment.CenterVertically + ) { + // Mic icon + Icon( + painter = painterResource(LucideR.drawable.lucide_ic_mic), + contentDescription = "Microphone", + modifier = Modifier.size(20.dp), + tint = phaseColor(state.phase) + ) + + Spacer(modifier = Modifier.width(8.dp)) + + // Waveform + WaveformBar( + phase = state.phase, + amplitude = state.amplitudeNormalized, + modifier = Modifier + .weight(1f) + .height(28.dp) + ) + + Spacer(modifier = Modifier.width(8.dp)) + + // Camera toggle + IconButton( + onClick = onCameraToggle, + modifier = Modifier.size(36.dp) + ) { + Icon( + painter = painterResource( + if (state.isCameraActive) LucideR.drawable.lucide_ic_camera_off + else LucideR.drawable.lucide_ic_camera + ), + contentDescription = if (state.isCameraActive) "Turn off camera" else "Turn on camera", + modifier = Modifier.size(18.dp), + tint = if (state.isCameraActive) GradientCyan else TextSecondary + ) + } + + // Close button + IconButton( + onClick = onClose, + modifier = Modifier.size(36.dp) + ) { + Icon( + painter = painterResource(LucideR.drawable.lucide_ic_x), + contentDescription = "Close", + modifier = Modifier.size(18.dp), + tint = TextSecondary + ) + } + } + } + } + } +} + +@Composable +private fun WaveformBar( + phase: VoicePhase, + amplitude: Float, + modifier: Modifier = Modifier +) { + val transition = rememberInfiniteTransition(label = "waveform") + val animPhase by transition.animateFloat( + initialValue = 0f, + targetValue = 2f * PI.toFloat(), + animationSpec = infiniteRepeatable( + animation = tween(durationMillis = 1500, easing = LinearEasing), + repeatMode = RepeatMode.Restart + ), + label = "wavePhase" + ) + + val color = phaseColor(phase) + + Canvas(modifier = modifier) { + val barCount = 32 + val barWidth = size.width / barCount + val centerY = size.height / 2f + + for (i in 0 until barCount) { + val x = i * barWidth + barWidth / 2f + val normalizedX = i.toFloat() / barCount + + val barHeight = when (phase) { + VoicePhase.LISTENING -> { + val wave = sin(normalizedX * 4f * PI.toFloat() + animPhase).coerceIn(-1f, 1f) + val base = 0.15f + val dynamic = amplitude * 0.85f * ((wave + 1f) / 2f) + (base + dynamic) * size.height + } + VoicePhase.THINKING -> { + val wave = sin(normalizedX * 3f * PI.toFloat() + animPhase * 2f) + (0.2f + 0.15f * ((wave + 1f) / 2f)) * size.height + } + VoicePhase.SPEAKING -> { + val wave = sin(normalizedX * 5f * PI.toFloat() + animPhase * 1.5f) + (0.2f + 0.4f * ((wave + 1f) / 2f)) * size.height + } + else -> 0.1f * size.height + } + + drawLine( + color = color.copy(alpha = 0.6f + 0.4f * (barHeight / size.height)), + start = Offset(x, centerY - barHeight / 2f), + end = Offset(x, centerY + barHeight / 2f), + strokeWidth = barWidth * 0.5f + ) + } + } +} + +private fun phaseColor(phase: VoicePhase): Color = when (phase) { + VoicePhase.LISTENING -> ListeningColor + VoicePhase.SENDING -> Color(0xFFFF8C42) + VoicePhase.THINKING -> ThinkingColor + VoicePhase.SPEAKING -> SpeakingColor + VoicePhase.ERROR -> ErrorColor + VoicePhase.IDLE -> Color(0xFF4A5568) +}