feat: add assistant overlay with Gemini-style pill bar UI

Implement ASSIST intent-based voice assistant that runs as a system
overlay. Launches via home button long-press, shows a compact pill bar
at screen bottom with waveform animation, camera toggle, and expandable
response text. Each session creates a fresh WebSocket connection
(client_type="assistant") with no DB persistence.

New files:
- AssistantConnection interface + AssistantConnectionImpl (clean arch)
- AssistantManager (voice loop with direct WS communication)
- AssistantPillBar (Gemini-style Compose UI)
- AssistantService (LifecycleService with overlay)
- AssistantActivity (transparent, permission checks)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
KoheiYamashita 2026-02-21 18:05:05 +09:00
parent 302985d17d
commit 9ea508a6e0
10 changed files with 957 additions and 5 deletions

View file

@ -5,6 +5,9 @@
<uses-permission android:name="android.permission.CAMERA" /> <uses-permission android:name="android.permission.CAMERA" />
<uses-permission android:name="android.permission.RECORD_AUDIO" /> <uses-permission android:name="android.permission.RECORD_AUDIO" />
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" /> <uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
<uses-permission android:name="android.permission.SYSTEM_ALERT_WINDOW" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_SPECIAL_USE" />
<queries> <queries>
<intent> <intent>
@ -47,6 +50,27 @@
<action android:name="io.picoclaw.android.AGENT_MESSAGE" /> <action android:name="io.picoclaw.android.AGENT_MESSAGE" />
</intent-filter> </intent-filter>
</receiver> </receiver>
<activity
android:name=".assistant.AssistantActivity"
android:exported="true"
android:excludeFromRecents="true"
android:launchMode="singleTask"
android:theme="@android:style/Theme.NoDisplay">
<intent-filter>
<action android:name="android.intent.action.ASSIST" />
<category android:name="android.intent.category.DEFAULT" />
</intent-filter>
</activity>
<service
android:name=".assistant.AssistantService"
android:exported="false"
android:foregroundServiceType="specialUse">
<property
android:name="android.app.PROPERTY_SPECIAL_USE_FGS_SUBTYPE"
android:value="AI voice assistant overlay for real-time conversation" />
</service>
</application> </application>
</manifest> </manifest>

View file

@ -0,0 +1,65 @@
package io.picoclaw.android.assistant
import android.Manifest
import android.content.Intent
import android.content.pm.PackageManager
import android.net.Uri
import android.os.Bundle
import android.provider.Settings
import androidx.activity.ComponentActivity
import androidx.activity.result.contract.ActivityResultContracts
import androidx.core.content.ContextCompat
class AssistantActivity : ComponentActivity() {
private val audioPermissionLauncher = registerForActivityResult(
ActivityResultContracts.RequestPermission()
) { granted ->
if (granted) {
launchServiceAndFinish()
} else {
finish()
}
}
private val overlayPermissionLauncher = registerForActivityResult(
ActivityResultContracts.StartActivityForResult()
) {
if (Settings.canDrawOverlays(this)) {
checkAudioPermission()
} else {
finish()
}
}
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
if (!Settings.canDrawOverlays(this)) {
val intent = Intent(
Settings.ACTION_MANAGE_OVERLAY_PERMISSION,
Uri.parse("package:$packageName")
)
overlayPermissionLauncher.launch(intent)
return
}
checkAudioPermission()
}
private fun checkAudioPermission() {
if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO)
!= PackageManager.PERMISSION_GRANTED
) {
audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO)
} else {
launchServiceAndFinish()
}
}
private fun launchServiceAndFinish() {
val serviceIntent = Intent(this, AssistantService::class.java)
startForegroundService(serviceIntent)
finish()
}
}

View file

@ -0,0 +1,168 @@
package io.picoclaw.android.assistant
import android.app.Notification
import android.content.Intent
import android.graphics.PixelFormat
import android.os.IBinder
import android.view.Gravity
import android.view.WindowManager
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.ui.platform.ComposeView
import androidx.core.app.NotificationCompat
import androidx.lifecycle.LifecycleService
import androidx.lifecycle.setViewTreeLifecycleOwner
import androidx.savedstate.SavedStateRegistry
import androidx.savedstate.SavedStateRegistryController
import androidx.savedstate.SavedStateRegistryOwner
import androidx.savedstate.setViewTreeSavedStateRegistryOwner
import io.ktor.client.HttpClient
import io.picoclaw.android.core.data.remote.WebSocketClient
import io.picoclaw.android.core.data.repository.AssistantConnectionImpl
import io.picoclaw.android.core.domain.repository.AssistantConnection
import io.picoclaw.android.core.domain.repository.TtsSettingsRepository
import io.picoclaw.android.core.ui.theme.PicoClawTheme
import io.picoclaw.android.feature.chat.assistant.AssistantManager
import io.picoclaw.android.feature.chat.assistant.AssistantPillBar
import io.picoclaw.android.feature.chat.voice.CameraCaptureManager
import io.picoclaw.android.feature.chat.voice.SpeechRecognizerWrapper
import io.picoclaw.android.feature.chat.voice.TextToSpeechWrapper
import io.picoclaw.android.receiver.NotificationHelper
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancel
import org.koin.android.ext.android.inject
class AssistantService : LifecycleService(), SavedStateRegistryOwner {
private val httpClient: HttpClient by inject()
private val ttsSettingsRepo: TtsSettingsRepository by inject()
private lateinit var serviceScope: CoroutineScope
private lateinit var connection: AssistantConnection
private lateinit var assistantManager: AssistantManager
private lateinit var ttsWrapper: TextToSpeechWrapper
private lateinit var sttWrapper: SpeechRecognizerWrapper
private lateinit var cameraCaptureManager: CameraCaptureManager
private var overlayView: ComposeView? = null
private val windowManager by lazy { getSystemService(WINDOW_SERVICE) as WindowManager }
private val savedStateRegistryController = SavedStateRegistryController.create(this)
override val savedStateRegistry: SavedStateRegistry
get() = savedStateRegistryController.savedStateRegistry
override fun onCreate() {
savedStateRegistryController.performAttach()
savedStateRegistryController.performRestore(null)
super.onCreate()
serviceScope = CoroutineScope(SupervisorJob() + Dispatchers.Main)
connection = AssistantConnectionImpl(httpClient)
sttWrapper = SpeechRecognizerWrapper(this)
ttsWrapper = TextToSpeechWrapper(this, ttsSettingsRepo.ttsConfig)
cameraCaptureManager = CameraCaptureManager(this)
assistantManager = AssistantManager(
sttWrapper = sttWrapper,
ttsWrapper = ttsWrapper,
connection = connection,
cameraCaptureManager = cameraCaptureManager,
contentResolver = contentResolver
)
}
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
super.onStartCommand(intent, flags, startId)
startForeground(NOTIFICATION_ID, buildNotification())
// Resolve wsUrl from the main WebSocketClient
val mainWsClient: WebSocketClient by inject()
connection.connect(mainWsClient.wsUrl)
addOverlay()
assistantManager.start(serviceScope)
return START_NOT_STICKY
}
override fun onDestroy() {
removeOverlay()
assistantManager.destroy()
ttsWrapper.destroy()
connection.disconnect()
serviceScope.cancel()
super.onDestroy()
}
override fun onBind(intent: Intent): IBinder? {
super.onBind(intent)
return null
}
private fun shutdown() {
stopForeground(STOP_FOREGROUND_REMOVE)
stopSelf()
}
private fun addOverlay() {
if (overlayView != null) return
val params = WindowManager.LayoutParams(
WindowManager.LayoutParams.MATCH_PARENT,
WindowManager.LayoutParams.WRAP_CONTENT,
WindowManager.LayoutParams.TYPE_APPLICATION_OVERLAY,
WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL or
WindowManager.LayoutParams.FLAG_LAYOUT_IN_SCREEN,
PixelFormat.TRANSLUCENT
).apply {
gravity = Gravity.BOTTOM
}
val view = ComposeView(this).apply {
setViewTreeLifecycleOwner(this@AssistantService)
setViewTreeSavedStateRegistryOwner(this@AssistantService)
setContent {
PicoClawTheme {
val state by assistantManager.state.collectAsState()
AssistantPillBar(
state = state,
onClose = { shutdown() },
onInterrupt = { assistantManager.interrupt() },
onCameraToggle = { assistantManager.toggleCamera() },
cameraCaptureManager = cameraCaptureManager
)
}
}
}
windowManager.addView(view, params)
overlayView = view
}
private fun removeOverlay() {
overlayView?.let {
windowManager.removeView(it)
overlayView = null
}
}
private fun buildNotification(): Notification {
return NotificationCompat.Builder(this, NotificationHelper.ASSISTANT_CHANNEL_ID)
.setSmallIcon(android.R.drawable.ic_btn_speak_now)
.setContentTitle("PicoClaw Assistant")
.setContentText("Listening...")
.setPriority(NotificationCompat.PRIORITY_LOW)
.setOngoing(true)
.build()
}
companion object {
private const val NOTIFICATION_ID = 2001
}
}

View file

@ -12,16 +12,30 @@ object NotificationHelper {
private const val CHANNEL_NAME = "Agent Messages" private const val CHANNEL_NAME = "Agent Messages"
private const val NOTIFICATION_ID = 1001 private const val NOTIFICATION_ID = 1001
const val ASSISTANT_CHANNEL_ID = "picoclaw_assistant"
private const val ASSISTANT_CHANNEL_NAME = "Assistant"
fun createNotificationChannel(context: Context) { fun createNotificationChannel(context: Context) {
val channel = NotificationChannel( val manager = context.getSystemService(NotificationManager::class.java)
val messageChannel = NotificationChannel(
CHANNEL_ID, CHANNEL_ID,
CHANNEL_NAME, CHANNEL_NAME,
NotificationManager.IMPORTANCE_DEFAULT NotificationManager.IMPORTANCE_DEFAULT
).apply { ).apply {
description = "Messages from PicoClaw agent" description = "Messages from PicoClaw agent"
} }
val manager = context.getSystemService(NotificationManager::class.java) manager.createNotificationChannel(messageChannel)
manager.createNotificationChannel(channel)
val assistantChannel = NotificationChannel(
ASSISTANT_CHANNEL_ID,
ASSISTANT_CHANNEL_NAME,
NotificationManager.IMPORTANCE_LOW
).apply {
description = "Assistant overlay service"
setShowBadge(false)
}
manager.createNotificationChannel(assistantChannel)
} }
fun showMessageNotification(context: Context, content: String) { fun showMessageNotification(context: Context, content: String) {

View file

@ -26,7 +26,8 @@ import kotlinx.serialization.json.Json
class WebSocketClient( class WebSocketClient(
private val client: HttpClient, private val client: HttpClient,
private val scope: CoroutineScope, private val scope: CoroutineScope,
private val clientId: String private val clientId: String,
private val clientType: String = "main"
) { ) {
private val _connectionState = MutableStateFlow(ConnectionState.DISCONNECTED) private val _connectionState = MutableStateFlow(ConnectionState.DISCONNECTED)
@ -48,7 +49,7 @@ class WebSocketClient(
while (isActive) { while (isActive) {
try { try {
_connectionState.value = ConnectionState.CONNECTING _connectionState.value = ConnectionState.CONNECTING
val url = "$wsUrl?client_id=$clientId&client_type=main" val url = "$wsUrl?client_id=$clientId&client_type=$clientType"
client.webSocket(url) { client.webSocket(url) {
session = this session = this
_connectionState.value = ConnectionState.CONNECTED _connectionState.value = ConnectionState.CONNECTED

View file

@ -0,0 +1,71 @@
package io.picoclaw.android.core.data.repository
import io.ktor.client.HttpClient
import io.picoclaw.android.core.data.remote.WebSocketClient
import io.picoclaw.android.core.data.remote.dto.WsIncoming
import io.picoclaw.android.core.domain.model.AssistantMessage
import io.picoclaw.android.core.domain.model.ConnectionState
import io.picoclaw.android.core.domain.repository.AssistantConnection
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancel
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.SharedFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asSharedFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
import java.util.UUID
class AssistantConnectionImpl(
private val httpClient: HttpClient
) : AssistantConnection {
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val clientId = UUID.randomUUID().toString()
private val wsClient = WebSocketClient(httpClient, scope, clientId, "assistant")
private val _messages = MutableSharedFlow<AssistantMessage>(extraBufferCapacity = 64)
override val messages: SharedFlow<AssistantMessage> = _messages.asSharedFlow()
private val _statusText = MutableStateFlow<String?>(null)
override val statusText: StateFlow<String?> = _statusText.asStateFlow()
override val connectionState: StateFlow<ConnectionState> = wsClient.connectionState
init {
scope.launch {
wsClient.incomingMessages.collect { dto ->
when (dto.type) {
"status" -> _statusText.value = dto.content
"status_end" -> _statusText.value = null
else -> {
_statusText.value = null
_messages.emit(AssistantMessage(content = dto.content, type = dto.type))
}
}
}
}
}
override fun connect(wsUrl: String) {
wsClient.wsUrl = wsUrl
wsClient.connect()
}
override fun disconnect() {
wsClient.disconnect()
scope.cancel()
}
override suspend fun send(text: String, images: List<String>, inputMode: String) {
val dto = WsIncoming(
content = text,
images = images.ifEmpty { null },
inputMode = inputMode
)
wsClient.send(dto)
}
}

View file

@ -0,0 +1,6 @@
package io.picoclaw.android.core.domain.model
data class AssistantMessage(
val content: String,
val type: String? = null
)

View file

@ -0,0 +1,16 @@
package io.picoclaw.android.core.domain.repository
import io.picoclaw.android.core.domain.model.AssistantMessage
import io.picoclaw.android.core.domain.model.ConnectionState
import kotlinx.coroutines.flow.SharedFlow
import kotlinx.coroutines.flow.StateFlow
interface AssistantConnection {
val messages: SharedFlow<AssistantMessage>
val statusText: StateFlow<String?>
val connectionState: StateFlow<ConnectionState>
fun connect(wsUrl: String)
fun disconnect()
suspend fun send(text: String, images: List<String> = emptyList(), inputMode: String = "assistant")
}

View file

@ -0,0 +1,275 @@
package io.picoclaw.android.feature.chat.assistant
import android.content.ContentResolver
import android.speech.SpeechRecognizer
import android.util.Base64
import android.util.Log
import io.picoclaw.android.core.domain.model.AssistantMessage
import io.picoclaw.android.core.domain.model.VoicePhase
import io.picoclaw.android.core.domain.repository.AssistantConnection
import io.picoclaw.android.feature.chat.voice.CameraCaptureManager
import io.picoclaw.android.feature.chat.voice.SpeechRecognizerWrapper
import io.picoclaw.android.feature.chat.voice.SttResult
import io.picoclaw.android.feature.chat.voice.TextToSpeechWrapper
import io.picoclaw.android.feature.chat.voice.VoiceModeState
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.Job
import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.flow.update
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.selects.onTimeout
import kotlinx.coroutines.selects.select
@OptIn(ExperimentalCoroutinesApi::class)
class AssistantManager(
private val sttWrapper: SpeechRecognizerWrapper,
private val ttsWrapper: TextToSpeechWrapper,
private val connection: AssistantConnection,
private val cameraCaptureManager: CameraCaptureManager,
private val contentResolver: ContentResolver
) {
private val _state = MutableStateFlow(VoiceModeState())
val state: StateFlow<VoiceModeState> = _state.asStateFlow()
private var loopJob: Job? = null
private var parentScope: CoroutineScope? = null
fun toggleCamera() {
_state.update { it.copy(isCameraActive = !it.isCameraActive) }
}
fun start(scope: CoroutineScope) {
if (loopJob?.isActive == true) return
parentScope = scope
_state.update {
VoiceModeState(isActive = true, phase = VoicePhase.LISTENING, isCameraActive = it.isCameraActive)
}
loopJob = scope.launch {
voiceLoop()
}
}
fun stop() {
loopJob?.cancel()
loopJob = null
parentScope = null
ttsWrapper.stop()
cameraCaptureManager.unbind()
_state.value = VoiceModeState()
}
fun interrupt() {
val scope = parentScope ?: return
if (loopJob?.isActive != true) return
ttsWrapper.stop()
loopJob?.cancel()
loopJob = null
_state.update {
VoiceModeState(isActive = true, phase = VoicePhase.LISTENING, isCameraActive = it.isCameraActive)
}
loopJob = scope.launch { voiceLoop() }
}
fun destroy() {
stop()
}
private suspend fun voiceLoop() = coroutineScope {
val speechQueue = Channel<String>(Channel.UNLIMITED)
val messageCollectorJob = launch {
connection.messages.collect { msg: AssistantMessage ->
if (msg.type == "warning" || msg.type == "error") {
_state.update { it.copy(responseText = msg.content) }
} else {
speechQueue.send(msg.content)
}
}
}
val statusCollectorJob = launch {
connection.statusText.collect { label ->
_state.update { it.copy(statusText = label) }
}
}
try {
while (isActive) {
_state.update {
it.copy(
phase = VoicePhase.LISTENING,
recognizedText = "", responseText = "",
statusText = null,
errorMessage = null, amplitudeNormalized = 0f
)
}
val userTextChannel = Channel<String?>(1)
val listenJob = launch {
val text = listen()
userTextChannel.send(text)
}
select<Unit> {
userTextChannel.onReceive { text ->
if (!text.isNullOrBlank()) {
_state.update { it.copy(phase = VoicePhase.SENDING, recognizedText = text) }
try {
val base64Images = if (_state.value.isCameraActive) {
captureAndEncode()
} else emptyList()
connection.send(text, base64Images)
} catch (e: Exception) {
Log.w(TAG, "Failed to send message", e)
_state.update {
it.copy(phase = VoicePhase.ERROR, errorMessage = "Failed to send")
}
delay(2000)
return@onReceive
}
awaitAndSpeakResponse(speechQueue)
} else {
drainSpeechQueue(speechQueue)
}
}
speechQueue.onReceive { content ->
listenJob.cancel()
speakAndDrain(content, speechQueue)
}
}
}
} finally {
messageCollectorJob.cancel()
statusCollectorJob.cancel()
speechQueue.close()
}
}
private sealed interface WaitResult {
data class Message(val content: String) : WaitResult
data object Heartbeat : WaitResult
data object Timeout : WaitResult
}
private suspend fun awaitAndSpeakResponse(
speechQueue: Channel<String>
) = coroutineScope {
val heartbeat = Channel<Unit>(Channel.CONFLATED)
val statusJob = launch {
connection.statusText.collect { label ->
if (label != null) heartbeat.trySend(Unit)
}
}
try {
_state.update { it.copy(phase = VoicePhase.THINKING, statusText = null) }
while (true) {
val result = select<WaitResult> {
speechQueue.onReceive { WaitResult.Message(it) }
heartbeat.onReceive { WaitResult.Heartbeat }
onTimeout(30_000) { WaitResult.Timeout }
}
when (result) {
WaitResult.Timeout -> {
_state.update {
it.copy(phase = VoicePhase.ERROR, errorMessage = "Response timed out")
}
delay(2000)
return@coroutineScope
}
WaitResult.Heartbeat -> continue
is WaitResult.Message -> {
speakAndDrain(result.content, speechQueue)
val currentStatus = connection.statusText.value
if (currentStatus == null) return@coroutineScope
_state.update {
it.copy(phase = VoicePhase.THINKING, statusText = currentStatus)
}
}
}
}
} finally {
statusJob.cancel()
}
}
private suspend fun speakAndDrain(firstContent: String, speechQueue: Channel<String>) {
_state.update { it.copy(phase = VoicePhase.SPEAKING, responseText = firstContent) }
ttsWrapper.speak(firstContent)
while (true) {
val next = speechQueue.tryReceive().getOrNull() ?: break
_state.update { it.copy(responseText = next) }
ttsWrapper.speak(next)
}
}
private suspend fun drainSpeechQueue(speechQueue: Channel<String>) {
val first = speechQueue.tryReceive().getOrNull() ?: return
speakAndDrain(first, speechQueue)
}
private suspend fun listen(): String? {
var finalText: String? = null
sttWrapper.startListening().collect { result ->
when (result) {
is SttResult.Partial -> {
_state.update { it.copy(recognizedText = result.text) }
}
is SttResult.Final -> {
finalText = result.text
}
is SttResult.RmsChanged -> {
val normalized = ((result.rms + 2f) / 12f).coerceIn(0f, 1f)
_state.update { it.copy(amplitudeNormalized = normalized) }
}
is SttResult.Error -> {
if (result.code == SpeechRecognizer.ERROR_NO_MATCH ||
result.code == SpeechRecognizer.ERROR_SPEECH_TIMEOUT
) {
finalText = ""
} else {
_state.update {
it.copy(
phase = VoicePhase.ERROR,
errorMessage = "Speech recognition error (code=${result.code})"
)
}
delay(2000)
finalText = null
}
}
}
}
return finalText
}
private suspend fun captureAndEncode(): List<String> {
val attachment = cameraCaptureManager.captureFrame() ?: return emptyList()
val uri = android.net.Uri.parse(attachment.uri)
return try {
val bytes = contentResolver.openInputStream(uri)?.use { it.readBytes() }
?: return emptyList()
listOf(Base64.encodeToString(bytes, Base64.NO_WRAP))
} catch (e: Exception) {
Log.w(TAG, "Failed to encode captured image", e)
emptyList()
}
}
companion object {
private const val TAG = "AssistantManager"
}
}

View file

@ -0,0 +1,312 @@
package io.picoclaw.android.feature.chat.assistant
import androidx.camera.view.PreviewView
import androidx.compose.animation.AnimatedVisibility
import androidx.compose.animation.animateContentSize
import androidx.compose.animation.core.LinearEasing
import androidx.compose.animation.core.RepeatMode
import androidx.compose.animation.core.animateFloat
import androidx.compose.animation.core.infiniteRepeatable
import androidx.compose.animation.core.rememberInfiniteTransition
import androidx.compose.animation.core.tween
import androidx.compose.animation.expandVertically
import androidx.compose.animation.shrinkVertically
import androidx.compose.foundation.Canvas
import androidx.compose.foundation.background
import androidx.compose.foundation.clickable
import androidx.compose.foundation.interaction.MutableInteractionSource
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButton
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.DisposableEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.remember
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.graphics.Brush
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.res.painterResource
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import androidx.compose.ui.viewinterop.AndroidView
import androidx.lifecycle.compose.LocalLifecycleOwner
import io.picoclaw.android.core.domain.model.VoicePhase
import io.picoclaw.android.core.ui.theme.GradientCyan
import io.picoclaw.android.core.ui.theme.TextPrimary
import io.picoclaw.android.core.ui.theme.TextSecondary
import io.picoclaw.android.feature.chat.voice.CameraCaptureManager
import io.picoclaw.android.feature.chat.voice.VoiceModeState
import com.composables.icons.lucide.R as LucideR
import kotlin.math.PI
import kotlin.math.sin
private val PillBackground = Color(0xE6141428)
private val ListeningColor = Color(0xFF00D4FF)
private val ThinkingColor = Color(0xFFA855F7)
private val SpeakingColor = Color(0xFF22C55E)
private val ErrorColor = Color(0xFFEF4444)
@Composable
fun AssistantPillBar(
state: VoiceModeState,
onClose: () -> Unit,
onInterrupt: () -> Unit,
onCameraToggle: () -> Unit,
cameraCaptureManager: CameraCaptureManager,
modifier: Modifier = Modifier
) {
val isExpanded = state.phase == VoicePhase.THINKING ||
state.phase == VoicePhase.SPEAKING ||
state.phase == VoicePhase.ERROR
val interruptable = state.phase != VoicePhase.LISTENING &&
state.phase != VoicePhase.IDLE
Column(
modifier = modifier
.fillMaxWidth()
.padding(horizontal = 8.dp, vertical = 8.dp)
) {
// Camera preview above the pill bar
AnimatedVisibility(
visible = state.isCameraActive,
enter = expandVertically(expandFrom = Alignment.Bottom),
exit = shrinkVertically(shrinkTowards = Alignment.Bottom)
) {
val lifecycleOwner = LocalLifecycleOwner.current
Box(
modifier = Modifier
.fillMaxWidth()
.padding(bottom = 8.dp),
contentAlignment = Alignment.CenterEnd
) {
Box(
modifier = Modifier
.width(120.dp)
.height(90.dp)
.clip(RoundedCornerShape(12.dp))
) {
AndroidView(
factory = { ctx ->
PreviewView(ctx).also { preview ->
cameraCaptureManager.bind(lifecycleOwner, preview)
}
},
modifier = Modifier.fillMaxSize()
)
}
}
DisposableEffect(Unit) {
onDispose {
cameraCaptureManager.unbind()
}
}
}
// Main pill bar
Box(
modifier = Modifier
.fillMaxWidth()
.clip(RoundedCornerShape(24.dp))
.background(
Brush.verticalGradient(
colors = listOf(
PillBackground,
PillBackground.copy(alpha = 0.95f)
)
)
)
.then(
if (interruptable) {
Modifier.clickable(
indication = null,
interactionSource = remember { MutableInteractionSource() }
) { onInterrupt() }
} else Modifier
)
.animateContentSize()
) {
Column(
modifier = Modifier
.fillMaxWidth()
.padding(16.dp)
) {
// Expanded content: response text area
AnimatedVisibility(
visible = isExpanded && (state.responseText.isNotEmpty() || state.errorMessage != null)
) {
Column(modifier = Modifier.padding(bottom = 12.dp)) {
if (state.responseText.isNotEmpty()) {
Text(
text = state.responseText,
style = MaterialTheme.typography.bodyMedium,
color = TextPrimary,
maxLines = 6,
overflow = TextOverflow.Ellipsis
)
}
if (state.errorMessage != null) {
Text(
text = state.errorMessage,
style = MaterialTheme.typography.bodySmall,
color = ErrorColor
)
}
}
}
// Status / recognized text row
AnimatedVisibility(
visible = isExpanded && (state.recognizedText.isNotEmpty() || state.statusText != null)
) {
Text(
text = state.statusText ?: state.recognizedText,
style = MaterialTheme.typography.bodySmall,
color = TextSecondary,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
modifier = Modifier.padding(bottom = 8.dp)
)
}
// Bottom row: waveform + controls
Row(
modifier = Modifier.fillMaxWidth(),
verticalAlignment = Alignment.CenterVertically
) {
// Mic icon
Icon(
painter = painterResource(LucideR.drawable.lucide_ic_mic),
contentDescription = "Microphone",
modifier = Modifier.size(20.dp),
tint = phaseColor(state.phase)
)
Spacer(modifier = Modifier.width(8.dp))
// Waveform
WaveformBar(
phase = state.phase,
amplitude = state.amplitudeNormalized,
modifier = Modifier
.weight(1f)
.height(28.dp)
)
Spacer(modifier = Modifier.width(8.dp))
// Camera toggle
IconButton(
onClick = onCameraToggle,
modifier = Modifier.size(36.dp)
) {
Icon(
painter = painterResource(
if (state.isCameraActive) LucideR.drawable.lucide_ic_camera_off
else LucideR.drawable.lucide_ic_camera
),
contentDescription = if (state.isCameraActive) "Turn off camera" else "Turn on camera",
modifier = Modifier.size(18.dp),
tint = if (state.isCameraActive) GradientCyan else TextSecondary
)
}
// Close button
IconButton(
onClick = onClose,
modifier = Modifier.size(36.dp)
) {
Icon(
painter = painterResource(LucideR.drawable.lucide_ic_x),
contentDescription = "Close",
modifier = Modifier.size(18.dp),
tint = TextSecondary
)
}
}
}
}
}
}
@Composable
private fun WaveformBar(
phase: VoicePhase,
amplitude: Float,
modifier: Modifier = Modifier
) {
val transition = rememberInfiniteTransition(label = "waveform")
val animPhase by transition.animateFloat(
initialValue = 0f,
targetValue = 2f * PI.toFloat(),
animationSpec = infiniteRepeatable(
animation = tween(durationMillis = 1500, easing = LinearEasing),
repeatMode = RepeatMode.Restart
),
label = "wavePhase"
)
val color = phaseColor(phase)
Canvas(modifier = modifier) {
val barCount = 32
val barWidth = size.width / barCount
val centerY = size.height / 2f
for (i in 0 until barCount) {
val x = i * barWidth + barWidth / 2f
val normalizedX = i.toFloat() / barCount
val barHeight = when (phase) {
VoicePhase.LISTENING -> {
val wave = sin(normalizedX * 4f * PI.toFloat() + animPhase).coerceIn(-1f, 1f)
val base = 0.15f
val dynamic = amplitude * 0.85f * ((wave + 1f) / 2f)
(base + dynamic) * size.height
}
VoicePhase.THINKING -> {
val wave = sin(normalizedX * 3f * PI.toFloat() + animPhase * 2f)
(0.2f + 0.15f * ((wave + 1f) / 2f)) * size.height
}
VoicePhase.SPEAKING -> {
val wave = sin(normalizedX * 5f * PI.toFloat() + animPhase * 1.5f)
(0.2f + 0.4f * ((wave + 1f) / 2f)) * size.height
}
else -> 0.1f * size.height
}
drawLine(
color = color.copy(alpha = 0.6f + 0.4f * (barHeight / size.height)),
start = Offset(x, centerY - barHeight / 2f),
end = Offset(x, centerY + barHeight / 2f),
strokeWidth = barWidth * 0.5f
)
}
}
}
private fun phaseColor(phase: VoicePhase): Color = when (phase) {
VoicePhase.LISTENING -> ListeningColor
VoicePhase.SENDING -> Color(0xFFFF8C42)
VoicePhase.THINKING -> ThinkingColor
VoicePhase.SPEAKING -> SpeakingColor
VoicePhase.ERROR -> ErrorColor
VoicePhase.IDLE -> Color(0xFF4A5568)
}