feat: add assistant overlay with Gemini-style pill bar UI
Implement ASSIST intent-based voice assistant that runs as a system overlay. Launches via home button long-press, shows a compact pill bar at screen bottom with waveform animation, camera toggle, and expandable response text. Each session creates a fresh WebSocket connection (client_type="assistant") with no DB persistence. New files: - AssistantConnection interface + AssistantConnectionImpl (clean arch) - AssistantManager (voice loop with direct WS communication) - AssistantPillBar (Gemini-style Compose UI) - AssistantService (LifecycleService with overlay) - AssistantActivity (transparent, permission checks) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
302985d17d
commit
9ea508a6e0
10 changed files with 957 additions and 5 deletions
|
|
@ -5,6 +5,9 @@
|
|||
<uses-permission android:name="android.permission.CAMERA" />
|
||||
<uses-permission android:name="android.permission.RECORD_AUDIO" />
|
||||
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
|
||||
<uses-permission android:name="android.permission.SYSTEM_ALERT_WINDOW" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_SPECIAL_USE" />
|
||||
|
||||
<queries>
|
||||
<intent>
|
||||
|
|
@ -47,6 +50,27 @@
|
|||
<action android:name="io.picoclaw.android.AGENT_MESSAGE" />
|
||||
</intent-filter>
|
||||
</receiver>
|
||||
|
||||
<activity
|
||||
android:name=".assistant.AssistantActivity"
|
||||
android:exported="true"
|
||||
android:excludeFromRecents="true"
|
||||
android:launchMode="singleTask"
|
||||
android:theme="@android:style/Theme.NoDisplay">
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.ASSIST" />
|
||||
<category android:name="android.intent.category.DEFAULT" />
|
||||
</intent-filter>
|
||||
</activity>
|
||||
|
||||
<service
|
||||
android:name=".assistant.AssistantService"
|
||||
android:exported="false"
|
||||
android:foregroundServiceType="specialUse">
|
||||
<property
|
||||
android:name="android.app.PROPERTY_SPECIAL_USE_FGS_SUBTYPE"
|
||||
android:value="AI voice assistant overlay for real-time conversation" />
|
||||
</service>
|
||||
</application>
|
||||
|
||||
</manifest>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,65 @@
|
|||
package io.picoclaw.android.assistant
|
||||
|
||||
import android.Manifest
|
||||
import android.content.Intent
|
||||
import android.content.pm.PackageManager
|
||||
import android.net.Uri
|
||||
import android.os.Bundle
|
||||
import android.provider.Settings
|
||||
import androidx.activity.ComponentActivity
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.core.content.ContextCompat
|
||||
|
||||
class AssistantActivity : ComponentActivity() {
|
||||
|
||||
private val audioPermissionLauncher = registerForActivityResult(
|
||||
ActivityResultContracts.RequestPermission()
|
||||
) { granted ->
|
||||
if (granted) {
|
||||
launchServiceAndFinish()
|
||||
} else {
|
||||
finish()
|
||||
}
|
||||
}
|
||||
|
||||
private val overlayPermissionLauncher = registerForActivityResult(
|
||||
ActivityResultContracts.StartActivityForResult()
|
||||
) {
|
||||
if (Settings.canDrawOverlays(this)) {
|
||||
checkAudioPermission()
|
||||
} else {
|
||||
finish()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
|
||||
if (!Settings.canDrawOverlays(this)) {
|
||||
val intent = Intent(
|
||||
Settings.ACTION_MANAGE_OVERLAY_PERMISSION,
|
||||
Uri.parse("package:$packageName")
|
||||
)
|
||||
overlayPermissionLauncher.launch(intent)
|
||||
return
|
||||
}
|
||||
|
||||
checkAudioPermission()
|
||||
}
|
||||
|
||||
private fun checkAudioPermission() {
|
||||
if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO)
|
||||
!= PackageManager.PERMISSION_GRANTED
|
||||
) {
|
||||
audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO)
|
||||
} else {
|
||||
launchServiceAndFinish()
|
||||
}
|
||||
}
|
||||
|
||||
private fun launchServiceAndFinish() {
|
||||
val serviceIntent = Intent(this, AssistantService::class.java)
|
||||
startForegroundService(serviceIntent)
|
||||
finish()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,168 @@
|
|||
package io.picoclaw.android.assistant
|
||||
|
||||
import android.app.Notification
|
||||
import android.content.Intent
|
||||
import android.graphics.PixelFormat
|
||||
import android.os.IBinder
|
||||
import android.view.Gravity
|
||||
import android.view.WindowManager
|
||||
import androidx.compose.runtime.collectAsState
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.ui.platform.ComposeView
|
||||
import androidx.core.app.NotificationCompat
|
||||
import androidx.lifecycle.LifecycleService
|
||||
import androidx.lifecycle.setViewTreeLifecycleOwner
|
||||
import androidx.savedstate.SavedStateRegistry
|
||||
import androidx.savedstate.SavedStateRegistryController
|
||||
import androidx.savedstate.SavedStateRegistryOwner
|
||||
import androidx.savedstate.setViewTreeSavedStateRegistryOwner
|
||||
import io.ktor.client.HttpClient
|
||||
import io.picoclaw.android.core.data.remote.WebSocketClient
|
||||
import io.picoclaw.android.core.data.repository.AssistantConnectionImpl
|
||||
import io.picoclaw.android.core.domain.repository.AssistantConnection
|
||||
import io.picoclaw.android.core.domain.repository.TtsSettingsRepository
|
||||
import io.picoclaw.android.core.ui.theme.PicoClawTheme
|
||||
import io.picoclaw.android.feature.chat.assistant.AssistantManager
|
||||
import io.picoclaw.android.feature.chat.assistant.AssistantPillBar
|
||||
import io.picoclaw.android.feature.chat.voice.CameraCaptureManager
|
||||
import io.picoclaw.android.feature.chat.voice.SpeechRecognizerWrapper
|
||||
import io.picoclaw.android.feature.chat.voice.TextToSpeechWrapper
|
||||
import io.picoclaw.android.receiver.NotificationHelper
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.cancel
|
||||
import org.koin.android.ext.android.inject
|
||||
|
||||
class AssistantService : LifecycleService(), SavedStateRegistryOwner {
|
||||
|
||||
private val httpClient: HttpClient by inject()
|
||||
private val ttsSettingsRepo: TtsSettingsRepository by inject()
|
||||
|
||||
private lateinit var serviceScope: CoroutineScope
|
||||
private lateinit var connection: AssistantConnection
|
||||
private lateinit var assistantManager: AssistantManager
|
||||
private lateinit var ttsWrapper: TextToSpeechWrapper
|
||||
private lateinit var sttWrapper: SpeechRecognizerWrapper
|
||||
private lateinit var cameraCaptureManager: CameraCaptureManager
|
||||
|
||||
private var overlayView: ComposeView? = null
|
||||
private val windowManager by lazy { getSystemService(WINDOW_SERVICE) as WindowManager }
|
||||
|
||||
private val savedStateRegistryController = SavedStateRegistryController.create(this)
|
||||
override val savedStateRegistry: SavedStateRegistry
|
||||
get() = savedStateRegistryController.savedStateRegistry
|
||||
|
||||
override fun onCreate() {
|
||||
savedStateRegistryController.performAttach()
|
||||
savedStateRegistryController.performRestore(null)
|
||||
super.onCreate()
|
||||
|
||||
serviceScope = CoroutineScope(SupervisorJob() + Dispatchers.Main)
|
||||
|
||||
connection = AssistantConnectionImpl(httpClient)
|
||||
|
||||
sttWrapper = SpeechRecognizerWrapper(this)
|
||||
ttsWrapper = TextToSpeechWrapper(this, ttsSettingsRepo.ttsConfig)
|
||||
cameraCaptureManager = CameraCaptureManager(this)
|
||||
|
||||
assistantManager = AssistantManager(
|
||||
sttWrapper = sttWrapper,
|
||||
ttsWrapper = ttsWrapper,
|
||||
connection = connection,
|
||||
cameraCaptureManager = cameraCaptureManager,
|
||||
contentResolver = contentResolver
|
||||
)
|
||||
}
|
||||
|
||||
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
|
||||
super.onStartCommand(intent, flags, startId)
|
||||
|
||||
startForeground(NOTIFICATION_ID, buildNotification())
|
||||
|
||||
// Resolve wsUrl from the main WebSocketClient
|
||||
val mainWsClient: WebSocketClient by inject()
|
||||
connection.connect(mainWsClient.wsUrl)
|
||||
|
||||
addOverlay()
|
||||
assistantManager.start(serviceScope)
|
||||
|
||||
return START_NOT_STICKY
|
||||
}
|
||||
|
||||
override fun onDestroy() {
|
||||
removeOverlay()
|
||||
assistantManager.destroy()
|
||||
ttsWrapper.destroy()
|
||||
connection.disconnect()
|
||||
serviceScope.cancel()
|
||||
super.onDestroy()
|
||||
}
|
||||
|
||||
override fun onBind(intent: Intent): IBinder? {
|
||||
super.onBind(intent)
|
||||
return null
|
||||
}
|
||||
|
||||
private fun shutdown() {
|
||||
stopForeground(STOP_FOREGROUND_REMOVE)
|
||||
stopSelf()
|
||||
}
|
||||
|
||||
private fun addOverlay() {
|
||||
if (overlayView != null) return
|
||||
|
||||
val params = WindowManager.LayoutParams(
|
||||
WindowManager.LayoutParams.MATCH_PARENT,
|
||||
WindowManager.LayoutParams.WRAP_CONTENT,
|
||||
WindowManager.LayoutParams.TYPE_APPLICATION_OVERLAY,
|
||||
WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL or
|
||||
WindowManager.LayoutParams.FLAG_LAYOUT_IN_SCREEN,
|
||||
PixelFormat.TRANSLUCENT
|
||||
).apply {
|
||||
gravity = Gravity.BOTTOM
|
||||
}
|
||||
|
||||
val view = ComposeView(this).apply {
|
||||
setViewTreeLifecycleOwner(this@AssistantService)
|
||||
setViewTreeSavedStateRegistryOwner(this@AssistantService)
|
||||
|
||||
setContent {
|
||||
PicoClawTheme {
|
||||
val state by assistantManager.state.collectAsState()
|
||||
AssistantPillBar(
|
||||
state = state,
|
||||
onClose = { shutdown() },
|
||||
onInterrupt = { assistantManager.interrupt() },
|
||||
onCameraToggle = { assistantManager.toggleCamera() },
|
||||
cameraCaptureManager = cameraCaptureManager
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
windowManager.addView(view, params)
|
||||
overlayView = view
|
||||
}
|
||||
|
||||
private fun removeOverlay() {
|
||||
overlayView?.let {
|
||||
windowManager.removeView(it)
|
||||
overlayView = null
|
||||
}
|
||||
}
|
||||
|
||||
private fun buildNotification(): Notification {
|
||||
return NotificationCompat.Builder(this, NotificationHelper.ASSISTANT_CHANNEL_ID)
|
||||
.setSmallIcon(android.R.drawable.ic_btn_speak_now)
|
||||
.setContentTitle("PicoClaw Assistant")
|
||||
.setContentText("Listening...")
|
||||
.setPriority(NotificationCompat.PRIORITY_LOW)
|
||||
.setOngoing(true)
|
||||
.build()
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val NOTIFICATION_ID = 2001
|
||||
}
|
||||
}
|
||||
|
|
@ -12,16 +12,30 @@ object NotificationHelper {
|
|||
private const val CHANNEL_NAME = "Agent Messages"
|
||||
private const val NOTIFICATION_ID = 1001
|
||||
|
||||
const val ASSISTANT_CHANNEL_ID = "picoclaw_assistant"
|
||||
private const val ASSISTANT_CHANNEL_NAME = "Assistant"
|
||||
|
||||
fun createNotificationChannel(context: Context) {
|
||||
val channel = NotificationChannel(
|
||||
val manager = context.getSystemService(NotificationManager::class.java)
|
||||
|
||||
val messageChannel = NotificationChannel(
|
||||
CHANNEL_ID,
|
||||
CHANNEL_NAME,
|
||||
NotificationManager.IMPORTANCE_DEFAULT
|
||||
).apply {
|
||||
description = "Messages from PicoClaw agent"
|
||||
}
|
||||
val manager = context.getSystemService(NotificationManager::class.java)
|
||||
manager.createNotificationChannel(channel)
|
||||
manager.createNotificationChannel(messageChannel)
|
||||
|
||||
val assistantChannel = NotificationChannel(
|
||||
ASSISTANT_CHANNEL_ID,
|
||||
ASSISTANT_CHANNEL_NAME,
|
||||
NotificationManager.IMPORTANCE_LOW
|
||||
).apply {
|
||||
description = "Assistant overlay service"
|
||||
setShowBadge(false)
|
||||
}
|
||||
manager.createNotificationChannel(assistantChannel)
|
||||
}
|
||||
|
||||
fun showMessageNotification(context: Context, content: String) {
|
||||
|
|
|
|||
|
|
@ -26,7 +26,8 @@ import kotlinx.serialization.json.Json
|
|||
class WebSocketClient(
|
||||
private val client: HttpClient,
|
||||
private val scope: CoroutineScope,
|
||||
private val clientId: String
|
||||
private val clientId: String,
|
||||
private val clientType: String = "main"
|
||||
) {
|
||||
|
||||
private val _connectionState = MutableStateFlow(ConnectionState.DISCONNECTED)
|
||||
|
|
@ -48,7 +49,7 @@ class WebSocketClient(
|
|||
while (isActive) {
|
||||
try {
|
||||
_connectionState.value = ConnectionState.CONNECTING
|
||||
val url = "$wsUrl?client_id=$clientId&client_type=main"
|
||||
val url = "$wsUrl?client_id=$clientId&client_type=$clientType"
|
||||
client.webSocket(url) {
|
||||
session = this
|
||||
_connectionState.value = ConnectionState.CONNECTED
|
||||
|
|
|
|||
|
|
@ -0,0 +1,71 @@
|
|||
package io.picoclaw.android.core.data.repository
|
||||
|
||||
import io.ktor.client.HttpClient
|
||||
import io.picoclaw.android.core.data.remote.WebSocketClient
|
||||
import io.picoclaw.android.core.data.remote.dto.WsIncoming
|
||||
import io.picoclaw.android.core.domain.model.AssistantMessage
|
||||
import io.picoclaw.android.core.domain.model.ConnectionState
|
||||
import io.picoclaw.android.core.domain.repository.AssistantConnection
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.cancel
|
||||
import kotlinx.coroutines.flow.MutableSharedFlow
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.SharedFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.flow.asSharedFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
import kotlinx.coroutines.launch
|
||||
import java.util.UUID
|
||||
|
||||
class AssistantConnectionImpl(
|
||||
private val httpClient: HttpClient
|
||||
) : AssistantConnection {
|
||||
|
||||
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val clientId = UUID.randomUUID().toString()
|
||||
private val wsClient = WebSocketClient(httpClient, scope, clientId, "assistant")
|
||||
|
||||
private val _messages = MutableSharedFlow<AssistantMessage>(extraBufferCapacity = 64)
|
||||
override val messages: SharedFlow<AssistantMessage> = _messages.asSharedFlow()
|
||||
|
||||
private val _statusText = MutableStateFlow<String?>(null)
|
||||
override val statusText: StateFlow<String?> = _statusText.asStateFlow()
|
||||
|
||||
override val connectionState: StateFlow<ConnectionState> = wsClient.connectionState
|
||||
|
||||
init {
|
||||
scope.launch {
|
||||
wsClient.incomingMessages.collect { dto ->
|
||||
when (dto.type) {
|
||||
"status" -> _statusText.value = dto.content
|
||||
"status_end" -> _statusText.value = null
|
||||
else -> {
|
||||
_statusText.value = null
|
||||
_messages.emit(AssistantMessage(content = dto.content, type = dto.type))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun connect(wsUrl: String) {
|
||||
wsClient.wsUrl = wsUrl
|
||||
wsClient.connect()
|
||||
}
|
||||
|
||||
override fun disconnect() {
|
||||
wsClient.disconnect()
|
||||
scope.cancel()
|
||||
}
|
||||
|
||||
override suspend fun send(text: String, images: List<String>, inputMode: String) {
|
||||
val dto = WsIncoming(
|
||||
content = text,
|
||||
images = images.ifEmpty { null },
|
||||
inputMode = inputMode
|
||||
)
|
||||
wsClient.send(dto)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
package io.picoclaw.android.core.domain.model
|
||||
|
||||
data class AssistantMessage(
|
||||
val content: String,
|
||||
val type: String? = null
|
||||
)
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
package io.picoclaw.android.core.domain.repository
|
||||
|
||||
import io.picoclaw.android.core.domain.model.AssistantMessage
|
||||
import io.picoclaw.android.core.domain.model.ConnectionState
|
||||
import kotlinx.coroutines.flow.SharedFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
|
||||
interface AssistantConnection {
|
||||
val messages: SharedFlow<AssistantMessage>
|
||||
val statusText: StateFlow<String?>
|
||||
val connectionState: StateFlow<ConnectionState>
|
||||
|
||||
fun connect(wsUrl: String)
|
||||
fun disconnect()
|
||||
suspend fun send(text: String, images: List<String> = emptyList(), inputMode: String = "assistant")
|
||||
}
|
||||
|
|
@ -0,0 +1,275 @@
|
|||
package io.picoclaw.android.feature.chat.assistant
|
||||
|
||||
import android.content.ContentResolver
|
||||
import android.speech.SpeechRecognizer
|
||||
import android.util.Base64
|
||||
import android.util.Log
|
||||
import io.picoclaw.android.core.domain.model.AssistantMessage
|
||||
import io.picoclaw.android.core.domain.model.VoicePhase
|
||||
import io.picoclaw.android.core.domain.repository.AssistantConnection
|
||||
import io.picoclaw.android.feature.chat.voice.CameraCaptureManager
|
||||
import io.picoclaw.android.feature.chat.voice.SpeechRecognizerWrapper
|
||||
import io.picoclaw.android.feature.chat.voice.SttResult
|
||||
import io.picoclaw.android.feature.chat.voice.TextToSpeechWrapper
|
||||
import io.picoclaw.android.feature.chat.voice.VoiceModeState
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.ExperimentalCoroutinesApi
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.channels.Channel
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
import kotlinx.coroutines.flow.update
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.selects.onTimeout
|
||||
import kotlinx.coroutines.selects.select
|
||||
|
||||
@OptIn(ExperimentalCoroutinesApi::class)
|
||||
class AssistantManager(
|
||||
private val sttWrapper: SpeechRecognizerWrapper,
|
||||
private val ttsWrapper: TextToSpeechWrapper,
|
||||
private val connection: AssistantConnection,
|
||||
private val cameraCaptureManager: CameraCaptureManager,
|
||||
private val contentResolver: ContentResolver
|
||||
) {
|
||||
|
||||
private val _state = MutableStateFlow(VoiceModeState())
|
||||
val state: StateFlow<VoiceModeState> = _state.asStateFlow()
|
||||
|
||||
private var loopJob: Job? = null
|
||||
private var parentScope: CoroutineScope? = null
|
||||
|
||||
fun toggleCamera() {
|
||||
_state.update { it.copy(isCameraActive = !it.isCameraActive) }
|
||||
}
|
||||
|
||||
fun start(scope: CoroutineScope) {
|
||||
if (loopJob?.isActive == true) return
|
||||
parentScope = scope
|
||||
_state.update {
|
||||
VoiceModeState(isActive = true, phase = VoicePhase.LISTENING, isCameraActive = it.isCameraActive)
|
||||
}
|
||||
loopJob = scope.launch {
|
||||
voiceLoop()
|
||||
}
|
||||
}
|
||||
|
||||
fun stop() {
|
||||
loopJob?.cancel()
|
||||
loopJob = null
|
||||
parentScope = null
|
||||
ttsWrapper.stop()
|
||||
cameraCaptureManager.unbind()
|
||||
_state.value = VoiceModeState()
|
||||
}
|
||||
|
||||
fun interrupt() {
|
||||
val scope = parentScope ?: return
|
||||
if (loopJob?.isActive != true) return
|
||||
|
||||
ttsWrapper.stop()
|
||||
loopJob?.cancel()
|
||||
loopJob = null
|
||||
|
||||
_state.update {
|
||||
VoiceModeState(isActive = true, phase = VoicePhase.LISTENING, isCameraActive = it.isCameraActive)
|
||||
}
|
||||
loopJob = scope.launch { voiceLoop() }
|
||||
}
|
||||
|
||||
fun destroy() {
|
||||
stop()
|
||||
}
|
||||
|
||||
private suspend fun voiceLoop() = coroutineScope {
|
||||
val speechQueue = Channel<String>(Channel.UNLIMITED)
|
||||
|
||||
val messageCollectorJob = launch {
|
||||
connection.messages.collect { msg: AssistantMessage ->
|
||||
if (msg.type == "warning" || msg.type == "error") {
|
||||
_state.update { it.copy(responseText = msg.content) }
|
||||
} else {
|
||||
speechQueue.send(msg.content)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val statusCollectorJob = launch {
|
||||
connection.statusText.collect { label ->
|
||||
_state.update { it.copy(statusText = label) }
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
while (isActive) {
|
||||
_state.update {
|
||||
it.copy(
|
||||
phase = VoicePhase.LISTENING,
|
||||
recognizedText = "", responseText = "",
|
||||
statusText = null,
|
||||
errorMessage = null, amplitudeNormalized = 0f
|
||||
)
|
||||
}
|
||||
|
||||
val userTextChannel = Channel<String?>(1)
|
||||
val listenJob = launch {
|
||||
val text = listen()
|
||||
userTextChannel.send(text)
|
||||
}
|
||||
|
||||
select<Unit> {
|
||||
userTextChannel.onReceive { text ->
|
||||
if (!text.isNullOrBlank()) {
|
||||
_state.update { it.copy(phase = VoicePhase.SENDING, recognizedText = text) }
|
||||
try {
|
||||
val base64Images = if (_state.value.isCameraActive) {
|
||||
captureAndEncode()
|
||||
} else emptyList()
|
||||
connection.send(text, base64Images)
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "Failed to send message", e)
|
||||
_state.update {
|
||||
it.copy(phase = VoicePhase.ERROR, errorMessage = "Failed to send")
|
||||
}
|
||||
delay(2000)
|
||||
return@onReceive
|
||||
}
|
||||
awaitAndSpeakResponse(speechQueue)
|
||||
} else {
|
||||
drainSpeechQueue(speechQueue)
|
||||
}
|
||||
}
|
||||
speechQueue.onReceive { content ->
|
||||
listenJob.cancel()
|
||||
speakAndDrain(content, speechQueue)
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
messageCollectorJob.cancel()
|
||||
statusCollectorJob.cancel()
|
||||
speechQueue.close()
|
||||
}
|
||||
}
|
||||
|
||||
private sealed interface WaitResult {
|
||||
data class Message(val content: String) : WaitResult
|
||||
data object Heartbeat : WaitResult
|
||||
data object Timeout : WaitResult
|
||||
}
|
||||
|
||||
private suspend fun awaitAndSpeakResponse(
|
||||
speechQueue: Channel<String>
|
||||
) = coroutineScope {
|
||||
val heartbeat = Channel<Unit>(Channel.CONFLATED)
|
||||
|
||||
val statusJob = launch {
|
||||
connection.statusText.collect { label ->
|
||||
if (label != null) heartbeat.trySend(Unit)
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
_state.update { it.copy(phase = VoicePhase.THINKING, statusText = null) }
|
||||
|
||||
while (true) {
|
||||
val result = select<WaitResult> {
|
||||
speechQueue.onReceive { WaitResult.Message(it) }
|
||||
heartbeat.onReceive { WaitResult.Heartbeat }
|
||||
onTimeout(30_000) { WaitResult.Timeout }
|
||||
}
|
||||
|
||||
when (result) {
|
||||
WaitResult.Timeout -> {
|
||||
_state.update {
|
||||
it.copy(phase = VoicePhase.ERROR, errorMessage = "Response timed out")
|
||||
}
|
||||
delay(2000)
|
||||
return@coroutineScope
|
||||
}
|
||||
WaitResult.Heartbeat -> continue
|
||||
is WaitResult.Message -> {
|
||||
speakAndDrain(result.content, speechQueue)
|
||||
val currentStatus = connection.statusText.value
|
||||
if (currentStatus == null) return@coroutineScope
|
||||
_state.update {
|
||||
it.copy(phase = VoicePhase.THINKING, statusText = currentStatus)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
statusJob.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun speakAndDrain(firstContent: String, speechQueue: Channel<String>) {
|
||||
_state.update { it.copy(phase = VoicePhase.SPEAKING, responseText = firstContent) }
|
||||
ttsWrapper.speak(firstContent)
|
||||
while (true) {
|
||||
val next = speechQueue.tryReceive().getOrNull() ?: break
|
||||
_state.update { it.copy(responseText = next) }
|
||||
ttsWrapper.speak(next)
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun drainSpeechQueue(speechQueue: Channel<String>) {
|
||||
val first = speechQueue.tryReceive().getOrNull() ?: return
|
||||
speakAndDrain(first, speechQueue)
|
||||
}
|
||||
|
||||
private suspend fun listen(): String? {
|
||||
var finalText: String? = null
|
||||
sttWrapper.startListening().collect { result ->
|
||||
when (result) {
|
||||
is SttResult.Partial -> {
|
||||
_state.update { it.copy(recognizedText = result.text) }
|
||||
}
|
||||
is SttResult.Final -> {
|
||||
finalText = result.text
|
||||
}
|
||||
is SttResult.RmsChanged -> {
|
||||
val normalized = ((result.rms + 2f) / 12f).coerceIn(0f, 1f)
|
||||
_state.update { it.copy(amplitudeNormalized = normalized) }
|
||||
}
|
||||
is SttResult.Error -> {
|
||||
if (result.code == SpeechRecognizer.ERROR_NO_MATCH ||
|
||||
result.code == SpeechRecognizer.ERROR_SPEECH_TIMEOUT
|
||||
) {
|
||||
finalText = ""
|
||||
} else {
|
||||
_state.update {
|
||||
it.copy(
|
||||
phase = VoicePhase.ERROR,
|
||||
errorMessage = "Speech recognition error (code=${result.code})"
|
||||
)
|
||||
}
|
||||
delay(2000)
|
||||
finalText = null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return finalText
|
||||
}
|
||||
|
||||
private suspend fun captureAndEncode(): List<String> {
|
||||
val attachment = cameraCaptureManager.captureFrame() ?: return emptyList()
|
||||
val uri = android.net.Uri.parse(attachment.uri)
|
||||
return try {
|
||||
val bytes = contentResolver.openInputStream(uri)?.use { it.readBytes() }
|
||||
?: return emptyList()
|
||||
listOf(Base64.encodeToString(bytes, Base64.NO_WRAP))
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "Failed to encode captured image", e)
|
||||
emptyList()
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val TAG = "AssistantManager"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,312 @@
|
|||
package io.picoclaw.android.feature.chat.assistant
|
||||
|
||||
import androidx.camera.view.PreviewView
|
||||
import androidx.compose.animation.AnimatedVisibility
|
||||
import androidx.compose.animation.animateContentSize
|
||||
import androidx.compose.animation.core.LinearEasing
|
||||
import androidx.compose.animation.core.RepeatMode
|
||||
import androidx.compose.animation.core.animateFloat
|
||||
import androidx.compose.animation.core.infiniteRepeatable
|
||||
import androidx.compose.animation.core.rememberInfiniteTransition
|
||||
import androidx.compose.animation.core.tween
|
||||
import androidx.compose.animation.expandVertically
|
||||
import androidx.compose.animation.shrinkVertically
|
||||
import androidx.compose.foundation.Canvas
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.clickable
|
||||
import androidx.compose.foundation.interaction.MutableInteractionSource
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.layout.width
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.IconButton
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.DisposableEffect
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.draw.clip
|
||||
import androidx.compose.ui.geometry.Offset
|
||||
import androidx.compose.ui.graphics.Brush
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.res.painterResource
|
||||
import androidx.compose.ui.text.style.TextOverflow
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.viewinterop.AndroidView
|
||||
import androidx.lifecycle.compose.LocalLifecycleOwner
|
||||
import io.picoclaw.android.core.domain.model.VoicePhase
|
||||
import io.picoclaw.android.core.ui.theme.GradientCyan
|
||||
import io.picoclaw.android.core.ui.theme.TextPrimary
|
||||
import io.picoclaw.android.core.ui.theme.TextSecondary
|
||||
import io.picoclaw.android.feature.chat.voice.CameraCaptureManager
|
||||
import io.picoclaw.android.feature.chat.voice.VoiceModeState
|
||||
import com.composables.icons.lucide.R as LucideR
|
||||
import kotlin.math.PI
|
||||
import kotlin.math.sin
|
||||
|
||||
private val PillBackground = Color(0xE6141428)
|
||||
private val ListeningColor = Color(0xFF00D4FF)
|
||||
private val ThinkingColor = Color(0xFFA855F7)
|
||||
private val SpeakingColor = Color(0xFF22C55E)
|
||||
private val ErrorColor = Color(0xFFEF4444)
|
||||
|
||||
@Composable
|
||||
fun AssistantPillBar(
|
||||
state: VoiceModeState,
|
||||
onClose: () -> Unit,
|
||||
onInterrupt: () -> Unit,
|
||||
onCameraToggle: () -> Unit,
|
||||
cameraCaptureManager: CameraCaptureManager,
|
||||
modifier: Modifier = Modifier
|
||||
) {
|
||||
val isExpanded = state.phase == VoicePhase.THINKING ||
|
||||
state.phase == VoicePhase.SPEAKING ||
|
||||
state.phase == VoicePhase.ERROR
|
||||
|
||||
val interruptable = state.phase != VoicePhase.LISTENING &&
|
||||
state.phase != VoicePhase.IDLE
|
||||
|
||||
Column(
|
||||
modifier = modifier
|
||||
.fillMaxWidth()
|
||||
.padding(horizontal = 8.dp, vertical = 8.dp)
|
||||
) {
|
||||
// Camera preview above the pill bar
|
||||
AnimatedVisibility(
|
||||
visible = state.isCameraActive,
|
||||
enter = expandVertically(expandFrom = Alignment.Bottom),
|
||||
exit = shrinkVertically(shrinkTowards = Alignment.Bottom)
|
||||
) {
|
||||
val lifecycleOwner = LocalLifecycleOwner.current
|
||||
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.padding(bottom = 8.dp),
|
||||
contentAlignment = Alignment.CenterEnd
|
||||
) {
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.width(120.dp)
|
||||
.height(90.dp)
|
||||
.clip(RoundedCornerShape(12.dp))
|
||||
) {
|
||||
AndroidView(
|
||||
factory = { ctx ->
|
||||
PreviewView(ctx).also { preview ->
|
||||
cameraCaptureManager.bind(lifecycleOwner, preview)
|
||||
}
|
||||
},
|
||||
modifier = Modifier.fillMaxSize()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
DisposableEffect(Unit) {
|
||||
onDispose {
|
||||
cameraCaptureManager.unbind()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Main pill bar
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.clip(RoundedCornerShape(24.dp))
|
||||
.background(
|
||||
Brush.verticalGradient(
|
||||
colors = listOf(
|
||||
PillBackground,
|
||||
PillBackground.copy(alpha = 0.95f)
|
||||
)
|
||||
)
|
||||
)
|
||||
.then(
|
||||
if (interruptable) {
|
||||
Modifier.clickable(
|
||||
indication = null,
|
||||
interactionSource = remember { MutableInteractionSource() }
|
||||
) { onInterrupt() }
|
||||
} else Modifier
|
||||
)
|
||||
.animateContentSize()
|
||||
) {
|
||||
Column(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.padding(16.dp)
|
||||
) {
|
||||
// Expanded content: response text area
|
||||
AnimatedVisibility(
|
||||
visible = isExpanded && (state.responseText.isNotEmpty() || state.errorMessage != null)
|
||||
) {
|
||||
Column(modifier = Modifier.padding(bottom = 12.dp)) {
|
||||
if (state.responseText.isNotEmpty()) {
|
||||
Text(
|
||||
text = state.responseText,
|
||||
style = MaterialTheme.typography.bodyMedium,
|
||||
color = TextPrimary,
|
||||
maxLines = 6,
|
||||
overflow = TextOverflow.Ellipsis
|
||||
)
|
||||
}
|
||||
if (state.errorMessage != null) {
|
||||
Text(
|
||||
text = state.errorMessage,
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = ErrorColor
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Status / recognized text row
|
||||
AnimatedVisibility(
|
||||
visible = isExpanded && (state.recognizedText.isNotEmpty() || state.statusText != null)
|
||||
) {
|
||||
Text(
|
||||
text = state.statusText ?: state.recognizedText,
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = TextSecondary,
|
||||
maxLines = 1,
|
||||
overflow = TextOverflow.Ellipsis,
|
||||
modifier = Modifier.padding(bottom = 8.dp)
|
||||
)
|
||||
}
|
||||
|
||||
// Bottom row: waveform + controls
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
verticalAlignment = Alignment.CenterVertically
|
||||
) {
|
||||
// Mic icon
|
||||
Icon(
|
||||
painter = painterResource(LucideR.drawable.lucide_ic_mic),
|
||||
contentDescription = "Microphone",
|
||||
modifier = Modifier.size(20.dp),
|
||||
tint = phaseColor(state.phase)
|
||||
)
|
||||
|
||||
Spacer(modifier = Modifier.width(8.dp))
|
||||
|
||||
// Waveform
|
||||
WaveformBar(
|
||||
phase = state.phase,
|
||||
amplitude = state.amplitudeNormalized,
|
||||
modifier = Modifier
|
||||
.weight(1f)
|
||||
.height(28.dp)
|
||||
)
|
||||
|
||||
Spacer(modifier = Modifier.width(8.dp))
|
||||
|
||||
// Camera toggle
|
||||
IconButton(
|
||||
onClick = onCameraToggle,
|
||||
modifier = Modifier.size(36.dp)
|
||||
) {
|
||||
Icon(
|
||||
painter = painterResource(
|
||||
if (state.isCameraActive) LucideR.drawable.lucide_ic_camera_off
|
||||
else LucideR.drawable.lucide_ic_camera
|
||||
),
|
||||
contentDescription = if (state.isCameraActive) "Turn off camera" else "Turn on camera",
|
||||
modifier = Modifier.size(18.dp),
|
||||
tint = if (state.isCameraActive) GradientCyan else TextSecondary
|
||||
)
|
||||
}
|
||||
|
||||
// Close button
|
||||
IconButton(
|
||||
onClick = onClose,
|
||||
modifier = Modifier.size(36.dp)
|
||||
) {
|
||||
Icon(
|
||||
painter = painterResource(LucideR.drawable.lucide_ic_x),
|
||||
contentDescription = "Close",
|
||||
modifier = Modifier.size(18.dp),
|
||||
tint = TextSecondary
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun WaveformBar(
|
||||
phase: VoicePhase,
|
||||
amplitude: Float,
|
||||
modifier: Modifier = Modifier
|
||||
) {
|
||||
val transition = rememberInfiniteTransition(label = "waveform")
|
||||
val animPhase by transition.animateFloat(
|
||||
initialValue = 0f,
|
||||
targetValue = 2f * PI.toFloat(),
|
||||
animationSpec = infiniteRepeatable(
|
||||
animation = tween(durationMillis = 1500, easing = LinearEasing),
|
||||
repeatMode = RepeatMode.Restart
|
||||
),
|
||||
label = "wavePhase"
|
||||
)
|
||||
|
||||
val color = phaseColor(phase)
|
||||
|
||||
Canvas(modifier = modifier) {
|
||||
val barCount = 32
|
||||
val barWidth = size.width / barCount
|
||||
val centerY = size.height / 2f
|
||||
|
||||
for (i in 0 until barCount) {
|
||||
val x = i * barWidth + barWidth / 2f
|
||||
val normalizedX = i.toFloat() / barCount
|
||||
|
||||
val barHeight = when (phase) {
|
||||
VoicePhase.LISTENING -> {
|
||||
val wave = sin(normalizedX * 4f * PI.toFloat() + animPhase).coerceIn(-1f, 1f)
|
||||
val base = 0.15f
|
||||
val dynamic = amplitude * 0.85f * ((wave + 1f) / 2f)
|
||||
(base + dynamic) * size.height
|
||||
}
|
||||
VoicePhase.THINKING -> {
|
||||
val wave = sin(normalizedX * 3f * PI.toFloat() + animPhase * 2f)
|
||||
(0.2f + 0.15f * ((wave + 1f) / 2f)) * size.height
|
||||
}
|
||||
VoicePhase.SPEAKING -> {
|
||||
val wave = sin(normalizedX * 5f * PI.toFloat() + animPhase * 1.5f)
|
||||
(0.2f + 0.4f * ((wave + 1f) / 2f)) * size.height
|
||||
}
|
||||
else -> 0.1f * size.height
|
||||
}
|
||||
|
||||
drawLine(
|
||||
color = color.copy(alpha = 0.6f + 0.4f * (barHeight / size.height)),
|
||||
start = Offset(x, centerY - barHeight / 2f),
|
||||
end = Offset(x, centerY + barHeight / 2f),
|
||||
strokeWidth = barWidth * 0.5f
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun phaseColor(phase: VoicePhase): Color = when (phase) {
|
||||
VoicePhase.LISTENING -> ListeningColor
|
||||
VoicePhase.SENDING -> Color(0xFFFF8C42)
|
||||
VoicePhase.THINKING -> ThinkingColor
|
||||
VoicePhase.SPEAKING -> SpeakingColor
|
||||
VoicePhase.ERROR -> ErrorColor
|
||||
VoicePhase.IDLE -> Color(0xFF4A5568)
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue