From c7f6095904521130e4517980673c50f382f47ecb Mon Sep 17 00:00:00 2001 From: Kohei Date: Thu, 19 Feb 2026 22:11:02 +0900 Subject: [PATCH] feat: add input_mode flag for voice-optimized LLM responses Voice mode sends input_mode="voice" through the WebSocket, causing the server to append voice-specific system prompt instructions that produce short, conversational responses without markdown formatting for TTS. Co-Authored-By: Claude Opus 4.6 --- .../android/core/data/mapper/MessageMapper.kt | 5 +++-- .../android/core/data/remote/dto/WsIncoming.kt | 3 ++- .../core/data/repository/ChatRepositoryImpl.kt | 4 ++-- .../core/domain/repository/ChatRepository.kt | 2 +- .../core/domain/usecase/SendMessageUseCase.kt | 4 ++-- .../feature/chat/voice/VoiceModeManager.kt | 2 +- pkg/agent/context.go | 10 ++++++++-- pkg/agent/loop.go | 13 +++++++++++++ pkg/agent/voice_prompt.go | 16 ++++++++++++++++ pkg/channels/websocket.go | 16 ++++++++++++---- 10 files changed, 60 insertions(+), 15 deletions(-) create mode 100644 pkg/agent/voice_prompt.go diff --git a/android/core/data/src/main/java/io/picoclaw/android/core/data/mapper/MessageMapper.kt b/android/core/data/src/main/java/io/picoclaw/android/core/data/mapper/MessageMapper.kt index bbf71b6f4..674ae404c 100644 --- a/android/core/data/src/main/java/io/picoclaw/android/core/data/mapper/MessageMapper.kt +++ b/android/core/data/src/main/java/io/picoclaw/android/core/data/mapper/MessageMapper.kt @@ -66,10 +66,11 @@ object MessageMapper { ) } - fun toWsIncoming(text: String, base64Images: List): WsIncoming { + fun toWsIncoming(text: String, base64Images: List, inputMode: String? = null): WsIncoming { return WsIncoming( content = text, - images = base64Images.ifEmpty { null } + images = base64Images.ifEmpty { null }, + inputMode = inputMode ) } } diff --git a/android/core/data/src/main/java/io/picoclaw/android/core/data/remote/dto/WsIncoming.kt b/android/core/data/src/main/java/io/picoclaw/android/core/data/remote/dto/WsIncoming.kt index ae53f91c3..ebc2add90 100644 --- a/android/core/data/src/main/java/io/picoclaw/android/core/data/remote/dto/WsIncoming.kt +++ b/android/core/data/src/main/java/io/picoclaw/android/core/data/remote/dto/WsIncoming.kt @@ -7,5 +7,6 @@ import kotlinx.serialization.Serializable data class WsIncoming( val content: String, @SerialName("sender_id") val senderId: String? = null, - val images: List? = null + val images: List? = null, + @SerialName("input_mode") val inputMode: String? = null ) diff --git a/android/core/data/src/main/java/io/picoclaw/android/core/data/repository/ChatRepositoryImpl.kt b/android/core/data/src/main/java/io/picoclaw/android/core/data/repository/ChatRepositoryImpl.kt index 3ce29a9a8..4b9355a4a 100644 --- a/android/core/data/src/main/java/io/picoclaw/android/core/data/repository/ChatRepositoryImpl.kt +++ b/android/core/data/src/main/java/io/picoclaw/android/core/data/repository/ChatRepositoryImpl.kt @@ -57,11 +57,11 @@ class ChatRepositoryImpl( } } - override suspend fun sendMessage(text: String, images: List) { + override suspend fun sendMessage(text: String, images: List, inputMode: String?) { val results = images.map { imageFileStorage.saveFromUri(it.uri) } val entity = MessageMapper.toEntity(text, results.map { it.imageData }, MessageStatus.SENDING) messageDao.insert(entity) - val wsDto = MessageMapper.toWsIncoming(text, results.map { it.base64 }) + val wsDto = MessageMapper.toWsIncoming(text, results.map { it.base64 }, inputMode) val success = webSocketClient.send(wsDto) messageDao.update(entity.copy(status = if (success) MessageStatus.SENT.name else MessageStatus.FAILED.name)) } diff --git a/android/core/domain/src/main/java/io/picoclaw/android/core/domain/repository/ChatRepository.kt b/android/core/domain/src/main/java/io/picoclaw/android/core/domain/repository/ChatRepository.kt index a4095b6e9..ddcc02b8f 100644 --- a/android/core/domain/src/main/java/io/picoclaw/android/core/domain/repository/ChatRepository.kt +++ b/android/core/domain/src/main/java/io/picoclaw/android/core/domain/repository/ChatRepository.kt @@ -9,7 +9,7 @@ interface ChatRepository { val messages: StateFlow> val connectionState: StateFlow val statusLabel: StateFlow - suspend fun sendMessage(text: String, images: List = emptyList()) + suspend fun sendMessage(text: String, images: List = emptyList(), inputMode: String? = null) fun loadMore() fun connect() fun disconnect() diff --git a/android/core/domain/src/main/java/io/picoclaw/android/core/domain/usecase/SendMessageUseCase.kt b/android/core/domain/src/main/java/io/picoclaw/android/core/domain/usecase/SendMessageUseCase.kt index 49eb8a02e..ea81a56e2 100644 --- a/android/core/domain/src/main/java/io/picoclaw/android/core/domain/usecase/SendMessageUseCase.kt +++ b/android/core/domain/src/main/java/io/picoclaw/android/core/domain/usecase/SendMessageUseCase.kt @@ -4,7 +4,7 @@ import io.picoclaw.android.core.domain.model.ImageAttachment import io.picoclaw.android.core.domain.repository.ChatRepository class SendMessageUseCase(private val repository: ChatRepository) { - suspend operator fun invoke(text: String, images: List = emptyList()) { - repository.sendMessage(text, images) + suspend operator fun invoke(text: String, images: List = emptyList(), inputMode: String? = null) { + repository.sendMessage(text, images, inputMode) } } diff --git a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeManager.kt b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeManager.kt index 06a6e7097..50bb7ae77 100644 --- a/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeManager.kt +++ b/android/feature/chat/src/main/java/io/picoclaw/android/feature/chat/voice/VoiceModeManager.kt @@ -119,7 +119,7 @@ class VoiceModeManager( if (!text.isNullOrBlank()) { _state.update { it.copy(phase = VoicePhase.SENDING, recognizedText = text) } try { - sendMessage(text) + sendMessage(text, inputMode = "voice") } catch (e: Exception) { _state.update { it.copy(phase = VoicePhase.ERROR, errorMessage = "送信に失敗しました") diff --git a/pkg/agent/context.go b/pkg/agent/context.go index 568e4ff1e..37de1c682 100644 --- a/pkg/agent/context.go +++ b/pkg/agent/context.go @@ -190,14 +190,20 @@ func (cb *ContextBuilder) LoadBootstrapFiles() string { return result } -func (cb *ContextBuilder) BuildMessages(history []providers.Message, summary string, currentMessage string, media []string, channel, chatID string) []providers.Message { +func (cb *ContextBuilder) BuildMessages(history []providers.Message, summary string, currentMessage string, media []string, channel, chatID, inputMode string) []providers.Message { messages := []providers.Message{} systemPrompt := cb.BuildSystemPrompt() // Add Current Session info if provided if channel != "" && chatID != "" { - systemPrompt += fmt.Sprintf("\n\n## Current Session\nChannel: %s\nChat ID: %s", channel, chatID) + systemPrompt += fmt.Sprintf("\n\n## Current Session\nChannel: %s\nChat ID: %s\nInput Mode: %s", + channel, chatID, inputMode) + } + + // Add voice mode instructions when input is from voice + if inputMode == "voice" { + systemPrompt += voiceModePrompt() } // Log system prompt summary for debugging (debug mode only) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index f40e037ba..54bc47d8f 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -67,6 +67,7 @@ type processOptions struct { EnableSummary bool // Whether to trigger summarization SendResponse bool // Whether to send response via bus NoHistory bool // If true, don't load session history (for heartbeat) + InputMode string // "voice" or "text" } // createToolRegistry creates a tool registry with common tools. @@ -367,6 +368,14 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage) return response, nil } + // Extract input_mode from metadata + inputMode := "text" + if msg.Metadata != nil { + if mode, ok := msg.Metadata["input_mode"]; ok && mode != "" { + inputMode = mode + } + } + // Process as user message return al.runAgentLoop(ctx, processOptions{ SessionKey: msg.SessionKey, @@ -377,6 +386,7 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage) DefaultResponse: "I've completed processing but have no response to give.", EnableSummary: true, SendResponse: false, + InputMode: inputMode, }) } @@ -463,6 +473,7 @@ func (al *AgentLoop) runAgentLoop(ctx context.Context, opts processOptions) (str opts.Media, opts.Channel, opts.ChatID, + opts.InputMode, ) // 3. Save user message to session @@ -657,6 +668,7 @@ func (al *AgentLoop) runLLMIteration(ctx context.Context, messages []providers.M nil, opts.Channel, opts.ChatID, + opts.InputMode, ) // Important: If we are in the middle of a tool loop (iteration > 1), @@ -720,6 +732,7 @@ func (al *AgentLoop) runLLMIteration(ctx context.Context, messages []providers.M nil, opts.Channel, opts.ChatID, + opts.InputMode, ) continue diff --git a/pkg/agent/voice_prompt.go b/pkg/agent/voice_prompt.go new file mode 100644 index 000000000..6b5f83766 --- /dev/null +++ b/pkg/agent/voice_prompt.go @@ -0,0 +1,16 @@ +package agent + +func voiceModePrompt() string { + return ` + +## Voice Mode Instructions + +The user is currently speaking to you via voice input. Your response will be read aloud by text-to-speech. + +- Keep responses short and conversational (1-3 sentences by default) +- Do NOT use markdown formatting (no headers, bold, code blocks, tables, bullet lists) +- Use natural spoken language as if having a conversation +- Spell out numbers and avoid special characters that sound awkward when spoken +- If the user explicitly asks for more detail, provide longer explanations but still in natural spoken language without markdown +- If code, file contents, or highly technical output is needed, briefly summarize and suggest switching to text mode for the full details` +} diff --git a/pkg/channels/websocket.go b/pkg/channels/websocket.go index c3ecbf5cf..67b376d8a 100644 --- a/pkg/channels/websocket.go +++ b/pkg/channels/websocket.go @@ -16,9 +16,10 @@ import ( // wsIncoming is the JSON message sent from APK to picoclaw. type wsIncoming struct { - Content string `json:"content"` - SenderID string `json:"sender_id,omitempty"` - Images []string `json:"images,omitempty"` + Content string `json:"content"` + SenderID string `json:"sender_id,omitempty"` + Images []string `json:"images,omitempty"` + InputMode string `json:"input_mode,omitempty"` } // wsOutgoing is the JSON message sent from picoclaw to APK. @@ -252,7 +253,14 @@ func (c *WebSocketChannel) readPump(conn *websocket.Conn, clientID, chatID strin "images": len(incoming.Images), }) - c.HandleMessage(senderID, chatID, content, media, nil) + inputMode := incoming.InputMode + if inputMode == "" { + inputMode = "text" + } + metadata := map[string]string{ + "input_mode": inputMode, + } + c.HandleMessage(senderID, chatID, content, media, metadata) } }