feat: add input_mode flag for voice-optimized LLM responses
Voice mode sends input_mode="voice" through the WebSocket, causing the server to append voice-specific system prompt instructions that produce short, conversational responses without markdown formatting for TTS. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Kohei
parent
87dac508c9
commit
c7f6095904
10 changed files with 60 additions and 15 deletions
|
|
@ -66,10 +66,11 @@ object MessageMapper {
|
|||
)
|
||||
}
|
||||
|
||||
fun toWsIncoming(text: String, base64Images: List<String>): WsIncoming {
|
||||
fun toWsIncoming(text: String, base64Images: List<String>, inputMode: String? = null): WsIncoming {
|
||||
return WsIncoming(
|
||||
content = text,
|
||||
images = base64Images.ifEmpty { null }
|
||||
images = base64Images.ifEmpty { null },
|
||||
inputMode = inputMode
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,5 +7,6 @@ import kotlinx.serialization.Serializable
|
|||
data class WsIncoming(
|
||||
val content: String,
|
||||
@SerialName("sender_id") val senderId: String? = null,
|
||||
val images: List<String>? = null
|
||||
val images: List<String>? = null,
|
||||
@SerialName("input_mode") val inputMode: String? = null
|
||||
)
|
||||
|
|
|
|||
|
|
@ -57,11 +57,11 @@ class ChatRepositoryImpl(
|
|||
}
|
||||
}
|
||||
|
||||
override suspend fun sendMessage(text: String, images: List<ImageAttachment>) {
|
||||
override suspend fun sendMessage(text: String, images: List<ImageAttachment>, inputMode: String?) {
|
||||
val results = images.map { imageFileStorage.saveFromUri(it.uri) }
|
||||
val entity = MessageMapper.toEntity(text, results.map { it.imageData }, MessageStatus.SENDING)
|
||||
messageDao.insert(entity)
|
||||
val wsDto = MessageMapper.toWsIncoming(text, results.map { it.base64 })
|
||||
val wsDto = MessageMapper.toWsIncoming(text, results.map { it.base64 }, inputMode)
|
||||
val success = webSocketClient.send(wsDto)
|
||||
messageDao.update(entity.copy(status = if (success) MessageStatus.SENT.name else MessageStatus.FAILED.name))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ interface ChatRepository {
|
|||
val messages: StateFlow<List<ChatMessage>>
|
||||
val connectionState: StateFlow<ConnectionState>
|
||||
val statusLabel: StateFlow<String?>
|
||||
suspend fun sendMessage(text: String, images: List<ImageAttachment> = emptyList())
|
||||
suspend fun sendMessage(text: String, images: List<ImageAttachment> = emptyList(), inputMode: String? = null)
|
||||
fun loadMore()
|
||||
fun connect()
|
||||
fun disconnect()
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import io.picoclaw.android.core.domain.model.ImageAttachment
|
|||
import io.picoclaw.android.core.domain.repository.ChatRepository
|
||||
|
||||
class SendMessageUseCase(private val repository: ChatRepository) {
|
||||
suspend operator fun invoke(text: String, images: List<ImageAttachment> = emptyList()) {
|
||||
repository.sendMessage(text, images)
|
||||
suspend operator fun invoke(text: String, images: List<ImageAttachment> = emptyList(), inputMode: String? = null) {
|
||||
repository.sendMessage(text, images, inputMode)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ class VoiceModeManager(
|
|||
if (!text.isNullOrBlank()) {
|
||||
_state.update { it.copy(phase = VoicePhase.SENDING, recognizedText = text) }
|
||||
try {
|
||||
sendMessage(text)
|
||||
sendMessage(text, inputMode = "voice")
|
||||
} catch (e: Exception) {
|
||||
_state.update {
|
||||
it.copy(phase = VoicePhase.ERROR, errorMessage = "送信に失敗しました")
|
||||
|
|
|
|||
|
|
@ -190,14 +190,20 @@ func (cb *ContextBuilder) LoadBootstrapFiles() string {
|
|||
return result
|
||||
}
|
||||
|
||||
func (cb *ContextBuilder) BuildMessages(history []providers.Message, summary string, currentMessage string, media []string, channel, chatID string) []providers.Message {
|
||||
func (cb *ContextBuilder) BuildMessages(history []providers.Message, summary string, currentMessage string, media []string, channel, chatID, inputMode string) []providers.Message {
|
||||
messages := []providers.Message{}
|
||||
|
||||
systemPrompt := cb.BuildSystemPrompt()
|
||||
|
||||
// Add Current Session info if provided
|
||||
if channel != "" && chatID != "" {
|
||||
systemPrompt += fmt.Sprintf("\n\n## Current Session\nChannel: %s\nChat ID: %s", channel, chatID)
|
||||
systemPrompt += fmt.Sprintf("\n\n## Current Session\nChannel: %s\nChat ID: %s\nInput Mode: %s",
|
||||
channel, chatID, inputMode)
|
||||
}
|
||||
|
||||
// Add voice mode instructions when input is from voice
|
||||
if inputMode == "voice" {
|
||||
systemPrompt += voiceModePrompt()
|
||||
}
|
||||
|
||||
// Log system prompt summary for debugging (debug mode only)
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ type processOptions struct {
|
|||
EnableSummary bool // Whether to trigger summarization
|
||||
SendResponse bool // Whether to send response via bus
|
||||
NoHistory bool // If true, don't load session history (for heartbeat)
|
||||
InputMode string // "voice" or "text"
|
||||
}
|
||||
|
||||
// createToolRegistry creates a tool registry with common tools.
|
||||
|
|
@ -367,6 +368,14 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage)
|
|||
return response, nil
|
||||
}
|
||||
|
||||
// Extract input_mode from metadata
|
||||
inputMode := "text"
|
||||
if msg.Metadata != nil {
|
||||
if mode, ok := msg.Metadata["input_mode"]; ok && mode != "" {
|
||||
inputMode = mode
|
||||
}
|
||||
}
|
||||
|
||||
// Process as user message
|
||||
return al.runAgentLoop(ctx, processOptions{
|
||||
SessionKey: msg.SessionKey,
|
||||
|
|
@ -377,6 +386,7 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage)
|
|||
DefaultResponse: "I've completed processing but have no response to give.",
|
||||
EnableSummary: true,
|
||||
SendResponse: false,
|
||||
InputMode: inputMode,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -463,6 +473,7 @@ func (al *AgentLoop) runAgentLoop(ctx context.Context, opts processOptions) (str
|
|||
opts.Media,
|
||||
opts.Channel,
|
||||
opts.ChatID,
|
||||
opts.InputMode,
|
||||
)
|
||||
|
||||
// 3. Save user message to session
|
||||
|
|
@ -657,6 +668,7 @@ func (al *AgentLoop) runLLMIteration(ctx context.Context, messages []providers.M
|
|||
nil,
|
||||
opts.Channel,
|
||||
opts.ChatID,
|
||||
opts.InputMode,
|
||||
)
|
||||
|
||||
// Important: If we are in the middle of a tool loop (iteration > 1),
|
||||
|
|
@ -720,6 +732,7 @@ func (al *AgentLoop) runLLMIteration(ctx context.Context, messages []providers.M
|
|||
nil,
|
||||
opts.Channel,
|
||||
opts.ChatID,
|
||||
opts.InputMode,
|
||||
)
|
||||
|
||||
continue
|
||||
|
|
|
|||
16
pkg/agent/voice_prompt.go
Normal file
16
pkg/agent/voice_prompt.go
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
package agent
|
||||
|
||||
func voiceModePrompt() string {
|
||||
return `
|
||||
|
||||
## Voice Mode Instructions
|
||||
|
||||
The user is currently speaking to you via voice input. Your response will be read aloud by text-to-speech.
|
||||
|
||||
- Keep responses short and conversational (1-3 sentences by default)
|
||||
- Do NOT use markdown formatting (no headers, bold, code blocks, tables, bullet lists)
|
||||
- Use natural spoken language as if having a conversation
|
||||
- Spell out numbers and avoid special characters that sound awkward when spoken
|
||||
- If the user explicitly asks for more detail, provide longer explanations but still in natural spoken language without markdown
|
||||
- If code, file contents, or highly technical output is needed, briefly summarize and suggest switching to text mode for the full details`
|
||||
}
|
||||
|
|
@ -16,9 +16,10 @@ import (
|
|||
|
||||
// wsIncoming is the JSON message sent from APK to picoclaw.
|
||||
type wsIncoming struct {
|
||||
Content string `json:"content"`
|
||||
SenderID string `json:"sender_id,omitempty"`
|
||||
Images []string `json:"images,omitempty"`
|
||||
Content string `json:"content"`
|
||||
SenderID string `json:"sender_id,omitempty"`
|
||||
Images []string `json:"images,omitempty"`
|
||||
InputMode string `json:"input_mode,omitempty"`
|
||||
}
|
||||
|
||||
// wsOutgoing is the JSON message sent from picoclaw to APK.
|
||||
|
|
@ -252,7 +253,14 @@ func (c *WebSocketChannel) readPump(conn *websocket.Conn, clientID, chatID strin
|
|||
"images": len(incoming.Images),
|
||||
})
|
||||
|
||||
c.HandleMessage(senderID, chatID, content, media, nil)
|
||||
inputMode := incoming.InputMode
|
||||
if inputMode == "" {
|
||||
inputMode = "text"
|
||||
}
|
||||
metadata := map[string]string{
|
||||
"input_mode": inputMode,
|
||||
}
|
||||
c.HandleMessage(senderID, chatID, content, media, metadata)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue