feat(voice/tts): add TTS synthesis and voice parameter on message tool
Adds outbound voice capability: the agent can now reply with audio by
setting voice=true on the message tool, useful when the user sends a voice
message or explicitly requests audio.
Changes:
- pkg/voice/synthesizer.go: Synthesizer interface (Synthesize, IsAvailable)
- pkg/voice/kokoro.go: KokoroSynthesizer — talks to any OpenAI-compatible
/v1/audio/speech endpoint (Kokoro, Piper, etc.). Health-check via GET
/v1/models. Returns a temp .mp3 path; caller cleans up.
- pkg/bus/types.go: add Media []string to OutboundMessage (backward-
compatible, omitempty). Enables any channel to receive file paths.
- pkg/channels/manager.go: add SendFileToChannel() — synchronous media
send that routes local file paths through the channel's Send().
- pkg/tools/message.go: add voice=true parameter + SynthesizeCallback +
SendMediaCallback. Voice path: synthesize → send file → cleanup.
Falls back to text if TTS unavailable. HasSentInRound fires for both.
- pkg/agent/loop.go: add SetVoiceCallbacks() to attach TTS to message tool
after channel manager is available.
- cmd/picoclaw/main.go: wire Kokoro TTS after channels init; attaches to
message tool via SetVoiceCallbacks().
Config example:
"tools": {
"tts": {
"enabled": true,
"api_base": "http://localhost:8100",
"voice": "en_us-lessac-medium"
}
}
Depends-on: feat(voice/stt): add local Whisper STT provider
This commit is contained in:
parent
21079e49ec
commit
c3a7629343
8 changed files with 288 additions and 15 deletions
|
|
@ -634,6 +634,27 @@ func gatewayCmd() {
|
|||
}
|
||||
}
|
||||
|
||||
// Attach TTS synthesis callbacks to the message tool (enables voice=true).
|
||||
if cfg.Tools.TTS.Enabled {
|
||||
synthesizer := voice.NewKokoroSynthesizer(cfg.Tools.TTS.APIBase, cfg.Tools.TTS.Voice)
|
||||
if synthesizer.IsAvailable() {
|
||||
logger.InfoCF("voice", "TTS enabled — voice=true supported in message tool", map[string]interface{}{
|
||||
"api_base": cfg.Tools.TTS.APIBase,
|
||||
"voice": cfg.Tools.TTS.Voice,
|
||||
})
|
||||
agentLoop.SetVoiceCallbacks(
|
||||
func(ctx context.Context, text string) (string, error) {
|
||||
return synthesizer.Synthesize(ctx, text)
|
||||
},
|
||||
func(ctx context.Context, channel, chatID string, filePaths []string) error {
|
||||
return channelManager.SendFileToChannel(ctx, channel, chatID, filePaths)
|
||||
},
|
||||
)
|
||||
} else {
|
||||
logger.WarnC("voice", "TTS configured but service not reachable — voice=true disabled")
|
||||
}
|
||||
}
|
||||
|
||||
enabledChannels := channelManager.GetEnabledChannels()
|
||||
if len(enabledChannels) > 0 {
|
||||
fmt.Printf("✓ Channels enabled: %s\n", enabledChannels)
|
||||
|
|
|
|||
|
|
@ -201,6 +201,17 @@ func (al *AgentLoop) RegisterTool(tool tools.Tool) {
|
|||
al.tools.Register(tool)
|
||||
}
|
||||
|
||||
// SetVoiceCallbacks attaches TTS synthesis and media-send callbacks to the
|
||||
// message tool so it can handle voice=true calls. Safe to call after init.
|
||||
func (al *AgentLoop) SetVoiceCallbacks(synth tools.SynthesizeCallback, sendMedia tools.SendMediaCallback) {
|
||||
if tool, ok := al.tools.Get("message"); ok {
|
||||
if mt, ok := tool.(*tools.MessageTool); ok {
|
||||
mt.SetSynthesizeCallback(synth)
|
||||
mt.SetSendMediaCallback(sendMedia)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (al *AgentLoop) SetChannelManager(cm *channels.Manager) {
|
||||
al.channelManager = cm
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,9 +11,10 @@ type InboundMessage struct {
|
|||
}
|
||||
|
||||
type OutboundMessage struct {
|
||||
Channel string `json:"channel"`
|
||||
ChatID string `json:"chat_id"`
|
||||
Content string `json:"content"`
|
||||
Channel string `json:"channel"`
|
||||
ChatID string `json:"chat_id"`
|
||||
Content string `json:"content"`
|
||||
Media []string `json:"media,omitempty"`
|
||||
}
|
||||
|
||||
type MessageHandler func(InboundMessage) error
|
||||
|
|
|
|||
|
|
@ -343,3 +343,23 @@ func (m *Manager) SendToChannel(ctx context.Context, channelName, chatID, conten
|
|||
|
||||
return channel.Send(ctx, msg)
|
||||
}
|
||||
|
||||
// SendFileToChannel sends one or more local media files to a channel synchronously.
|
||||
// The caller is responsible for cleaning up the files after this returns.
|
||||
func (m *Manager) SendFileToChannel(ctx context.Context, channelName, chatID string, filePaths []string) error {
|
||||
m.mu.RLock()
|
||||
channel, exists := m.channels[channelName]
|
||||
m.mu.RUnlock()
|
||||
|
||||
if !exists {
|
||||
return fmt.Errorf("channel %s not found", channelName)
|
||||
}
|
||||
|
||||
msg := bus.OutboundMessage{
|
||||
Channel: channelName,
|
||||
ChatID: chatID,
|
||||
Media: filePaths,
|
||||
}
|
||||
|
||||
return channel.Send(ctx, msg)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -216,9 +216,16 @@ type WhisperConfig struct {
|
|||
APIBase string `json:"api_base" env:"PICOCLAW_TOOLS_WHISPER_API_BASE"`
|
||||
}
|
||||
|
||||
type TTSConfig struct {
|
||||
Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_TTS_ENABLED"`
|
||||
APIBase string `json:"api_base" env:"PICOCLAW_TOOLS_TTS_API_BASE"`
|
||||
Voice string `json:"voice" env:"PICOCLAW_TOOLS_TTS_VOICE"`
|
||||
}
|
||||
|
||||
type ToolsConfig struct {
|
||||
Web WebToolsConfig `json:"web"`
|
||||
Whisper WhisperConfig `json:"whisper"`
|
||||
TTS TTSConfig `json:"tts"`
|
||||
}
|
||||
|
||||
func DefaultConfig() *Config {
|
||||
|
|
@ -332,6 +339,11 @@ func DefaultConfig() *Config {
|
|||
Enabled: false,
|
||||
APIBase: "http://localhost:8200",
|
||||
},
|
||||
TTS: TTSConfig{
|
||||
Enabled: false,
|
||||
APIBase: "http://localhost:8100",
|
||||
Voice: "en_us-lessac-medium",
|
||||
},
|
||||
},
|
||||
Heartbeat: HeartbeatConfig{
|
||||
Enabled: true,
|
||||
|
|
|
|||
|
|
@ -3,15 +3,27 @@ package tools
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
// SendCallback sends a plain-text message to a channel/chat.
|
||||
type SendCallback func(channel, chatID, content string) error
|
||||
|
||||
// SendMediaCallback sends one or more local media files to a channel/chat.
|
||||
// The callback owns the call; the caller is responsible for cleaning up files afterward.
|
||||
type SendMediaCallback func(ctx context.Context, channel, chatID string, filePaths []string) error
|
||||
|
||||
// SynthesizeCallback converts text to an audio file and returns the local path.
|
||||
// The caller must delete the file when done.
|
||||
type SynthesizeCallback func(ctx context.Context, text string) (filePath string, err error)
|
||||
|
||||
type MessageTool struct {
|
||||
sendCallback SendCallback
|
||||
defaultChannel string
|
||||
defaultChatID string
|
||||
sentInRound bool // Tracks whether a message was sent in the current processing round
|
||||
sendCallback SendCallback
|
||||
sendMediaCallback SendMediaCallback
|
||||
synthesizeCallback SynthesizeCallback
|
||||
defaultChannel string
|
||||
defaultChatID string
|
||||
sentInRound bool
|
||||
}
|
||||
|
||||
func NewMessageTool() *MessageTool {
|
||||
|
|
@ -23,7 +35,9 @@ func (t *MessageTool) Name() string {
|
|||
}
|
||||
|
||||
func (t *MessageTool) Description() string {
|
||||
return "Send a message to user on a chat channel. Use this when you want to communicate something."
|
||||
return `Send a message or voice reply to the user.
|
||||
Set voice=true to reply with audio (uses TTS). Use voice when the user sent a voice message or explicitly asks for audio.
|
||||
Default is text. voice=true requires the TTS service to be available.`
|
||||
}
|
||||
|
||||
func (t *MessageTool) Parameters() map[string]interface{} {
|
||||
|
|
@ -32,15 +46,19 @@ func (t *MessageTool) Parameters() map[string]interface{} {
|
|||
"properties": map[string]interface{}{
|
||||
"content": map[string]interface{}{
|
||||
"type": "string",
|
||||
"description": "The message content to send",
|
||||
"description": "The message text to send (also used as TTS input when voice=true)",
|
||||
},
|
||||
"voice": map[string]interface{}{
|
||||
"type": "boolean",
|
||||
"description": "Set to true to send a voice/audio message via TTS instead of text",
|
||||
},
|
||||
"channel": map[string]interface{}{
|
||||
"type": "string",
|
||||
"description": "Optional: target channel (telegram, whatsapp, etc.)",
|
||||
"description": "Optional: target channel override",
|
||||
},
|
||||
"chat_id": map[string]interface{}{
|
||||
"type": "string",
|
||||
"description": "Optional: target chat/user ID",
|
||||
"description": "Optional: target chat ID override",
|
||||
},
|
||||
},
|
||||
"required": []string{"content"},
|
||||
|
|
@ -50,10 +68,9 @@ func (t *MessageTool) Parameters() map[string]interface{} {
|
|||
func (t *MessageTool) SetContext(channel, chatID string) {
|
||||
t.defaultChannel = channel
|
||||
t.defaultChatID = chatID
|
||||
t.sentInRound = false // Reset send tracking for new processing round
|
||||
t.sentInRound = false
|
||||
}
|
||||
|
||||
// HasSentInRound returns true if the message tool sent a message during the current round.
|
||||
func (t *MessageTool) HasSentInRound() bool {
|
||||
return t.sentInRound
|
||||
}
|
||||
|
|
@ -62,12 +79,21 @@ func (t *MessageTool) SetSendCallback(callback SendCallback) {
|
|||
t.sendCallback = callback
|
||||
}
|
||||
|
||||
func (t *MessageTool) SetSendMediaCallback(callback SendMediaCallback) {
|
||||
t.sendMediaCallback = callback
|
||||
}
|
||||
|
||||
func (t *MessageTool) SetSynthesizeCallback(callback SynthesizeCallback) {
|
||||
t.synthesizeCallback = callback
|
||||
}
|
||||
|
||||
func (t *MessageTool) Execute(ctx context.Context, args map[string]interface{}) *ToolResult {
|
||||
content, ok := args["content"].(string)
|
||||
if !ok {
|
||||
if !ok || content == "" {
|
||||
return &ToolResult{ForLLM: "content is required", IsError: true}
|
||||
}
|
||||
|
||||
voice, _ := args["voice"].(bool)
|
||||
channel, _ := args["channel"].(string)
|
||||
chatID, _ := args["chat_id"].(string)
|
||||
|
||||
|
|
@ -82,6 +108,37 @@ func (t *MessageTool) Execute(ctx context.Context, args map[string]interface{})
|
|||
return &ToolResult{ForLLM: "No target channel/chat specified", IsError: true}
|
||||
}
|
||||
|
||||
// Voice path
|
||||
if voice {
|
||||
if t.synthesizeCallback == nil || t.sendMediaCallback == nil {
|
||||
return &ToolResult{ForLLM: "TTS not available — sending as text instead", IsError: false}
|
||||
}
|
||||
|
||||
audioPath, err := t.synthesizeCallback(ctx, content)
|
||||
if err != nil {
|
||||
return &ToolResult{
|
||||
ForLLM: fmt.Sprintf("TTS synthesis failed: %v — falling back to text", err),
|
||||
IsError: false,
|
||||
}
|
||||
}
|
||||
defer os.Remove(audioPath)
|
||||
|
||||
if err := t.sendMediaCallback(ctx, channel, chatID, []string{audioPath}); err != nil {
|
||||
return &ToolResult{
|
||||
ForLLM: fmt.Sprintf("failed to send audio: %v", err),
|
||||
IsError: true,
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
|
||||
t.sentInRound = true
|
||||
return &ToolResult{
|
||||
ForLLM: fmt.Sprintf("Voice message sent to %s:%s", channel, chatID),
|
||||
Silent: true,
|
||||
}
|
||||
}
|
||||
|
||||
// Text path
|
||||
if t.sendCallback == nil {
|
||||
return &ToolResult{ForLLM: "Message sending not configured", IsError: true}
|
||||
}
|
||||
|
|
@ -95,7 +152,6 @@ func (t *MessageTool) Execute(ctx context.Context, args map[string]interface{})
|
|||
}
|
||||
|
||||
t.sentInRound = true
|
||||
// Silent: user already received the message directly
|
||||
return &ToolResult{
|
||||
ForLLM: fmt.Sprintf("Message sent to %s:%s", channel, chatID),
|
||||
Silent: true,
|
||||
|
|
|
|||
142
pkg/voice/kokoro.go
Normal file
142
pkg/voice/kokoro.go
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
package voice
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/sipeed/picoclaw/pkg/logger"
|
||||
)
|
||||
|
||||
// KokoroSynthesizer uses a Kokoro TTS server (OpenAI-compatible /v1/audio/speech API).
|
||||
type KokoroSynthesizer struct {
|
||||
apiBase string
|
||||
voice string
|
||||
model string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
type kokoroRequest struct {
|
||||
Model string `json:"model"`
|
||||
Input string `json:"input"`
|
||||
Voice string `json:"voice"`
|
||||
Format string `json:"response_format,omitempty"`
|
||||
}
|
||||
|
||||
// NewKokoroSynthesizer creates a Kokoro TTS client.
|
||||
// apiBase defaults to "http://localhost:8102".
|
||||
// voice defaults to "af_nova".
|
||||
func NewKokoroSynthesizer(apiBase, voice string) *KokoroSynthesizer {
|
||||
if apiBase == "" {
|
||||
apiBase = "http://localhost:8102"
|
||||
}
|
||||
if voice == "" {
|
||||
voice = "af_nova"
|
||||
}
|
||||
|
||||
logger.InfoCF("voice", "Creating Kokoro TTS synthesizer", map[string]interface{}{
|
||||
"api_base": apiBase,
|
||||
"voice": voice,
|
||||
})
|
||||
|
||||
return &KokoroSynthesizer{
|
||||
apiBase: apiBase,
|
||||
voice: voice,
|
||||
model: "kokoro",
|
||||
httpClient: &http.Client{
|
||||
Timeout: 60 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Synthesize converts text to audio, writes it to a temp file, and returns the path.
|
||||
// The caller must delete the file when done.
|
||||
func (s *KokoroSynthesizer) Synthesize(ctx context.Context, text string) (string, error) {
|
||||
logger.InfoCF("voice", "Synthesizing speech", map[string]interface{}{
|
||||
"text_length": len(text),
|
||||
"voice": s.voice,
|
||||
})
|
||||
|
||||
reqBody := kokoroRequest{
|
||||
Model: s.model,
|
||||
Input: text,
|
||||
Voice: s.voice,
|
||||
Format: "mp3",
|
||||
}
|
||||
|
||||
bodyBytes, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to marshal TTS request: %w", err)
|
||||
}
|
||||
|
||||
url := s.apiBase + "/v1/audio/speech"
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create TTS request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := s.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("TTS request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return "", fmt.Errorf("Kokoro TTS error (status %d): %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
// Write audio to temp file
|
||||
tmpFile, err := os.CreateTemp("", "picoclaw-tts-*.mp3")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create temp audio file: %w", err)
|
||||
}
|
||||
defer tmpFile.Close()
|
||||
|
||||
written, err := io.Copy(tmpFile, resp.Body)
|
||||
if err != nil {
|
||||
os.Remove(tmpFile.Name())
|
||||
return "", fmt.Errorf("failed to write TTS audio: %w", err)
|
||||
}
|
||||
|
||||
logger.InfoCF("voice", "Speech synthesized successfully", map[string]interface{}{
|
||||
"path": tmpFile.Name(),
|
||||
"size_bytes": written,
|
||||
"voice": s.voice,
|
||||
})
|
||||
|
||||
return tmpFile.Name(), nil
|
||||
}
|
||||
|
||||
// IsAvailable checks if the Kokoro TTS server is reachable.
|
||||
func (s *KokoroSynthesizer) IsAvailable() bool {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", s.apiBase+"/v1/models", nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
resp, err := s.httpClient.Do(req)
|
||||
if err != nil {
|
||||
logger.DebugCF("voice", "Kokoro TTS health check failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return false
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
available := resp.StatusCode == http.StatusOK
|
||||
logger.DebugCF("voice", "Kokoro TTS availability", map[string]interface{}{
|
||||
"available": available,
|
||||
"status_code": resp.StatusCode,
|
||||
})
|
||||
return available
|
||||
}
|
||||
10
pkg/voice/synthesizer.go
Normal file
10
pkg/voice/synthesizer.go
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
package voice
|
||||
|
||||
import "context"
|
||||
|
||||
// Synthesizer converts text to audio and returns the file path of the resulting audio file.
|
||||
// The caller is responsible for cleaning up the returned temp file.
|
||||
type Synthesizer interface {
|
||||
Synthesize(ctx context.Context, text string) (filePath string, err error)
|
||||
IsAvailable() bool
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue