feat(voice/tts): add TTS synthesis and voice parameter on message tool

Adds outbound voice capability: the agent can now reply with audio by
setting voice=true on the message tool, useful when the user sends a voice
message or explicitly requests audio.

Changes:
- pkg/voice/synthesizer.go: Synthesizer interface (Synthesize, IsAvailable)
- pkg/voice/kokoro.go: KokoroSynthesizer — talks to any OpenAI-compatible
  /v1/audio/speech endpoint (Kokoro, Piper, etc.). Health-check via GET
  /v1/models. Returns a temp .mp3 path; caller cleans up.
- pkg/bus/types.go: add Media []string to OutboundMessage (backward-
  compatible, omitempty). Enables any channel to receive file paths.
- pkg/channels/manager.go: add SendFileToChannel() — synchronous media
  send that routes local file paths through the channel's Send().
- pkg/tools/message.go: add voice=true parameter + SynthesizeCallback +
  SendMediaCallback. Voice path: synthesize → send file → cleanup.
  Falls back to text if TTS unavailable. HasSentInRound fires for both.
- pkg/agent/loop.go: add SetVoiceCallbacks() to attach TTS to message tool
  after channel manager is available.
- cmd/picoclaw/main.go: wire Kokoro TTS after channels init; attaches to
  message tool via SetVoiceCallbacks().

Config example:
  "tools": {
    "tts": {
      "enabled": true,
      "api_base": "http://localhost:8100",
      "voice": "en_us-lessac-medium"
    }
  }

Depends-on: feat(voice/stt): add local Whisper STT provider
This commit is contained in:
Myka 2026-02-17 11:11:51 +03:00
parent 21079e49ec
commit c3a7629343
8 changed files with 288 additions and 15 deletions

View file

@ -634,6 +634,27 @@ func gatewayCmd() {
}
}
// Attach TTS synthesis callbacks to the message tool (enables voice=true).
if cfg.Tools.TTS.Enabled {
synthesizer := voice.NewKokoroSynthesizer(cfg.Tools.TTS.APIBase, cfg.Tools.TTS.Voice)
if synthesizer.IsAvailable() {
logger.InfoCF("voice", "TTS enabled — voice=true supported in message tool", map[string]interface{}{
"api_base": cfg.Tools.TTS.APIBase,
"voice": cfg.Tools.TTS.Voice,
})
agentLoop.SetVoiceCallbacks(
func(ctx context.Context, text string) (string, error) {
return synthesizer.Synthesize(ctx, text)
},
func(ctx context.Context, channel, chatID string, filePaths []string) error {
return channelManager.SendFileToChannel(ctx, channel, chatID, filePaths)
},
)
} else {
logger.WarnC("voice", "TTS configured but service not reachable — voice=true disabled")
}
}
enabledChannels := channelManager.GetEnabledChannels()
if len(enabledChannels) > 0 {
fmt.Printf("✓ Channels enabled: %s\n", enabledChannels)

View file

@ -201,6 +201,17 @@ func (al *AgentLoop) RegisterTool(tool tools.Tool) {
al.tools.Register(tool)
}
// SetVoiceCallbacks attaches TTS synthesis and media-send callbacks to the
// message tool so it can handle voice=true calls. Safe to call after init.
func (al *AgentLoop) SetVoiceCallbacks(synth tools.SynthesizeCallback, sendMedia tools.SendMediaCallback) {
if tool, ok := al.tools.Get("message"); ok {
if mt, ok := tool.(*tools.MessageTool); ok {
mt.SetSynthesizeCallback(synth)
mt.SetSendMediaCallback(sendMedia)
}
}
}
func (al *AgentLoop) SetChannelManager(cm *channels.Manager) {
al.channelManager = cm
}

View file

@ -14,6 +14,7 @@ type OutboundMessage struct {
Channel string `json:"channel"`
ChatID string `json:"chat_id"`
Content string `json:"content"`
Media []string `json:"media,omitempty"`
}
type MessageHandler func(InboundMessage) error

View file

@ -343,3 +343,23 @@ func (m *Manager) SendToChannel(ctx context.Context, channelName, chatID, conten
return channel.Send(ctx, msg)
}
// SendFileToChannel sends one or more local media files to a channel synchronously.
// The caller is responsible for cleaning up the files after this returns.
func (m *Manager) SendFileToChannel(ctx context.Context, channelName, chatID string, filePaths []string) error {
m.mu.RLock()
channel, exists := m.channels[channelName]
m.mu.RUnlock()
if !exists {
return fmt.Errorf("channel %s not found", channelName)
}
msg := bus.OutboundMessage{
Channel: channelName,
ChatID: chatID,
Media: filePaths,
}
return channel.Send(ctx, msg)
}

View file

@ -216,9 +216,16 @@ type WhisperConfig struct {
APIBase string `json:"api_base" env:"PICOCLAW_TOOLS_WHISPER_API_BASE"`
}
type TTSConfig struct {
Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_TTS_ENABLED"`
APIBase string `json:"api_base" env:"PICOCLAW_TOOLS_TTS_API_BASE"`
Voice string `json:"voice" env:"PICOCLAW_TOOLS_TTS_VOICE"`
}
type ToolsConfig struct {
Web WebToolsConfig `json:"web"`
Whisper WhisperConfig `json:"whisper"`
TTS TTSConfig `json:"tts"`
}
func DefaultConfig() *Config {
@ -332,6 +339,11 @@ func DefaultConfig() *Config {
Enabled: false,
APIBase: "http://localhost:8200",
},
TTS: TTSConfig{
Enabled: false,
APIBase: "http://localhost:8100",
Voice: "en_us-lessac-medium",
},
},
Heartbeat: HeartbeatConfig{
Enabled: true,

View file

@ -3,15 +3,27 @@ package tools
import (
"context"
"fmt"
"os"
)
// SendCallback sends a plain-text message to a channel/chat.
type SendCallback func(channel, chatID, content string) error
// SendMediaCallback sends one or more local media files to a channel/chat.
// The callback owns the call; the caller is responsible for cleaning up files afterward.
type SendMediaCallback func(ctx context.Context, channel, chatID string, filePaths []string) error
// SynthesizeCallback converts text to an audio file and returns the local path.
// The caller must delete the file when done.
type SynthesizeCallback func(ctx context.Context, text string) (filePath string, err error)
type MessageTool struct {
sendCallback SendCallback
sendMediaCallback SendMediaCallback
synthesizeCallback SynthesizeCallback
defaultChannel string
defaultChatID string
sentInRound bool // Tracks whether a message was sent in the current processing round
sentInRound bool
}
func NewMessageTool() *MessageTool {
@ -23,7 +35,9 @@ func (t *MessageTool) Name() string {
}
func (t *MessageTool) Description() string {
return "Send a message to user on a chat channel. Use this when you want to communicate something."
return `Send a message or voice reply to the user.
Set voice=true to reply with audio (uses TTS). Use voice when the user sent a voice message or explicitly asks for audio.
Default is text. voice=true requires the TTS service to be available.`
}
func (t *MessageTool) Parameters() map[string]interface{} {
@ -32,15 +46,19 @@ func (t *MessageTool) Parameters() map[string]interface{} {
"properties": map[string]interface{}{
"content": map[string]interface{}{
"type": "string",
"description": "The message content to send",
"description": "The message text to send (also used as TTS input when voice=true)",
},
"voice": map[string]interface{}{
"type": "boolean",
"description": "Set to true to send a voice/audio message via TTS instead of text",
},
"channel": map[string]interface{}{
"type": "string",
"description": "Optional: target channel (telegram, whatsapp, etc.)",
"description": "Optional: target channel override",
},
"chat_id": map[string]interface{}{
"type": "string",
"description": "Optional: target chat/user ID",
"description": "Optional: target chat ID override",
},
},
"required": []string{"content"},
@ -50,10 +68,9 @@ func (t *MessageTool) Parameters() map[string]interface{} {
func (t *MessageTool) SetContext(channel, chatID string) {
t.defaultChannel = channel
t.defaultChatID = chatID
t.sentInRound = false // Reset send tracking for new processing round
t.sentInRound = false
}
// HasSentInRound returns true if the message tool sent a message during the current round.
func (t *MessageTool) HasSentInRound() bool {
return t.sentInRound
}
@ -62,12 +79,21 @@ func (t *MessageTool) SetSendCallback(callback SendCallback) {
t.sendCallback = callback
}
func (t *MessageTool) SetSendMediaCallback(callback SendMediaCallback) {
t.sendMediaCallback = callback
}
func (t *MessageTool) SetSynthesizeCallback(callback SynthesizeCallback) {
t.synthesizeCallback = callback
}
func (t *MessageTool) Execute(ctx context.Context, args map[string]interface{}) *ToolResult {
content, ok := args["content"].(string)
if !ok {
if !ok || content == "" {
return &ToolResult{ForLLM: "content is required", IsError: true}
}
voice, _ := args["voice"].(bool)
channel, _ := args["channel"].(string)
chatID, _ := args["chat_id"].(string)
@ -82,6 +108,37 @@ func (t *MessageTool) Execute(ctx context.Context, args map[string]interface{})
return &ToolResult{ForLLM: "No target channel/chat specified", IsError: true}
}
// Voice path
if voice {
if t.synthesizeCallback == nil || t.sendMediaCallback == nil {
return &ToolResult{ForLLM: "TTS not available — sending as text instead", IsError: false}
}
audioPath, err := t.synthesizeCallback(ctx, content)
if err != nil {
return &ToolResult{
ForLLM: fmt.Sprintf("TTS synthesis failed: %v — falling back to text", err),
IsError: false,
}
}
defer os.Remove(audioPath)
if err := t.sendMediaCallback(ctx, channel, chatID, []string{audioPath}); err != nil {
return &ToolResult{
ForLLM: fmt.Sprintf("failed to send audio: %v", err),
IsError: true,
Err: err,
}
}
t.sentInRound = true
return &ToolResult{
ForLLM: fmt.Sprintf("Voice message sent to %s:%s", channel, chatID),
Silent: true,
}
}
// Text path
if t.sendCallback == nil {
return &ToolResult{ForLLM: "Message sending not configured", IsError: true}
}
@ -95,7 +152,6 @@ func (t *MessageTool) Execute(ctx context.Context, args map[string]interface{})
}
t.sentInRound = true
// Silent: user already received the message directly
return &ToolResult{
ForLLM: fmt.Sprintf("Message sent to %s:%s", channel, chatID),
Silent: true,

142
pkg/voice/kokoro.go Normal file
View file

@ -0,0 +1,142 @@
package voice
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"time"
"github.com/sipeed/picoclaw/pkg/logger"
)
// KokoroSynthesizer uses a Kokoro TTS server (OpenAI-compatible /v1/audio/speech API).
type KokoroSynthesizer struct {
apiBase string
voice string
model string
httpClient *http.Client
}
type kokoroRequest struct {
Model string `json:"model"`
Input string `json:"input"`
Voice string `json:"voice"`
Format string `json:"response_format,omitempty"`
}
// NewKokoroSynthesizer creates a Kokoro TTS client.
// apiBase defaults to "http://localhost:8102".
// voice defaults to "af_nova".
func NewKokoroSynthesizer(apiBase, voice string) *KokoroSynthesizer {
if apiBase == "" {
apiBase = "http://localhost:8102"
}
if voice == "" {
voice = "af_nova"
}
logger.InfoCF("voice", "Creating Kokoro TTS synthesizer", map[string]interface{}{
"api_base": apiBase,
"voice": voice,
})
return &KokoroSynthesizer{
apiBase: apiBase,
voice: voice,
model: "kokoro",
httpClient: &http.Client{
Timeout: 60 * time.Second,
},
}
}
// Synthesize converts text to audio, writes it to a temp file, and returns the path.
// The caller must delete the file when done.
func (s *KokoroSynthesizer) Synthesize(ctx context.Context, text string) (string, error) {
logger.InfoCF("voice", "Synthesizing speech", map[string]interface{}{
"text_length": len(text),
"voice": s.voice,
})
reqBody := kokoroRequest{
Model: s.model,
Input: text,
Voice: s.voice,
Format: "mp3",
}
bodyBytes, err := json.Marshal(reqBody)
if err != nil {
return "", fmt.Errorf("failed to marshal TTS request: %w", err)
}
url := s.apiBase + "/v1/audio/speech"
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes))
if err != nil {
return "", fmt.Errorf("failed to create TTS request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
resp, err := s.httpClient.Do(req)
if err != nil {
return "", fmt.Errorf("TTS request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return "", fmt.Errorf("Kokoro TTS error (status %d): %s", resp.StatusCode, string(body))
}
// Write audio to temp file
tmpFile, err := os.CreateTemp("", "picoclaw-tts-*.mp3")
if err != nil {
return "", fmt.Errorf("failed to create temp audio file: %w", err)
}
defer tmpFile.Close()
written, err := io.Copy(tmpFile, resp.Body)
if err != nil {
os.Remove(tmpFile.Name())
return "", fmt.Errorf("failed to write TTS audio: %w", err)
}
logger.InfoCF("voice", "Speech synthesized successfully", map[string]interface{}{
"path": tmpFile.Name(),
"size_bytes": written,
"voice": s.voice,
})
return tmpFile.Name(), nil
}
// IsAvailable checks if the Kokoro TTS server is reachable.
func (s *KokoroSynthesizer) IsAvailable() bool {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, "GET", s.apiBase+"/v1/models", nil)
if err != nil {
return false
}
resp, err := s.httpClient.Do(req)
if err != nil {
logger.DebugCF("voice", "Kokoro TTS health check failed", map[string]interface{}{
"error": err.Error(),
})
return false
}
defer resp.Body.Close()
available := resp.StatusCode == http.StatusOK
logger.DebugCF("voice", "Kokoro TTS availability", map[string]interface{}{
"available": available,
"status_code": resp.StatusCode,
})
return available
}

10
pkg/voice/synthesizer.go Normal file
View file

@ -0,0 +1,10 @@
package voice
import "context"
// Synthesizer converts text to audio and returns the file path of the resulting audio file.
// The caller is responsible for cleaning up the returned temp file.
type Synthesizer interface {
Synthesize(ctx context.Context, text string) (filePath string, err error)
IsAvailable() bool
}