diff --git a/cmd/picoclaw/main.go b/cmd/picoclaw/main.go index 36a6997bb..a00302cb0 100644 --- a/cmd/picoclaw/main.go +++ b/cmd/picoclaw/main.go @@ -634,6 +634,27 @@ func gatewayCmd() { } } + // Attach TTS synthesis callbacks to the message tool (enables voice=true). + if cfg.Tools.TTS.Enabled { + synthesizer := voice.NewKokoroSynthesizer(cfg.Tools.TTS.APIBase, cfg.Tools.TTS.Voice) + if synthesizer.IsAvailable() { + logger.InfoCF("voice", "TTS enabled — voice=true supported in message tool", map[string]interface{}{ + "api_base": cfg.Tools.TTS.APIBase, + "voice": cfg.Tools.TTS.Voice, + }) + agentLoop.SetVoiceCallbacks( + func(ctx context.Context, text string) (string, error) { + return synthesizer.Synthesize(ctx, text) + }, + func(ctx context.Context, channel, chatID string, filePaths []string) error { + return channelManager.SendFileToChannel(ctx, channel, chatID, filePaths) + }, + ) + } else { + logger.WarnC("voice", "TTS configured but service not reachable — voice=true disabled") + } + } + enabledChannels := channelManager.GetEnabledChannels() if len(enabledChannels) > 0 { fmt.Printf("✓ Channels enabled: %s\n", enabledChannels) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index cd4276155..bbb7a10e0 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -201,6 +201,17 @@ func (al *AgentLoop) RegisterTool(tool tools.Tool) { al.tools.Register(tool) } +// SetVoiceCallbacks attaches TTS synthesis and media-send callbacks to the +// message tool so it can handle voice=true calls. Safe to call after init. +func (al *AgentLoop) SetVoiceCallbacks(synth tools.SynthesizeCallback, sendMedia tools.SendMediaCallback) { + if tool, ok := al.tools.Get("message"); ok { + if mt, ok := tool.(*tools.MessageTool); ok { + mt.SetSynthesizeCallback(synth) + mt.SetSendMediaCallback(sendMedia) + } + } +} + func (al *AgentLoop) SetChannelManager(cm *channels.Manager) { al.channelManager = cm } diff --git a/pkg/bus/types.go b/pkg/bus/types.go index 44f9181a5..82c6b4754 100644 --- a/pkg/bus/types.go +++ b/pkg/bus/types.go @@ -11,9 +11,10 @@ type InboundMessage struct { } type OutboundMessage struct { - Channel string `json:"channel"` - ChatID string `json:"chat_id"` - Content string `json:"content"` + Channel string `json:"channel"` + ChatID string `json:"chat_id"` + Content string `json:"content"` + Media []string `json:"media,omitempty"` } type MessageHandler func(InboundMessage) error diff --git a/pkg/channels/manager.go b/pkg/channels/manager.go index 7f6abc4cb..ff523fcab 100644 --- a/pkg/channels/manager.go +++ b/pkg/channels/manager.go @@ -343,3 +343,23 @@ func (m *Manager) SendToChannel(ctx context.Context, channelName, chatID, conten return channel.Send(ctx, msg) } + +// SendFileToChannel sends one or more local media files to a channel synchronously. +// The caller is responsible for cleaning up the files after this returns. +func (m *Manager) SendFileToChannel(ctx context.Context, channelName, chatID string, filePaths []string) error { + m.mu.RLock() + channel, exists := m.channels[channelName] + m.mu.RUnlock() + + if !exists { + return fmt.Errorf("channel %s not found", channelName) + } + + msg := bus.OutboundMessage{ + Channel: channelName, + ChatID: chatID, + Media: filePaths, + } + + return channel.Send(ctx, msg) +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 77c529cf6..2755ed573 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -216,9 +216,16 @@ type WhisperConfig struct { APIBase string `json:"api_base" env:"PICOCLAW_TOOLS_WHISPER_API_BASE"` } +type TTSConfig struct { + Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_TTS_ENABLED"` + APIBase string `json:"api_base" env:"PICOCLAW_TOOLS_TTS_API_BASE"` + Voice string `json:"voice" env:"PICOCLAW_TOOLS_TTS_VOICE"` +} + type ToolsConfig struct { Web WebToolsConfig `json:"web"` Whisper WhisperConfig `json:"whisper"` + TTS TTSConfig `json:"tts"` } func DefaultConfig() *Config { @@ -332,6 +339,11 @@ func DefaultConfig() *Config { Enabled: false, APIBase: "http://localhost:8200", }, + TTS: TTSConfig{ + Enabled: false, + APIBase: "http://localhost:8100", + Voice: "en_us-lessac-medium", + }, }, Heartbeat: HeartbeatConfig{ Enabled: true, diff --git a/pkg/tools/message.go b/pkg/tools/message.go index abedb1316..c781d0a82 100644 --- a/pkg/tools/message.go +++ b/pkg/tools/message.go @@ -3,15 +3,27 @@ package tools import ( "context" "fmt" + "os" ) +// SendCallback sends a plain-text message to a channel/chat. type SendCallback func(channel, chatID, content string) error +// SendMediaCallback sends one or more local media files to a channel/chat. +// The callback owns the call; the caller is responsible for cleaning up files afterward. +type SendMediaCallback func(ctx context.Context, channel, chatID string, filePaths []string) error + +// SynthesizeCallback converts text to an audio file and returns the local path. +// The caller must delete the file when done. +type SynthesizeCallback func(ctx context.Context, text string) (filePath string, err error) + type MessageTool struct { - sendCallback SendCallback - defaultChannel string - defaultChatID string - sentInRound bool // Tracks whether a message was sent in the current processing round + sendCallback SendCallback + sendMediaCallback SendMediaCallback + synthesizeCallback SynthesizeCallback + defaultChannel string + defaultChatID string + sentInRound bool } func NewMessageTool() *MessageTool { @@ -23,7 +35,9 @@ func (t *MessageTool) Name() string { } func (t *MessageTool) Description() string { - return "Send a message to user on a chat channel. Use this when you want to communicate something." + return `Send a message or voice reply to the user. +Set voice=true to reply with audio (uses TTS). Use voice when the user sent a voice message or explicitly asks for audio. +Default is text. voice=true requires the TTS service to be available.` } func (t *MessageTool) Parameters() map[string]interface{} { @@ -32,15 +46,19 @@ func (t *MessageTool) Parameters() map[string]interface{} { "properties": map[string]interface{}{ "content": map[string]interface{}{ "type": "string", - "description": "The message content to send", + "description": "The message text to send (also used as TTS input when voice=true)", + }, + "voice": map[string]interface{}{ + "type": "boolean", + "description": "Set to true to send a voice/audio message via TTS instead of text", }, "channel": map[string]interface{}{ "type": "string", - "description": "Optional: target channel (telegram, whatsapp, etc.)", + "description": "Optional: target channel override", }, "chat_id": map[string]interface{}{ "type": "string", - "description": "Optional: target chat/user ID", + "description": "Optional: target chat ID override", }, }, "required": []string{"content"}, @@ -50,10 +68,9 @@ func (t *MessageTool) Parameters() map[string]interface{} { func (t *MessageTool) SetContext(channel, chatID string) { t.defaultChannel = channel t.defaultChatID = chatID - t.sentInRound = false // Reset send tracking for new processing round + t.sentInRound = false } -// HasSentInRound returns true if the message tool sent a message during the current round. func (t *MessageTool) HasSentInRound() bool { return t.sentInRound } @@ -62,12 +79,21 @@ func (t *MessageTool) SetSendCallback(callback SendCallback) { t.sendCallback = callback } +func (t *MessageTool) SetSendMediaCallback(callback SendMediaCallback) { + t.sendMediaCallback = callback +} + +func (t *MessageTool) SetSynthesizeCallback(callback SynthesizeCallback) { + t.synthesizeCallback = callback +} + func (t *MessageTool) Execute(ctx context.Context, args map[string]interface{}) *ToolResult { content, ok := args["content"].(string) - if !ok { + if !ok || content == "" { return &ToolResult{ForLLM: "content is required", IsError: true} } + voice, _ := args["voice"].(bool) channel, _ := args["channel"].(string) chatID, _ := args["chat_id"].(string) @@ -82,6 +108,37 @@ func (t *MessageTool) Execute(ctx context.Context, args map[string]interface{}) return &ToolResult{ForLLM: "No target channel/chat specified", IsError: true} } + // Voice path + if voice { + if t.synthesizeCallback == nil || t.sendMediaCallback == nil { + return &ToolResult{ForLLM: "TTS not available — sending as text instead", IsError: false} + } + + audioPath, err := t.synthesizeCallback(ctx, content) + if err != nil { + return &ToolResult{ + ForLLM: fmt.Sprintf("TTS synthesis failed: %v — falling back to text", err), + IsError: false, + } + } + defer os.Remove(audioPath) + + if err := t.sendMediaCallback(ctx, channel, chatID, []string{audioPath}); err != nil { + return &ToolResult{ + ForLLM: fmt.Sprintf("failed to send audio: %v", err), + IsError: true, + Err: err, + } + } + + t.sentInRound = true + return &ToolResult{ + ForLLM: fmt.Sprintf("Voice message sent to %s:%s", channel, chatID), + Silent: true, + } + } + + // Text path if t.sendCallback == nil { return &ToolResult{ForLLM: "Message sending not configured", IsError: true} } @@ -95,7 +152,6 @@ func (t *MessageTool) Execute(ctx context.Context, args map[string]interface{}) } t.sentInRound = true - // Silent: user already received the message directly return &ToolResult{ ForLLM: fmt.Sprintf("Message sent to %s:%s", channel, chatID), Silent: true, diff --git a/pkg/voice/kokoro.go b/pkg/voice/kokoro.go new file mode 100644 index 000000000..c1e79e364 --- /dev/null +++ b/pkg/voice/kokoro.go @@ -0,0 +1,142 @@ +package voice + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/sipeed/picoclaw/pkg/logger" +) + +// KokoroSynthesizer uses a Kokoro TTS server (OpenAI-compatible /v1/audio/speech API). +type KokoroSynthesizer struct { + apiBase string + voice string + model string + httpClient *http.Client +} + +type kokoroRequest struct { + Model string `json:"model"` + Input string `json:"input"` + Voice string `json:"voice"` + Format string `json:"response_format,omitempty"` +} + +// NewKokoroSynthesizer creates a Kokoro TTS client. +// apiBase defaults to "http://localhost:8102". +// voice defaults to "af_nova". +func NewKokoroSynthesizer(apiBase, voice string) *KokoroSynthesizer { + if apiBase == "" { + apiBase = "http://localhost:8102" + } + if voice == "" { + voice = "af_nova" + } + + logger.InfoCF("voice", "Creating Kokoro TTS synthesizer", map[string]interface{}{ + "api_base": apiBase, + "voice": voice, + }) + + return &KokoroSynthesizer{ + apiBase: apiBase, + voice: voice, + model: "kokoro", + httpClient: &http.Client{ + Timeout: 60 * time.Second, + }, + } +} + +// Synthesize converts text to audio, writes it to a temp file, and returns the path. +// The caller must delete the file when done. +func (s *KokoroSynthesizer) Synthesize(ctx context.Context, text string) (string, error) { + logger.InfoCF("voice", "Synthesizing speech", map[string]interface{}{ + "text_length": len(text), + "voice": s.voice, + }) + + reqBody := kokoroRequest{ + Model: s.model, + Input: text, + Voice: s.voice, + Format: "mp3", + } + + bodyBytes, err := json.Marshal(reqBody) + if err != nil { + return "", fmt.Errorf("failed to marshal TTS request: %w", err) + } + + url := s.apiBase + "/v1/audio/speech" + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes)) + if err != nil { + return "", fmt.Errorf("failed to create TTS request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := s.httpClient.Do(req) + if err != nil { + return "", fmt.Errorf("TTS request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("Kokoro TTS error (status %d): %s", resp.StatusCode, string(body)) + } + + // Write audio to temp file + tmpFile, err := os.CreateTemp("", "picoclaw-tts-*.mp3") + if err != nil { + return "", fmt.Errorf("failed to create temp audio file: %w", err) + } + defer tmpFile.Close() + + written, err := io.Copy(tmpFile, resp.Body) + if err != nil { + os.Remove(tmpFile.Name()) + return "", fmt.Errorf("failed to write TTS audio: %w", err) + } + + logger.InfoCF("voice", "Speech synthesized successfully", map[string]interface{}{ + "path": tmpFile.Name(), + "size_bytes": written, + "voice": s.voice, + }) + + return tmpFile.Name(), nil +} + +// IsAvailable checks if the Kokoro TTS server is reachable. +func (s *KokoroSynthesizer) IsAvailable() bool { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "GET", s.apiBase+"/v1/models", nil) + if err != nil { + return false + } + + resp, err := s.httpClient.Do(req) + if err != nil { + logger.DebugCF("voice", "Kokoro TTS health check failed", map[string]interface{}{ + "error": err.Error(), + }) + return false + } + defer resp.Body.Close() + + available := resp.StatusCode == http.StatusOK + logger.DebugCF("voice", "Kokoro TTS availability", map[string]interface{}{ + "available": available, + "status_code": resp.StatusCode, + }) + return available +} diff --git a/pkg/voice/synthesizer.go b/pkg/voice/synthesizer.go new file mode 100644 index 000000000..5e2efeed6 --- /dev/null +++ b/pkg/voice/synthesizer.go @@ -0,0 +1,10 @@ +package voice + +import "context" + +// Synthesizer converts text to audio and returns the file path of the resulting audio file. +// The caller is responsible for cleaning up the returned temp file. +type Synthesizer interface { + Synthesize(ctx context.Context, text string) (filePath string, err error) + IsAvailable() bool +}