From cf315091df1e864037c687ff6c276e0a05ecfd8b Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 07:53:28 +0100 Subject: [PATCH] add tts support --- pkg/channels/discord/discord.go | 23 +++++++ pkg/channels/discord/init.go | 7 ++- pkg/channels/discord/voice.go | 55 ++++++++++++++++ pkg/voice/tts.go | 108 ++++++++++++++++++++++++++++++++ 4 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 pkg/voice/tts.go diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index 26332455a..06a0f175f 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -21,6 +21,7 @@ import ( "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/media" "github.com/sipeed/picoclaw/pkg/utils" + "github.com/sipeed/picoclaw/pkg/voice" ) const ( @@ -43,6 +44,7 @@ type DiscordChannel struct { typingStop map[string]chan struct{} // chatID → stop signal botUserID string // stored for mention checking bus *bus.MessageBus + tts voice.TTSProvider } func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordChannel, error) { @@ -146,6 +148,14 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) erro return nil } + if c.tts != nil { + if ch, err := c.session.State.Channel(channelID); err == nil && ch.GuildID != "" { + if vc, ok := c.session.VoiceConnections[ch.GuildID]; ok && vc != nil { + go c.playTTS(context.Background(), vc, msg.Content) + } + } + } + return c.sendChunk(ctx, channelID, msg.Content, msg.ReplyToMessageID) } @@ -640,3 +650,16 @@ func (c *DiscordChannel) listenVoiceControl(ctx context.Context) { } } } + +func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnection, text string) { + stream, err := c.tts.Synthesize(ctx, text) + if err != nil { + logger.ErrorCF("discord", "TTS synthesize failed", map[string]any{"error": err.Error()}) + return + } + defer stream.Close() + + if err := streamOggOpusToDiscord(vc, stream); err != nil { + logger.ErrorCF("discord", "TTS playback failed", map[string]any{"error": err.Error()}) + } +} diff --git a/pkg/channels/discord/init.go b/pkg/channels/discord/init.go index 15a539804..13e4fbc91 100644 --- a/pkg/channels/discord/init.go +++ b/pkg/channels/discord/init.go @@ -4,10 +4,15 @@ import ( "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/voice" ) func init() { channels.RegisterFactory("discord", func(cfg *config.Config, b *bus.MessageBus) (channels.Channel, error) { - return NewDiscordChannel(cfg.Channels.Discord, b) + ch, err := NewDiscordChannel(cfg.Channels.Discord, b) + if err == nil { + ch.tts = voice.DetectTTS(cfg) + } + return ch, err }) } diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index 23386e642..72f767b85 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -1,7 +1,9 @@ package discord import ( + "bytes" "fmt" + "io" "time" "github.com/bwmarrin/discordgo" @@ -40,6 +42,59 @@ func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.M return false } +func VoiceReceiveActive(vc *discordgo.VoiceConnection) bool { + return vc != nil && vc.OpusRecv != nil +} + +func streamOggOpusToDiscord(vc *discordgo.VoiceConnection, r io.Reader) error { + // Wait for the speaking transition to register + vc.Speaking(true) + defer vc.Speaking(false) + + var packet []byte + header := make([]byte, 27) + + for { + if _, err := io.ReadFull(r, header); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + return nil + } + return fmt.Errorf("failed to read ogg header: %w", err) + } + if string(header[:4]) != "OggS" { + return fmt.Errorf("invalid ogg magic string") + } + + pageSegments := int(header[26]) + segmentTable := make([]byte, pageSegments) + if _, err := io.ReadFull(r, segmentTable); err != nil { + return fmt.Errorf("failed to read segment table: %w", err) + } + + for _, lacing := range segmentTable { + segment := make([]byte, lacing) + if _, err := io.ReadFull(r, segment); err != nil { + return fmt.Errorf("failed to read segment data: %w", err) + } + + packet = append(packet, segment...) + + // If lacing is less than 255, the packet is complete + if lacing < 255 { + if len(packet) > 0 { + // Ignore Ogg Opus headers + if !bytes.HasPrefix(packet, []byte("OpusHead")) && !bytes.HasPrefix(packet, []byte("OpusTags")) { + // Pacing is handled natively by vc.OpusSend blocking (it has an internal ticker) + vc.OpusSend <- packet + } + // Start new packet + packet = nil + } + } + } + } +} + func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID string, chatID string) { logger.InfoCF("discord", "Started listening for voice", map[string]any{"guild": guildID}) diff --git a/pkg/voice/tts.go b/pkg/voice/tts.go new file mode 100644 index 000000000..8de0bbc9c --- /dev/null +++ b/pkg/voice/tts.go @@ -0,0 +1,108 @@ +package voice + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/logger" +) + +type TTSProvider interface { + Name() string + Synthesize(ctx context.Context, text string) (io.ReadCloser, error) +} + +type OpenAITTSProvider struct { + apiKey string + apiBase string + voice string + model string + httpClient *http.Client +} + +func NewOpenAITTSProvider(apiKey string, apiBase string, proxyURL string) *OpenAITTSProvider { + if apiBase == "" || apiBase == "https://api.openai.com/v1" { + apiBase = "https://api.openai.com/v1/audio/speech" + } else if !strings.HasSuffix(apiBase, "/audio/speech") { + // Just in case they provide openrouter base or standard base + apiBase = strings.TrimSuffix(apiBase, "/") + "/audio/speech" + } + + client := &http.Client{ + Timeout: 60 * time.Second, + } + + if proxyURL != "" { + if pURL, err := url.Parse(proxyURL); err == nil { + client.Transport = &http.Transport{ + Proxy: http.ProxyURL(pURL), + } + } + } + + return &OpenAITTSProvider{ + apiKey: apiKey, + apiBase: apiBase, + voice: "alloy", + model: "tts-1", + httpClient: client, + } +} + +func (t *OpenAITTSProvider) Name() string { + return "openai-tts" +} + +func (t *OpenAITTSProvider) Synthesize(ctx context.Context, text string) (io.ReadCloser, error) { + logger.InfoCF("voice-tts", "Starting TTS synthesis", map[string]any{"text_len": len(text)}) + + reqBody := map[string]any{ + "model": t.model, + "input": text, + "voice": t.voice, + "response_format": "opus", + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", t.apiBase, bytes.NewReader(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+t.apiKey) + + resp, err := t.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + + if resp.StatusCode != http.StatusOK { + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body)) + } + + return resp.Body, nil +} + +func DetectTTS(cfg *config.Config) TTSProvider { + for _, mc := range cfg.ModelList { + if strings.Contains(strings.ToLower(mc.ModelName), "tts") && mc.APIKey != "" { + return NewOpenAITTSProvider(mc.APIKey, mc.APIBase, mc.Proxy) + } + } + return nil +}