From 48c16f77eb5e5fcb40e262c2b96b4f59c87f4d07 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 05:52:21 +0100 Subject: [PATCH 01/51] init commit --- go.mod | 5 +- go.sum | 6 + pkg/bus/bus.go | 28 +++++ pkg/bus/types.go | 20 ++++ pkg/channels/discord/discord.go | 6 + pkg/channels/discord/voice.go | 86 ++++++++++++++ pkg/gateway/gateway.go | 16 +++ pkg/voice/agent.go | 195 ++++++++++++++++++++++++++++++++ pkg/voice/transcriber.go | 70 +++++++----- 9 files changed, 402 insertions(+), 30 deletions(-) create mode 100644 pkg/channels/discord/voice.go create mode 100644 pkg/voice/agent.go diff --git a/go.mod b/go.mod index cfc930d37..3fcc360a6 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/sipeed/picoclaw go 1.25.8 require ( - github.com/BurntSushi/toml v1.6.0 fyne.io/systray v1.12.0 + github.com/BurntSushi/toml v1.6.0 github.com/adhocore/gronx v1.19.6 github.com/anthropics/anthropic-sdk-go v1.26.0 github.com/bwmarrin/discordgo v0.29.0 @@ -22,6 +22,8 @@ require ( github.com/mymmrac/telego v1.7.0 github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1 github.com/openai/openai-go/v3 v3.22.0 + github.com/pion/rtp v1.8.7 + github.com/pion/webrtc/v3 v3.3.6 github.com/rivo/tview v0.42.0 github.com/rs/zerolog v1.34.0 github.com/slack-go/slack v0.17.3 @@ -53,6 +55,7 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 // indirect + github.com/pion/randutil v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect diff --git a/go.sum b/go.sum index f24b997d4..f95643290 100644 --- a/go.sum +++ b/go.sum @@ -162,6 +162,12 @@ github.com/openai/openai-go/v3 v3.22.0 h1:6MEoNoV8sbjOVmXdvhmuX3BjVbVdcExbVyGixi github.com/openai/openai-go/v3 v3.22.0/go.mod h1:cdufnVK14cWcT9qA1rRtrXx4FTRsgbDPW7Ia7SS5cZo= github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 h1:rh2lKw/P/EqHa724vYH2+VVQ1YnW4u6EOXl0PMAovZE= github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= +github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= +github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= +github.com/pion/rtp v1.8.7 h1:qslKkG8qxvQ7hqaxkmL7Pl0XcUm+/Er7nMnu6Vq+ZxM= +github.com/pion/rtp v1.8.7/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/webrtc/v3 v3.3.6 h1:7XAh4RPtlY1Vul6/GmZrv7z+NnxKA6If0KStXBI2ZLE= +github.com/pion/webrtc/v3 v3.3.6/go.mod h1:zyN7th4mZpV27eXybfR/cnUf3J2DRy8zw/mdjD9JTNM= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/pkg/bus/bus.go b/pkg/bus/bus.go index 37fcb74c5..a9c74ef90 100644 --- a/pkg/bus/bus.go +++ b/pkg/bus/bus.go @@ -34,6 +34,8 @@ type MessageBus struct { inbound chan InboundMessage outbound chan OutboundMessage outboundMedia chan OutboundMediaMessage + audioChunks chan AudioChunk + voiceControls chan VoiceControl closeOnce sync.Once done chan struct{} @@ -47,6 +49,8 @@ func NewMessageBus() *MessageBus { inbound: make(chan InboundMessage, defaultBusBufferSize), outbound: make(chan OutboundMessage, defaultBusBufferSize), outboundMedia: make(chan OutboundMediaMessage, defaultBusBufferSize), + audioChunks: make(chan AudioChunk, defaultBusBufferSize*4), // Audio chunks need more buffer + voiceControls: make(chan VoiceControl, defaultBusBufferSize), done: make(chan struct{}), } } @@ -103,6 +107,22 @@ func (mb *MessageBus) OutboundMediaChan() <-chan OutboundMediaMessage { return mb.outboundMedia } +func (mb *MessageBus) PublishAudioChunk(ctx context.Context, chunk AudioChunk) error { + return publish(ctx, mb, mb.audioChunks, chunk) +} + +func (mb *MessageBus) AudioChunksChan() <-chan AudioChunk { + return mb.audioChunks +} + +func (mb *MessageBus) PublishVoiceControl(ctx context.Context, ctrl VoiceControl) error { + return publish(ctx, mb, mb.voiceControls, ctrl) +} + +func (mb *MessageBus) VoiceControlsChan() <-chan VoiceControl { + return mb.voiceControls +} + // SetStreamDelegate registers a StreamDelegate (typically the channel Manager). func (mb *MessageBus) SetStreamDelegate(d StreamDelegate) { mb.streamDelegate.Store(d) @@ -132,6 +152,8 @@ func (mb *MessageBus) Close() { close(mb.inbound) close(mb.outbound) close(mb.outboundMedia) + close(mb.audioChunks) + close(mb.voiceControls) // clean up any remaining messages in channels drained := 0 @@ -144,6 +166,12 @@ func (mb *MessageBus) Close() { for range mb.outboundMedia { drained++ } + for range mb.audioChunks { + drained++ + } + for range mb.voiceControls { + drained++ + } if drained > 0 { logger.DebugCF("bus", "Drained buffered messages during close", map[string]any{ diff --git a/pkg/bus/types.go b/pkg/bus/types.go index 12da3f1dd..9c637f3e7 100644 --- a/pkg/bus/types.go +++ b/pkg/bus/types.go @@ -51,3 +51,23 @@ type OutboundMediaMessage struct { ChatID string `json:"chat_id"` Parts []MediaPart `json:"parts"` } + +// AudioChunk represents a chunk of streaming voice data. +type AudioChunk struct { + SessionID string `json:"session_id"` + SpeakerID string `json:"speaker_id"` // User ID or SSRC + ChatID string `json:"chat_id"` // Where to respond + Sequence uint64 `json:"sequence"` + Timestamp uint32 `json:"timestamp"` + SampleRate int `json:"sample_rate"` + Channels int `json:"channels"` + Format string `json:"format"` // "opus", "pcm", etc + Data []byte `json:"data"` +} + +// VoiceControl represents state or commands for voice sessions. +type VoiceControl struct { + SessionID string `json:"session_id"` + Type string `json:"type"` // "state", "command" + Action string `json:"action"` // "idle", "listening", "start", "stop" +} diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index 83a04907c..6e5319080 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -42,6 +42,7 @@ type DiscordChannel struct { typingMu sync.Mutex typingStop map[string]chan struct{} // chatID → stop signal botUserID string // stored for mention checking + bus *bus.MessageBus } func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordChannel, error) { @@ -73,6 +74,7 @@ func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordC config: cfg, ctx: context.Background(), typingStop: make(map[string]chan struct{}), + bus: bus, }, nil } @@ -321,6 +323,10 @@ func (c *DiscordChannel) handleMessage(s *discordgo.Session, m *discordgo.Messag return } + if c.handleVoiceCommand(s, m) { + return + } + // Check allowlist first to avoid downloading attachments for rejected users sender := bus.SenderInfo{ Platform: "discord", diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go new file mode 100644 index 000000000..e2aae52ed --- /dev/null +++ b/pkg/channels/discord/voice.go @@ -0,0 +1,86 @@ +package discord + +import ( + "fmt" + + "github.com/bwmarrin/discordgo" + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/logger" +) + +func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.MessageCreate) bool { + if m.Content == "!vc join" { + vs, err := s.State.VoiceState(m.GuildID, m.Author.ID) + if err != nil || vs == nil { + s.ChannelMessageSend(m.ChannelID, "You need to be in a voice channel first!") + return true + } + + logger.InfoCF("discord", "Joining voice channel", map[string]any{"channel": vs.ChannelID}) + vc, err := s.ChannelVoiceJoin(m.GuildID, vs.ChannelID, false, false) + if err != nil { + s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("Failed to join voice channel: %v", err)) + return true + } + + go c.receiveVoice(vc, m.GuildID, m.ChannelID) + s.ChannelMessageSend(m.ChannelID, "Joined Voice Channel! Listening for audio...") + return true + } else if m.Content == "!vc leave" { + vc, exists := s.VoiceConnections[m.GuildID] + if exists && vc != nil { + vc.Disconnect() + s.ChannelMessageSend(m.ChannelID, "Left Voice Channel.") + } else { + s.ChannelMessageSend(m.ChannelID, "Not in a voice channel.") + } + return true + } + return false +} + +func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID string, chatID string) { + logger.InfoCF("discord", "Started listening for voice", map[string]any{"guild": guildID}) + + sessionID := fmt.Sprintf("discord_vc_%s", guildID) + + c.bus.PublishVoiceControl(c.ctx, bus.VoiceControl{ + SessionID: sessionID, + Type: "state", + Action: "listening", + }) + + var sequence uint64 = 0 + + for { + select { + case <-c.ctx.Done(): + return + case p, ok := <-vc.OpusRecv: + if !ok { + logger.InfoCF("discord", "Voice channel closed", map[string]any{"guild": guildID}) + return + } + + if p == nil { + continue + } + + sequence++ + + chunk := bus.AudioChunk{ + SessionID: sessionID, + SpeakerID: fmt.Sprintf("%d", p.SSRC), + ChatID: chatID, + Sequence: sequence, + Timestamp: p.Timestamp, + SampleRate: 48000, + Channels: 2, + Format: "opus", + Data: p.Opus, + } + + c.bus.PublishAudioChunk(c.ctx, chunk) + } + } +} diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go index 9a2706b3b..3a0478d1d 100644 --- a/pkg/gateway/gateway.go +++ b/pkg/gateway/gateway.go @@ -55,6 +55,7 @@ type services struct { ChannelManager *channels.Manager DeviceService *devices.Service HealthServer *health.Server + VoiceAgentCancel context.CancelFunc manualReloadChan chan struct{} reloading atomic.Bool } @@ -286,6 +287,12 @@ func setupAndStartServices( if transcriber := voice.DetectTranscriber(cfg); transcriber != nil { agentLoop.SetTranscriber(transcriber) logger.InfoCF("voice", "Transcription enabled (agent-level)", map[string]any{"provider": transcriber.Name()}) + + // Start Voice Agent Orchestrator + vaCtx, vaCancel := context.WithCancel(context.Background()) + runningServices.VoiceAgentCancel = vaCancel + voiceAgent := voice.NewAgent(msgBus, transcriber) + voiceAgent.Start(vaCtx) } enabledChannels := runningServices.ChannelManager.GetEnabledChannels() @@ -332,6 +339,9 @@ func stopAndCleanupServices(runningServices *services, shutdownTimeout time.Dura if !isReload && runningServices.ChannelManager != nil { runningServices.ChannelManager.StopAll(shutdownCtx) } + if runningServices.VoiceAgentCancel != nil { + runningServices.VoiceAgentCancel() + } if runningServices.DeviceService != nil { runningServices.DeviceService.Stop() } @@ -516,6 +526,12 @@ func restartServices( al.SetTranscriber(transcriber) if transcriber != nil { logger.InfoCF("voice", "Transcription re-enabled (agent-level)", map[string]any{"provider": transcriber.Name()}) + + // Start Voice Agent Orchestrator on reload + vaCtx, vaCancel := context.WithCancel(context.Background()) + runningServices.VoiceAgentCancel = vaCancel + voiceAgent := voice.NewAgent(msgBus, transcriber) + voiceAgent.Start(vaCtx) } else { logger.InfoCF("voice", "Transcription disabled", nil) } diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go new file mode 100644 index 000000000..4362eb863 --- /dev/null +++ b/pkg/voice/agent.go @@ -0,0 +1,195 @@ +package voice + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/pion/rtp" + "github.com/pion/webrtc/v3/pkg/media/oggwriter" + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/logger" +) + +type speechAccumulator struct { + writer *oggwriter.OggWriter + file string + lastAudioAt time.Time + mu sync.Mutex + closed bool + chatID string + speakerID string +} + +func (a *speechAccumulator) Push(chunk bus.AudioChunk) { + a.mu.Lock() + defer a.mu.Unlock() + + if a.closed { + return + } + + a.lastAudioAt = time.Now() + + pkt := &rtp.Packet{ + Header: rtp.Header{ + SequenceNumber: uint16(chunk.Sequence), + Timestamp: chunk.Timestamp, + SSRC: uint32(chunk.Sequence), // Arbitrary dummy + }, + Payload: chunk.Data, + } + + if err := a.writer.WriteRTP(pkt); err != nil { + logger.ErrorCF("voice-agent", "Failed to write RTP", map[string]any{"error": err}) + } +} + +func (a *speechAccumulator) Close() { + a.mu.Lock() + defer a.mu.Unlock() + if !a.closed { + a.writer.Close() + a.closed = true + } +} + +type Agent struct { + bus *bus.MessageBus + transcriber Transcriber + + mu sync.Mutex + sessions map[string]*speechAccumulator // keyed by sessionID_speakerID +} + +func NewAgent(mb *bus.MessageBus, t Transcriber) *Agent { + return &Agent{ + bus: mb, + transcriber: t, + sessions: make(map[string]*speechAccumulator), + } +} + +func (a *Agent) Start(ctx context.Context) { + logger.InfoCF("voice-agent", "Started Voice Agent orchestrator", nil) + go a.listenChunks(ctx) + go a.vadTick(ctx) +} + +func (a *Agent) listenChunks(ctx context.Context) { + chunks := a.bus.AudioChunksChan() + for { + select { + case <-ctx.Done(): + return + case chunk := <-chunks: + a.handleChunk(chunk) + } + } +} + +func (a *Agent) handleChunk(chunk bus.AudioChunk) { + a.mu.Lock() + defer a.mu.Unlock() + + key := fmt.Sprintf("%s_%s", chunk.SessionID, chunk.SpeakerID) + + acc, exists := a.sessions[key] + if !exists { + filename := filepath.Join(os.TempDir(), fmt.Sprintf("voice_%s_%d.ogg", key, time.Now().UnixNano())) + writer, err := oggwriter.New(filename, uint32(chunk.SampleRate), uint16(chunk.Channels)) + if err != nil { + logger.ErrorCF("voice-agent", "Failed to create OggWriter", map[string]any{"error": err}) + return + } + + acc = &speechAccumulator{ + writer: writer, + file: filename, + lastAudioAt: time.Now(), + chatID: chunk.ChatID, + speakerID: chunk.SpeakerID, + } + a.sessions[key] = acc + logger.DebugCF("voice-agent", "Started accumulating voice", map[string]any{"key": key, "file": filename}) + } + + acc.Push(chunk) +} + +func (a *Agent) vadTick(ctx context.Context) { + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + a.checkSilence(ctx) + } + } +} + +func (a *Agent) checkSilence(ctx context.Context) { + a.mu.Lock() + now := time.Now() + var finished []*speechAccumulator + + for key, acc := range a.sessions { + acc.mu.Lock() + last := acc.lastAudioAt + acc.mu.Unlock() + + if now.Sub(last) > 1500*time.Millisecond { + acc.Close() + delete(a.sessions, key) + finished = append(finished, acc) + } + } + a.mu.Unlock() + + for _, acc := range finished { + go a.processUtterance(ctx, acc) + } +} + +func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) { + defer os.Remove(acc.file) + + logger.InfoCF("voice-agent", "User finished speaking, transcribing...", map[string]any{"file": acc.file}) + + if a.transcriber == nil { + logger.ErrorCF("voice-agent", "No STT configured!", nil) + return + } + + res, err := a.transcriber.Transcribe(ctx, acc.file) + if err != nil { + logger.ErrorCF("voice-agent", "Transcription failed", map[string]any{"error": err}) + return + } + + if res.Text == "" { + logger.DebugCF("voice-agent", "Ignored empty transcription", map[string]any{"file": acc.file}) + return + } + + logger.InfoCF("voice-agent", "Transcription result", map[string]any{"text": res.Text, "duration": res.Duration}) + + channelType := "discord" + + a.bus.PublishInbound(ctx, bus.InboundMessage{ + Channel: channelType, + SenderID: acc.speakerID, + ChatID: acc.chatID, + Content: res.Text, + Peer: bus.Peer{Kind: "channel", ID: acc.chatID}, + Metadata: map[string]string{ + "is_voice": "true", + }, + }) +} diff --git a/pkg/voice/transcriber.go b/pkg/voice/transcriber.go index e949d7a22..439b18820 100644 --- a/pkg/voice/transcriber.go +++ b/pkg/voice/transcriber.go @@ -21,6 +21,7 @@ import ( type Transcriber interface { Name() string Transcribe(ctx context.Context, audioFilePath string) (*TranscriptionResponse, error) + TranscribeData(ctx context.Context, data []byte, filename string) (*TranscriptionResponse, error) } type GroqTranscriber struct { @@ -48,45 +49,24 @@ func NewGroqTranscriber(apiKey string) *GroqTranscriber { } } -func (t *GroqTranscriber) Transcribe(ctx context.Context, audioFilePath string) (*TranscriptionResponse, error) { - logger.InfoCF("voice", "Starting transcription", map[string]any{"audio_file": audioFilePath}) - - audioFile, err := os.Open(audioFilePath) - if err != nil { - logger.ErrorCF("voice", "Failed to open audio file", map[string]any{"path": audioFilePath, "error": err}) - return nil, fmt.Errorf("failed to open audio file: %w", err) - } - defer audioFile.Close() - - fileInfo, err := audioFile.Stat() - if err != nil { - logger.ErrorCF("voice", "Failed to get file info", map[string]any{"path": audioFilePath, "error": err}) - return nil, fmt.Errorf("failed to get file info: %w", err) - } - - logger.DebugCF("voice", "Audio file details", map[string]any{ - "size_bytes": fileInfo.Size(), - "file_name": filepath.Base(audioFilePath), - }) +func (t *GroqTranscriber) TranscribeData(ctx context.Context, data []byte, filename string) (*TranscriptionResponse, error) { + logger.InfoCF("voice", "Starting memory transcription", map[string]any{"filename": filename, "bytes": len(data)}) var requestBody bytes.Buffer writer := multipart.NewWriter(&requestBody) - part, err := writer.CreateFormFile("file", filepath.Base(audioFilePath)) + part, err := writer.CreateFormFile("file", filename) if err != nil { logger.ErrorCF("voice", "Failed to create form file", map[string]any{"error": err}) return nil, fmt.Errorf("failed to create form file: %w", err) } - copied, err := io.Copy(part, audioFile) - if err != nil { + if _, err := io.Copy(part, bytes.NewReader(data)); err != nil { logger.ErrorCF("voice", "Failed to copy file content", map[string]any{"error": err}) return nil, fmt.Errorf("failed to copy file content: %w", err) } - logger.DebugCF("voice", "File copied to request", map[string]any{"bytes_copied": copied}) - - if err = writer.WriteField("model", "whisper-large-v3"); err != nil { + if err = writer.WriteField("model", "whisper-large-v3-turbo"); err != nil { logger.ErrorCF("voice", "Failed to write model field", map[string]any{"error": err}) return nil, fmt.Errorf("failed to write model field: %w", err) } @@ -101,20 +81,52 @@ func (t *GroqTranscriber) Transcribe(ctx context.Context, audioFilePath string) return nil, fmt.Errorf("failed to close multipart writer: %w", err) } + return t.doRequest(ctx, &requestBody, writer.FormDataContentType(), int64(len(data))) +} + +func (t *GroqTranscriber) Transcribe(ctx context.Context, audioFilePath string) (*TranscriptionResponse, error) { + logger.InfoCF("voice", "Starting transcription", map[string]any{"audio_file": audioFilePath}) + + audioFile, err := os.Open(audioFilePath) + if err != nil { + return nil, fmt.Errorf("failed to open audio file: %w", err) + } + defer audioFile.Close() + + fileInfo, err := audioFile.Stat() + if err != nil { + return nil, err + } + + var requestBody bytes.Buffer + writer := multipart.NewWriter(&requestBody) + part, err := writer.CreateFormFile("file", filepath.Base(audioFilePath)) + if err != nil { + return nil, err + } + io.Copy(part, audioFile) + writer.WriteField("model", "whisper-large-v3") + writer.WriteField("response_format", "json") + writer.Close() + + return t.doRequest(ctx, &requestBody, writer.FormDataContentType(), fileInfo.Size()) +} + +func (t *GroqTranscriber) doRequest(ctx context.Context, requestBody *bytes.Buffer, contentType string, fileSize int64) (*TranscriptionResponse, error) { url := t.apiBase + "/audio/transcriptions" - req, err := http.NewRequestWithContext(ctx, "POST", url, &requestBody) + req, err := http.NewRequestWithContext(ctx, "POST", url, requestBody) if err != nil { logger.ErrorCF("voice", "Failed to create request", map[string]any{"error": err}) return nil, fmt.Errorf("failed to create request: %w", err) } - req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Content-Type", contentType) req.Header.Set("Authorization", "Bearer "+t.apiKey) logger.DebugCF("voice", "Sending transcription request to Groq API", map[string]any{ "url": url, "request_size_bytes": requestBody.Len(), - "file_size_bytes": fileInfo.Size(), + "file_size_bytes": fileSize, }) resp, err := t.httpClient.Do(req) From 87e607b6238c09f9c61cc1d8e2f0fdc694ea0a9f Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 07:18:14 +0100 Subject: [PATCH 02/51] update with corrent transcriber and leave_tool --- go.mod | 3 ++ go.sum | 7 +++-- pkg/agent/loop.go | 3 ++ pkg/bus/types.go | 3 +- pkg/channels/discord/discord.go | 30 +++++++++++++++++++ pkg/channels/discord/voice.go | 24 +++++++++++++-- pkg/tools/voice_leave.go | 52 +++++++++++++++++++++++++++++++++ pkg/voice/agent.go | 7 ++++- pkg/voice/transcriber.go | 4 +-- 9 files changed, 123 insertions(+), 10 deletions(-) create mode 100644 pkg/tools/voice_leave.go diff --git a/go.mod b/go.mod index 3fcc360a6..91d158d25 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( require ( filippo.io/edwards25519 v1.2.0 // indirect github.com/beeper/argo-go v1.1.2 // indirect + github.com/cloudflare/circl v1.6.3 // indirect github.com/coder/websocket v1.8.14 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -101,3 +102,5 @@ require ( golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect ) + +replace github.com/bwmarrin/discordgo => github.com/yeongaori/discordgo-fork v0.0.0-20260319072544-e8e546f5d532 diff --git a/go.sum b/go.sum index f95643290..16d618f5b 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,6 @@ github.com/anthropics/anthropic-sdk-go v1.26.0 h1:oUTzFaUpAevfuELAP1sjL6CQJ9HHAf github.com/anthropics/anthropic-sdk-go v1.26.0/go.mod h1:qUKmaW+uuPB64iy1l+4kOSvaLqPXnHTTBKH6RVZ7q5Q= github.com/beeper/argo-go v1.1.2 h1:UQI2G8F+NLfGTOmTUI0254pGKx/HUU/etbUGTJv91Fs= github.com/beeper/argo-go v1.1.2/go.mod h1:M+LJAnyowKVQ6Rdj6XYGEn+qcVFkb3R/MUpqkGR0hM4= -github.com/bwmarrin/discordgo v0.29.0 h1:FmWeXFaKUwrcL3Cx65c20bTRW+vOb6k8AnaP+EgjDno= -github.com/bwmarrin/discordgo v0.29.0/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -31,6 +29,8 @@ github.com/caarlos0/env/v11 v11.4.0 h1:Kcb6t5kIIr4XkoQC9AF2j+8E1Jsrl3Wz/hhm1LtoG github.com/caarlos0/env/v11 v11.4.0/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g= @@ -236,6 +236,8 @@ github.com/vektah/gqlparser/v2 v2.5.27 h1:RHPD3JOplpk5mP5JGX8RKZkt2/Vwj/PZv0HxTd github.com/vektah/gqlparser/v2 v2.5.27/go.mod h1:D1/VCZtV3LPnQrcPBeR/q5jkSQIPti0uYCP/RI0gIeo= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yeongaori/discordgo-fork v0.0.0-20260319072544-e8e546f5d532 h1:gxFHYeUDGziRb0zXYEqBFohC+NJbIW9L0tddaXMWr2o= +github.com/yeongaori/discordgo-fork v0.0.0-20260319072544-e8e546f5d532/go.mod h1:A0FcMFJKJ9fRjgSuZ2o+pIQ6mPS81SVuiLN2vYTa7Ao= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -255,7 +257,6 @@ golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index ed5c73afc..c626d4238 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -199,6 +199,9 @@ func registerSharedTools( agent.Tools.Register(messageTool) } + // Always register Voice Leave Tool (it inherently checks if channel == "discord") + agent.Tools.Register(tools.NewVoiceLeaveTool(msgBus)) + // Send file tool (outbound media via MediaStore — store injected later by SetMediaStore) if cfg.Tools.IsToolEnabled("send_file") { sendFileTool := tools.NewSendFileTool( diff --git a/pkg/bus/types.go b/pkg/bus/types.go index 9c637f3e7..b4e99d955 100644 --- a/pkg/bus/types.go +++ b/pkg/bus/types.go @@ -68,6 +68,7 @@ type AudioChunk struct { // VoiceControl represents state or commands for voice sessions. type VoiceControl struct { SessionID string `json:"session_id"` - Type string `json:"type"` // "state", "command" + ChatID string `json:"chat_id"` + Type string `json:"type"` // "state", "command" Action string `json:"action"` // "idle", "listening", "start", "stop" } diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index 6e5319080..956d84e76 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -91,6 +91,8 @@ func (c *DiscordChannel) Start(ctx context.Context) error { c.botUserID = botUser.ID c.session.AddHandler(c.handleMessage) + + go c.listenVoiceControl(c.ctx) if err := c.session.Open(); err != nil { return fmt.Errorf("failed to open discord session: %w", err) @@ -618,3 +620,31 @@ func (c *DiscordChannel) stripBotMention(text string) string { text = strings.ReplaceAll(text, fmt.Sprintf("<@!%s>", c.botUserID), "") return strings.TrimSpace(text) } + +func (c *DiscordChannel) listenVoiceControl(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case ctrl := <-c.bus.VoiceControlsChan(): + if ctrl.Type == "command" && ctrl.Action == "leave" { + var guildID string + if strings.HasPrefix(ctrl.SessionID, "discord_vc_") { + guildID = strings.TrimPrefix(ctrl.SessionID, "discord_vc_") + } else if ctrl.ChatID != "" { + ch, err := c.session.State.Channel(ctrl.ChatID) + if err == nil { + guildID = ch.GuildID + } + } + + if guildID != "" { + vc, exists := c.session.VoiceConnections[guildID] + if exists && vc != nil { + vc.Disconnect(ctx) + } + } + } + } + } +} diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index e2aae52ed..23386e642 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -2,6 +2,7 @@ package discord import ( "fmt" + "time" "github.com/bwmarrin/discordgo" "github.com/sipeed/picoclaw/pkg/bus" @@ -17,7 +18,7 @@ func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.M } logger.InfoCF("discord", "Joining voice channel", map[string]any{"channel": vs.ChannelID}) - vc, err := s.ChannelVoiceJoin(m.GuildID, vs.ChannelID, false, false) + vc, err := s.ChannelVoiceJoin(c.ctx, m.GuildID, vs.ChannelID, false, false) if err != nil { s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("Failed to join voice channel: %v", err)) return true @@ -29,7 +30,7 @@ func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.M } else if m.Content == "!vc leave" { vc, exists := s.VoiceConnections[m.GuildID] if exists && vc != nil { - vc.Disconnect() + vc.Disconnect(c.ctx) s.ChannelMessageSend(m.ChannelID, "Left Voice Channel.") } else { s.ChannelMessageSend(m.ChannelID, "Not in a voice channel.") @@ -42,6 +43,17 @@ func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.M func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID string, chatID string) { logger.InfoCF("discord", "Started listening for voice", map[string]any{"guild": guildID}) + go func() { + time.Sleep(250 * time.Millisecond) // Wait a bit for connection to settle + vc.Speaking(true) + for i := 0; i < 5; i++ { + vc.OpusSend <- []byte{0xF8, 0xFF, 0xFE} + time.Sleep(20 * time.Millisecond) + } + vc.Speaking(false) + logger.DebugCF("discord", "Sent wake-up silence frames", nil) + }() + sessionID := fmt.Sprintf("discord_vc_%s", guildID) c.bus.PublishVoiceControl(c.ctx, bus.VoiceControl{ @@ -62,7 +74,13 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str return } - if p == nil { + logger.DebugCF("discord", "Received Opus packet", map[string]any{ + "seq": p.Sequence, + "len": len(p.Opus), + "ssrc": p.SSRC, + }) + + if p == nil || len(p.Opus) == 0 { continue } diff --git a/pkg/tools/voice_leave.go b/pkg/tools/voice_leave.go new file mode 100644 index 000000000..882e7cb9d --- /dev/null +++ b/pkg/tools/voice_leave.go @@ -0,0 +1,52 @@ +package tools + +import ( + "context" + + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/logger" +) + +type VoiceLeaveTool struct { + bus *bus.MessageBus +} + +func NewVoiceLeaveTool(mb *bus.MessageBus) *VoiceLeaveTool { + return &VoiceLeaveTool{bus: mb} +} + +func (t *VoiceLeaveTool) Name() string { + return "voice_leave" +} + +func (t *VoiceLeaveTool) Description() string { + return "Disconnects the bot from the current voice channel. Use this tool when the user says goodbye or explicitly asks you to leave the voice chat." +} + +func (t *VoiceLeaveTool) Parameters() map[string]any { + return map[string]any{ + "type": "object", + "properties": map[string]any{}, + } +} + +func (t *VoiceLeaveTool) Execute(ctx context.Context, args map[string]any) *ToolResult { + channel := ToolChannel(ctx) + chatID := ToolChatID(ctx) + + if channel != "discord" { + return &ToolResult{ForLLM: "Can only leave voice channels on Discord", IsError: true} + } + + t.bus.PublishVoiceControl(ctx, bus.VoiceControl{ + ChatID: chatID, + Type: "command", + Action: "leave", + }) + + logger.InfoCF("agent", "Voice command triggered via tool: leave", map[string]any{"chat_id": chatID}) + + return &ToolResult{ + ForLLM: "Successfully sent disconnect command to voice adapter.", + } +} diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index 4362eb863..40bc02c57 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "sync" "time" @@ -22,6 +23,7 @@ type speechAccumulator struct { closed bool chatID string speakerID string + sessionID string } func (a *speechAccumulator) Push(chunk bus.AudioChunk) { @@ -112,6 +114,7 @@ func (a *Agent) handleChunk(chunk bus.AudioChunk) { lastAudioAt: time.Now(), chatID: chunk.ChatID, speakerID: chunk.SpeakerID, + sessionID: chunk.SessionID, } a.sessions[key] = acc logger.DebugCF("voice-agent", "Started accumulating voice", map[string]any{"key": key, "file": filename}) @@ -182,11 +185,13 @@ func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) { channelType := "discord" + oralPrompt := "\n\n[SYSTEM]: The user just spoke this to you over voice chat. Please reply in a highly concise, conversational, oral style suitable for text-to-speech. Do not use markdown, emojis, asterisks, or code blocks. Speak naturally. If the user expresses that they want to end the call, say goodbye and use the voice_leave tool." + a.bus.PublishInbound(ctx, bus.InboundMessage{ Channel: channelType, SenderID: acc.speakerID, ChatID: acc.chatID, - Content: res.Text, + Content: res.Text + oralPrompt, Peer: bus.Peer{Kind: "channel", ID: acc.chatID}, Metadata: map[string]string{ "is_voice": "true", diff --git a/pkg/voice/transcriber.go b/pkg/voice/transcriber.go index 439b18820..f0ce8b5cd 100644 --- a/pkg/voice/transcriber.go +++ b/pkg/voice/transcriber.go @@ -182,9 +182,9 @@ func DetectTranscriber(cfg *config.Config) Transcriber { if key := cfg.Providers.Groq.APIKey; key != "" { return NewGroqTranscriber(key) } - // Fall back to any model-list entry that uses the groq/ protocol. + // Fall back to any model-list entry that uses the groq/ protocol or is explicitly named groq. for _, mc := range cfg.ModelList { - if strings.HasPrefix(mc.Model, "groq/") && mc.APIKey != "" { + if (strings.HasPrefix(mc.Model, "groq/") || mc.ModelName == "groq" || mc.Model == "whisper-large-v3-turbo") && mc.APIKey != "" { return NewGroqTranscriber(mc.APIKey) } } From b2f182bb1054fc50cb88847a12084eda5a189423 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 07:18:39 +0100 Subject: [PATCH 03/51] fix lint --- pkg/voice/agent.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index 40bc02c57..233da6019 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "path/filepath" - "strings" "sync" "time" From 36bf992b623caa6eb9323ea0f41abf92caa78881 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 07:21:33 +0100 Subject: [PATCH 04/51] remove leave tool due to delay --- pkg/agent/loop.go | 3 -- pkg/bus/types.go | 3 +- pkg/channels/discord/discord.go | 11 +------ pkg/tools/voice_leave.go | 52 --------------------------------- pkg/voice/agent.go | 19 +++++++++++- 5 files changed, 20 insertions(+), 68 deletions(-) delete mode 100644 pkg/tools/voice_leave.go diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index c626d4238..ed5c73afc 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -199,9 +199,6 @@ func registerSharedTools( agent.Tools.Register(messageTool) } - // Always register Voice Leave Tool (it inherently checks if channel == "discord") - agent.Tools.Register(tools.NewVoiceLeaveTool(msgBus)) - // Send file tool (outbound media via MediaStore — store injected later by SetMediaStore) if cfg.Tools.IsToolEnabled("send_file") { sendFileTool := tools.NewSendFileTool( diff --git a/pkg/bus/types.go b/pkg/bus/types.go index b4e99d955..9c637f3e7 100644 --- a/pkg/bus/types.go +++ b/pkg/bus/types.go @@ -68,7 +68,6 @@ type AudioChunk struct { // VoiceControl represents state or commands for voice sessions. type VoiceControl struct { SessionID string `json:"session_id"` - ChatID string `json:"chat_id"` - Type string `json:"type"` // "state", "command" + Type string `json:"type"` // "state", "command" Action string `json:"action"` // "idle", "listening", "start", "stop" } diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index 956d84e76..73d1721cf 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -628,17 +628,8 @@ func (c *DiscordChannel) listenVoiceControl(ctx context.Context) { return case ctrl := <-c.bus.VoiceControlsChan(): if ctrl.Type == "command" && ctrl.Action == "leave" { - var guildID string if strings.HasPrefix(ctrl.SessionID, "discord_vc_") { - guildID = strings.TrimPrefix(ctrl.SessionID, "discord_vc_") - } else if ctrl.ChatID != "" { - ch, err := c.session.State.Channel(ctrl.ChatID) - if err == nil { - guildID = ch.GuildID - } - } - - if guildID != "" { + guildID := strings.TrimPrefix(ctrl.SessionID, "discord_vc_") vc, exists := c.session.VoiceConnections[guildID] if exists && vc != nil { vc.Disconnect(ctx) diff --git a/pkg/tools/voice_leave.go b/pkg/tools/voice_leave.go deleted file mode 100644 index 882e7cb9d..000000000 --- a/pkg/tools/voice_leave.go +++ /dev/null @@ -1,52 +0,0 @@ -package tools - -import ( - "context" - - "github.com/sipeed/picoclaw/pkg/bus" - "github.com/sipeed/picoclaw/pkg/logger" -) - -type VoiceLeaveTool struct { - bus *bus.MessageBus -} - -func NewVoiceLeaveTool(mb *bus.MessageBus) *VoiceLeaveTool { - return &VoiceLeaveTool{bus: mb} -} - -func (t *VoiceLeaveTool) Name() string { - return "voice_leave" -} - -func (t *VoiceLeaveTool) Description() string { - return "Disconnects the bot from the current voice channel. Use this tool when the user says goodbye or explicitly asks you to leave the voice chat." -} - -func (t *VoiceLeaveTool) Parameters() map[string]any { - return map[string]any{ - "type": "object", - "properties": map[string]any{}, - } -} - -func (t *VoiceLeaveTool) Execute(ctx context.Context, args map[string]any) *ToolResult { - channel := ToolChannel(ctx) - chatID := ToolChatID(ctx) - - if channel != "discord" { - return &ToolResult{ForLLM: "Can only leave voice channels on Discord", IsError: true} - } - - t.bus.PublishVoiceControl(ctx, bus.VoiceControl{ - ChatID: chatID, - Type: "command", - Action: "leave", - }) - - logger.InfoCF("agent", "Voice command triggered via tool: leave", map[string]any{"chat_id": chatID}) - - return &ToolResult{ - ForLLM: "Successfully sent disconnect command to voice adapter.", - } -} diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index 233da6019..6a2abd24f 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "sync" "time" @@ -184,7 +185,23 @@ func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) { channelType := "discord" - oralPrompt := "\n\n[SYSTEM]: The user just spoke this to you over voice chat. Please reply in a highly concise, conversational, oral style suitable for text-to-speech. Do not use markdown, emojis, asterisks, or code blocks. Speak naturally. If the user expresses that they want to end the call, say goodbye and use the voice_leave tool." + text := strings.ToLower(strings.TrimSpace(res.Text)) + if strings.Contains(text, "leave the voice channel") || strings.Contains(text, "leave voice") || strings.Contains(text, "disconnect voice") { + logger.InfoCF("voice-agent", "Voice command triggered: leave", nil) + a.bus.PublishVoiceControl(ctx, bus.VoiceControl{ + SessionID: acc.sessionID, + Type: "command", + Action: "leave", + }) + a.bus.PublishOutbound(ctx, bus.OutboundMessage{ + Channel: channelType, + ChatID: acc.chatID, + Content: "Goodbye! Leaving the voice channel.", + }) + return + } + + oralPrompt := "\n\n[SYSTEM]: The user just spoke this to you over voice chat. Please reply in a highly concise, conversational, oral style suitable for text-to-speech. Do not use markdown, emojis, asterisks, or code blocks. Speak naturally." a.bus.PublishInbound(ctx, bus.InboundMessage{ Channel: channelType, From d572f8124f6c17080c6e40b81e0a24a2b44471b6 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 07:53:28 +0100 Subject: [PATCH 05/51] add tts support --- pkg/channels/discord/discord.go | 23 +++++++ pkg/channels/discord/init.go | 7 ++- pkg/channels/discord/voice.go | 55 ++++++++++++++++ pkg/voice/tts.go | 108 ++++++++++++++++++++++++++++++++ 4 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 pkg/voice/tts.go diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index 73d1721cf..88ac0f675 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -21,6 +21,7 @@ import ( "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/media" "github.com/sipeed/picoclaw/pkg/utils" + "github.com/sipeed/picoclaw/pkg/voice" ) const ( @@ -43,6 +44,7 @@ type DiscordChannel struct { typingStop map[string]chan struct{} // chatID → stop signal botUserID string // stored for mention checking bus *bus.MessageBus + tts voice.TTSProvider } func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordChannel, error) { @@ -146,6 +148,14 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) erro return nil } + if c.tts != nil { + if ch, err := c.session.State.Channel(channelID); err == nil && ch.GuildID != "" { + if vc, ok := c.session.VoiceConnections[ch.GuildID]; ok && vc != nil { + go c.playTTS(context.Background(), vc, msg.Content) + } + } + } + return c.sendChunk(ctx, channelID, msg.Content, msg.ReplyToMessageID) } @@ -639,3 +649,16 @@ func (c *DiscordChannel) listenVoiceControl(ctx context.Context) { } } } + +func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnection, text string) { + stream, err := c.tts.Synthesize(ctx, text) + if err != nil { + logger.ErrorCF("discord", "TTS synthesize failed", map[string]any{"error": err.Error()}) + return + } + defer stream.Close() + + if err := streamOggOpusToDiscord(vc, stream); err != nil { + logger.ErrorCF("discord", "TTS playback failed", map[string]any{"error": err.Error()}) + } +} diff --git a/pkg/channels/discord/init.go b/pkg/channels/discord/init.go index 15a539804..13e4fbc91 100644 --- a/pkg/channels/discord/init.go +++ b/pkg/channels/discord/init.go @@ -4,10 +4,15 @@ import ( "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/voice" ) func init() { channels.RegisterFactory("discord", func(cfg *config.Config, b *bus.MessageBus) (channels.Channel, error) { - return NewDiscordChannel(cfg.Channels.Discord, b) + ch, err := NewDiscordChannel(cfg.Channels.Discord, b) + if err == nil { + ch.tts = voice.DetectTTS(cfg) + } + return ch, err }) } diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index 23386e642..72f767b85 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -1,7 +1,9 @@ package discord import ( + "bytes" "fmt" + "io" "time" "github.com/bwmarrin/discordgo" @@ -40,6 +42,59 @@ func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.M return false } +func VoiceReceiveActive(vc *discordgo.VoiceConnection) bool { + return vc != nil && vc.OpusRecv != nil +} + +func streamOggOpusToDiscord(vc *discordgo.VoiceConnection, r io.Reader) error { + // Wait for the speaking transition to register + vc.Speaking(true) + defer vc.Speaking(false) + + var packet []byte + header := make([]byte, 27) + + for { + if _, err := io.ReadFull(r, header); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + return nil + } + return fmt.Errorf("failed to read ogg header: %w", err) + } + if string(header[:4]) != "OggS" { + return fmt.Errorf("invalid ogg magic string") + } + + pageSegments := int(header[26]) + segmentTable := make([]byte, pageSegments) + if _, err := io.ReadFull(r, segmentTable); err != nil { + return fmt.Errorf("failed to read segment table: %w", err) + } + + for _, lacing := range segmentTable { + segment := make([]byte, lacing) + if _, err := io.ReadFull(r, segment); err != nil { + return fmt.Errorf("failed to read segment data: %w", err) + } + + packet = append(packet, segment...) + + // If lacing is less than 255, the packet is complete + if lacing < 255 { + if len(packet) > 0 { + // Ignore Ogg Opus headers + if !bytes.HasPrefix(packet, []byte("OpusHead")) && !bytes.HasPrefix(packet, []byte("OpusTags")) { + // Pacing is handled natively by vc.OpusSend blocking (it has an internal ticker) + vc.OpusSend <- packet + } + // Start new packet + packet = nil + } + } + } + } +} + func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID string, chatID string) { logger.InfoCF("discord", "Started listening for voice", map[string]any{"guild": guildID}) diff --git a/pkg/voice/tts.go b/pkg/voice/tts.go new file mode 100644 index 000000000..8de0bbc9c --- /dev/null +++ b/pkg/voice/tts.go @@ -0,0 +1,108 @@ +package voice + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/logger" +) + +type TTSProvider interface { + Name() string + Synthesize(ctx context.Context, text string) (io.ReadCloser, error) +} + +type OpenAITTSProvider struct { + apiKey string + apiBase string + voice string + model string + httpClient *http.Client +} + +func NewOpenAITTSProvider(apiKey string, apiBase string, proxyURL string) *OpenAITTSProvider { + if apiBase == "" || apiBase == "https://api.openai.com/v1" { + apiBase = "https://api.openai.com/v1/audio/speech" + } else if !strings.HasSuffix(apiBase, "/audio/speech") { + // Just in case they provide openrouter base or standard base + apiBase = strings.TrimSuffix(apiBase, "/") + "/audio/speech" + } + + client := &http.Client{ + Timeout: 60 * time.Second, + } + + if proxyURL != "" { + if pURL, err := url.Parse(proxyURL); err == nil { + client.Transport = &http.Transport{ + Proxy: http.ProxyURL(pURL), + } + } + } + + return &OpenAITTSProvider{ + apiKey: apiKey, + apiBase: apiBase, + voice: "alloy", + model: "tts-1", + httpClient: client, + } +} + +func (t *OpenAITTSProvider) Name() string { + return "openai-tts" +} + +func (t *OpenAITTSProvider) Synthesize(ctx context.Context, text string) (io.ReadCloser, error) { + logger.InfoCF("voice-tts", "Starting TTS synthesis", map[string]any{"text_len": len(text)}) + + reqBody := map[string]any{ + "model": t.model, + "input": text, + "voice": t.voice, + "response_format": "opus", + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", t.apiBase, bytes.NewReader(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+t.apiKey) + + resp, err := t.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + + if resp.StatusCode != http.StatusOK { + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body)) + } + + return resp.Body, nil +} + +func DetectTTS(cfg *config.Config) TTSProvider { + for _, mc := range cfg.ModelList { + if strings.Contains(strings.ToLower(mc.ModelName), "tts") && mc.APIKey != "" { + return NewOpenAITTSProvider(mc.APIKey, mc.APIBase, mc.Proxy) + } + } + return nil +} From 57a468507bd7e557673f4d9a0f282cd764a3b885 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 08:05:54 +0100 Subject: [PATCH 06/51] refactor for #1648 --- pkg/agent/loop.go | 6 +- pkg/{voice/transcriber.go => asr/asr.go} | 2 +- .../transcriber_test.go => asr/asr_test.go} | 2 +- pkg/audio/ogg.go | 55 +++++++++++++++++++ pkg/bus/types.go | 2 +- pkg/channels/discord/discord.go | 6 +- pkg/channels/discord/init.go | 4 +- pkg/gateway/gateway.go | 5 +- pkg/{voice => tts}/tts.go | 2 +- pkg/voice/agent.go | 5 +- 10 files changed, 73 insertions(+), 16 deletions(-) rename pkg/{voice/transcriber.go => asr/asr.go} (99%) rename pkg/{voice/transcriber_test.go => asr/asr_test.go} (99%) create mode 100644 pkg/audio/ogg.go rename pkg/{voice => tts}/tts.go (99%) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index ed5c73afc..54a2bfb96 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -19,6 +19,7 @@ import ( "time" "unicode/utf8" + "github.com/sipeed/picoclaw/pkg/asr" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/commands" @@ -32,7 +33,6 @@ import ( "github.com/sipeed/picoclaw/pkg/state" "github.com/sipeed/picoclaw/pkg/tools" "github.com/sipeed/picoclaw/pkg/utils" - "github.com/sipeed/picoclaw/pkg/voice" ) type AgentLoop struct { @@ -45,7 +45,7 @@ type AgentLoop struct { fallback *providers.FallbackChain channelManager *channels.Manager mediaStore media.MediaStore - transcriber voice.Transcriber + transcriber asr.Transcriber cmdRegistry *commands.Registry mcp mcpRuntime mu sync.RWMutex @@ -502,7 +502,7 @@ func (al *AgentLoop) SetMediaStore(s media.MediaStore) { } // SetTranscriber injects a voice transcriber for agent-level audio transcription. -func (al *AgentLoop) SetTranscriber(t voice.Transcriber) { +func (al *AgentLoop) SetTranscriber(t asr.Transcriber) { al.transcriber = t } diff --git a/pkg/voice/transcriber.go b/pkg/asr/asr.go similarity index 99% rename from pkg/voice/transcriber.go rename to pkg/asr/asr.go index f0ce8b5cd..dc22588f0 100644 --- a/pkg/voice/transcriber.go +++ b/pkg/asr/asr.go @@ -1,4 +1,4 @@ -package voice +package asr import ( "bytes" diff --git a/pkg/voice/transcriber_test.go b/pkg/asr/asr_test.go similarity index 99% rename from pkg/voice/transcriber_test.go rename to pkg/asr/asr_test.go index 9b6add333..4e8733327 100644 --- a/pkg/voice/transcriber_test.go +++ b/pkg/asr/asr_test.go @@ -1,4 +1,4 @@ -package voice +package asr import ( "context" diff --git a/pkg/audio/ogg.go b/pkg/audio/ogg.go new file mode 100644 index 000000000..49a2b93f9 --- /dev/null +++ b/pkg/audio/ogg.go @@ -0,0 +1,55 @@ +package audio + +import ( + "bytes" + "fmt" + "io" +) + +// DecodeOggOpus reads an Ogg format stream and extracts individual Opus payloads. +// It calls onFrame for every complete Opus frame found in the stream. +func DecodeOggOpus(r io.Reader, onFrame func([]byte) error) error { + var packet []byte + header := make([]byte, 27) + + for { + if _, err := io.ReadFull(r, header); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + return nil + } + return fmt.Errorf("failed to read ogg header: %w", err) + } + if string(header[:4]) != "OggS" { + return fmt.Errorf("invalid ogg magic string") + } + + pageSegments := int(header[26]) + segmentTable := make([]byte, pageSegments) + if _, err := io.ReadFull(r, segmentTable); err != nil { + return fmt.Errorf("failed to read segment table: %w", err) + } + + for _, lacing := range segmentTable { + segment := make([]byte, lacing) + if _, err := io.ReadFull(r, segment); err != nil { + return fmt.Errorf("failed to read segment data: %w", err) + } + + packet = append(packet, segment...) + + // If lacing is less than 255, the packet is complete + if lacing < 255 { + if len(packet) > 0 { + // Ignore Ogg Opus headers + if !bytes.HasPrefix(packet, []byte("OpusHead")) && !bytes.HasPrefix(packet, []byte("OpusTags")) { + if err := onFrame(packet); err != nil { + return err + } + } + // Start new packet + packet = nil + } + } + } + } +} diff --git a/pkg/bus/types.go b/pkg/bus/types.go index 9c637f3e7..e036b4ede 100644 --- a/pkg/bus/types.go +++ b/pkg/bus/types.go @@ -68,6 +68,6 @@ type AudioChunk struct { // VoiceControl represents state or commands for voice sessions. type VoiceControl struct { SessionID string `json:"session_id"` - Type string `json:"type"` // "state", "command" + Type string `json:"type"` // "state", "command" Action string `json:"action"` // "idle", "listening", "start", "stop" } diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index 88ac0f675..08e7fe718 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -20,8 +20,8 @@ import ( "github.com/sipeed/picoclaw/pkg/identity" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/media" + "github.com/sipeed/picoclaw/pkg/tts" "github.com/sipeed/picoclaw/pkg/utils" - "github.com/sipeed/picoclaw/pkg/voice" ) const ( @@ -44,7 +44,7 @@ type DiscordChannel struct { typingStop map[string]chan struct{} // chatID → stop signal botUserID string // stored for mention checking bus *bus.MessageBus - tts voice.TTSProvider + tts tts.TTSProvider } func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordChannel, error) { @@ -93,7 +93,7 @@ func (c *DiscordChannel) Start(ctx context.Context) error { c.botUserID = botUser.ID c.session.AddHandler(c.handleMessage) - + go c.listenVoiceControl(c.ctx) if err := c.session.Open(); err != nil { diff --git a/pkg/channels/discord/init.go b/pkg/channels/discord/init.go index 13e4fbc91..e6be8ff36 100644 --- a/pkg/channels/discord/init.go +++ b/pkg/channels/discord/init.go @@ -4,14 +4,14 @@ import ( "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/config" - "github.com/sipeed/picoclaw/pkg/voice" + "github.com/sipeed/picoclaw/pkg/tts" ) func init() { channels.RegisterFactory("discord", func(cfg *config.Config, b *bus.MessageBus) (channels.Channel, error) { ch, err := NewDiscordChannel(cfg.Channels.Discord, b) if err == nil { - ch.tts = voice.DetectTTS(cfg) + ch.tts = tts.DetectTTS(cfg) } return ch, err }) diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go index 3a0478d1d..9037095ae 100644 --- a/pkg/gateway/gateway.go +++ b/pkg/gateway/gateway.go @@ -12,6 +12,7 @@ import ( "time" "github.com/sipeed/picoclaw/pkg/agent" + "github.com/sipeed/picoclaw/pkg/asr" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" _ "github.com/sipeed/picoclaw/pkg/channels/dingtalk" @@ -284,7 +285,7 @@ func setupAndStartServices( agentLoop.SetChannelManager(runningServices.ChannelManager) agentLoop.SetMediaStore(runningServices.MediaStore) - if transcriber := voice.DetectTranscriber(cfg); transcriber != nil { + if transcriber := asr.DetectTranscriber(cfg); transcriber != nil { agentLoop.SetTranscriber(transcriber) logger.InfoCF("voice", "Transcription enabled (agent-level)", map[string]any{"provider": transcriber.Name()}) @@ -522,7 +523,7 @@ func restartServices( fmt.Println(" ✓ Device event service restarted") } - transcriber := voice.DetectTranscriber(cfg) + transcriber := asr.DetectTranscriber(cfg) al.SetTranscriber(transcriber) if transcriber != nil { logger.InfoCF("voice", "Transcription re-enabled (agent-level)", map[string]any{"provider": transcriber.Name()}) diff --git a/pkg/voice/tts.go b/pkg/tts/tts.go similarity index 99% rename from pkg/voice/tts.go rename to pkg/tts/tts.go index 8de0bbc9c..63b4ecd24 100644 --- a/pkg/voice/tts.go +++ b/pkg/tts/tts.go @@ -1,4 +1,4 @@ -package voice +package tts import ( "bytes" diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index 6a2abd24f..6a330d3d4 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -11,6 +11,7 @@ import ( "github.com/pion/rtp" "github.com/pion/webrtc/v3/pkg/media/oggwriter" + "github.com/sipeed/picoclaw/pkg/asr" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/logger" ) @@ -61,13 +62,13 @@ func (a *speechAccumulator) Close() { type Agent struct { bus *bus.MessageBus - transcriber Transcriber + transcriber asr.Transcriber mu sync.Mutex sessions map[string]*speechAccumulator // keyed by sessionID_speakerID } -func NewAgent(mb *bus.MessageBus, t Transcriber) *Agent { +func NewAgent(mb *bus.MessageBus, t asr.Transcriber) *Agent { return &Agent{ bus: mb, transcriber: t, From e6c95021a4c4c4671c30b357836e4fc8c8ef8718 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 08:24:17 +0100 Subject: [PATCH 07/51] update with streaming support --- pkg/audio/sentence.go | 96 +++++++++++++++++++++++++++++++++ pkg/channels/discord/discord.go | 88 +++++++++++++++++++++++++++--- pkg/channels/discord/voice.go | 39 ++++++++++++-- 3 files changed, 213 insertions(+), 10 deletions(-) create mode 100644 pkg/audio/sentence.go diff --git a/pkg/audio/sentence.go b/pkg/audio/sentence.go new file mode 100644 index 000000000..c7a9b2f26 --- /dev/null +++ b/pkg/audio/sentence.go @@ -0,0 +1,96 @@ +package audio + +import ( + "strings" + "unicode" +) + +// SplitSentences splits text into sentence-sized chunks suitable for TTS synthesis. +// It splits on sentence-ending punctuation (.!?\n) while avoiding false splits +// on abbreviations and decimal numbers. Very short fragments are merged with +// the next sentence to prevent choppy playback. +func SplitSentences(text string) []string { + if text == "" { + return nil + } + + var sentences []string + var current strings.Builder + runes := []rune(text) + + for i := 0; i < len(runes); i++ { + r := runes[i] + current.WriteRune(r) + + if r == '\n' { + s := strings.TrimSpace(current.String()) + if s != "" { + sentences = append(sentences, s) + } + current.Reset() + continue + } + + if r == '.' || r == '!' || r == '?' { + // Avoid splitting on decimal numbers like "3.14" + if r == '.' && i > 0 && unicode.IsDigit(runes[i-1]) && + i+1 < len(runes) && unicode.IsDigit(runes[i+1]) { + continue + } + + // Consume trailing punctuation and spaces (e.g., "..." or "?!") + for i+1 < len(runes) && (runes[i+1] == '.' || runes[i+1] == '!' || runes[i+1] == '?' || runes[i+1] == ' ') { + i++ + current.WriteRune(runes[i]) + } + + s := strings.TrimSpace(current.String()) + if s != "" { + sentences = append(sentences, s) + } + current.Reset() + } + } + + // Flush remaining text + if s := strings.TrimSpace(current.String()); s != "" { + sentences = append(sentences, s) + } + + // Merge very short fragments with the next sentence + return mergeShorties(sentences, 15) +} + +// mergeShorties merges sentences shorter than minLen characters with the following sentence. +func mergeShorties(sentences []string, minLen int) []string { + if len(sentences) <= 1 { + return sentences + } + + var merged []string + var buf string + + for _, s := range sentences { + if buf != "" { + buf += " " + s + if len([]rune(buf)) >= minLen { + merged = append(merged, buf) + buf = "" + } + } else if len([]rune(s)) < minLen { + buf = s + } else { + merged = append(merged, s) + } + } + + if buf != "" { + if len(merged) > 0 { + merged[len(merged)-1] += " " + buf + } else { + merged = append(merged, buf) + } + } + + return merged +} diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index 08e7fe718..c32a6409c 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -3,6 +3,7 @@ package discord import ( "context" "fmt" + "io" "net/http" "net/url" "os" @@ -14,6 +15,7 @@ import ( "github.com/bwmarrin/discordgo" "github.com/gorilla/websocket" + "github.com/sipeed/picoclaw/pkg/audio" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/config" @@ -45,6 +47,10 @@ type DiscordChannel struct { botUserID string // stored for mention checking bus *bus.MessageBus tts tts.TTSProvider + + // TTS interruption: cancel active playback when user speaks + ttsMu sync.Mutex + cancelTTS context.CancelFunc } func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordChannel, error) { @@ -151,7 +157,16 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) erro if c.tts != nil { if ch, err := c.session.State.Channel(channelID); err == nil && ch.GuildID != "" { if vc, ok := c.session.VoiceConnections[ch.GuildID]; ok && vc != nil { - go c.playTTS(context.Background(), vc, msg.Content) + // Cancel any previous TTS playback + c.ttsMu.Lock() + if c.cancelTTS != nil { + c.cancelTTS() + } + ttsCtx, ttsCancel := context.WithCancel(context.Background()) + c.cancelTTS = ttsCancel + c.ttsMu.Unlock() + + go c.playTTS(ttsCtx, vc, msg.Content) } } } @@ -651,14 +666,73 @@ func (c *DiscordChannel) listenVoiceControl(ctx context.Context) { } func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnection, text string) { - stream, err := c.tts.Synthesize(ctx, text) - if err != nil { - logger.ErrorCF("discord", "TTS synthesize failed", map[string]any{"error": err.Error()}) + sentences := audio.SplitSentences(text) + if len(sentences) == 0 { return } - defer stream.Close() - if err := streamOggOpusToDiscord(vc, stream); err != nil { - logger.ErrorCF("discord", "TTS playback failed", map[string]any{"error": err.Error()}) + logger.InfoCF("discord", "Starting streamed TTS", map[string]any{"sentences": len(sentences)}) + + // Pipeline: prefetch next sentence's audio while playing current + type ttResult struct { + stream io.ReadCloser + err error + } + + var prefetch chan ttResult + + for i, sentence := range sentences { + // Check for cancellation (interruption) + select { + case <-ctx.Done(): + logger.InfoCF("discord", "TTS interrupted", map[string]any{"at_sentence": i}) + return + default: + } + + // Start prefetching the NEXT sentence while we process the current one + var nextPrefetch chan ttResult + if i+1 < len(sentences) { + nextPrefetch = make(chan ttResult, 1) + nextSentence := sentences[i+1] + go func() { + s, e := c.tts.Synthesize(ctx, nextSentence) + nextPrefetch <- ttResult{s, e} + }() + } + + // Get the current sentence's audio + var stream io.ReadCloser + var err error + + if prefetch != nil { + // Use prefetched result from previous iteration + result := <-prefetch + stream, err = result.stream, result.err + } else { + // First sentence: synthesize directly + stream, err = c.tts.Synthesize(ctx, sentence) + } + + if err != nil { + logger.ErrorCF("discord", "TTS synthesize failed", map[string]any{"error": err.Error(), "sentence": i}) + prefetch = nextPrefetch + continue + } + + if err := streamOggOpusToDiscord(ctx, vc, stream); err != nil { + logger.ErrorCF("discord", "TTS playback failed", map[string]any{"error": err.Error(), "sentence": i}) + } + stream.Close() + + prefetch = nextPrefetch + } + + // Drain any leftover prefetch + if prefetch != nil { + result := <-prefetch + if result.stream != nil { + result.stream.Close() + } } } diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index 72f767b85..4c37e952f 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -2,6 +2,7 @@ package discord import ( "bytes" + "context" "fmt" "io" "time" @@ -46,7 +47,7 @@ func VoiceReceiveActive(vc *discordgo.VoiceConnection) bool { return vc != nil && vc.OpusRecv != nil } -func streamOggOpusToDiscord(vc *discordgo.VoiceConnection, r io.Reader) error { +func streamOggOpusToDiscord(ctx context.Context, vc *discordgo.VoiceConnection, r io.Reader) error { // Wait for the speaking transition to register vc.Speaking(true) defer vc.Speaking(false) @@ -55,6 +56,13 @@ func streamOggOpusToDiscord(vc *discordgo.VoiceConnection, r io.Reader) error { header := make([]byte, 27) for { + // Check for interruption + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + if _, err := io.ReadFull(r, header); err != nil { if err == io.EOF || err == io.ErrUnexpectedEOF { return nil @@ -84,8 +92,11 @@ func streamOggOpusToDiscord(vc *discordgo.VoiceConnection, r io.Reader) error { if len(packet) > 0 { // Ignore Ogg Opus headers if !bytes.HasPrefix(packet, []byte("OpusHead")) && !bytes.HasPrefix(packet, []byte("OpusTags")) { - // Pacing is handled natively by vc.OpusSend blocking (it has an internal ticker) - vc.OpusSend <- packet + select { + case <-ctx.Done(): + return ctx.Err() + case vc.OpusSend <- packet: + } } // Start new packet packet = nil @@ -118,6 +129,8 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str }) var sequence uint64 = 0 + var interruptCount int + var lastInterruptAt time.Time for { select { @@ -139,6 +152,26 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str continue } + // Interruption detection: if user sends voice while TTS is playing, + // cancel TTS after a short debounce (3 packets in 200ms) + now := time.Now() + if now.Sub(lastInterruptAt) > 500*time.Millisecond { + interruptCount = 0 + } + interruptCount++ + lastInterruptAt = now + + if interruptCount >= 3 { + c.ttsMu.Lock() + if c.cancelTTS != nil { + c.cancelTTS() + c.cancelTTS = nil + logger.InfoCF("discord", "TTS interrupted by user voice", nil) + } + c.ttsMu.Unlock() + interruptCount = 0 + } + sequence++ chunk := bus.AudioChunk{ From 20f87c65e846e55a72b9873f2fd8c7ead524fff8 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 08:46:51 +0100 Subject: [PATCH 08/51] make fmt --- pkg/asr/asr.go | 16 +++++++++++++--- pkg/channels/discord/voice.go | 1 + pkg/voice/agent.go | 4 +++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/pkg/asr/asr.go b/pkg/asr/asr.go index dc22588f0..d30ea3fef 100644 --- a/pkg/asr/asr.go +++ b/pkg/asr/asr.go @@ -49,7 +49,11 @@ func NewGroqTranscriber(apiKey string) *GroqTranscriber { } } -func (t *GroqTranscriber) TranscribeData(ctx context.Context, data []byte, filename string) (*TranscriptionResponse, error) { +func (t *GroqTranscriber) TranscribeData( + ctx context.Context, + data []byte, + filename string, +) (*TranscriptionResponse, error) { logger.InfoCF("voice", "Starting memory transcription", map[string]any{"filename": filename, "bytes": len(data)}) var requestBody bytes.Buffer @@ -112,7 +116,12 @@ func (t *GroqTranscriber) Transcribe(ctx context.Context, audioFilePath string) return t.doRequest(ctx, &requestBody, writer.FormDataContentType(), fileInfo.Size()) } -func (t *GroqTranscriber) doRequest(ctx context.Context, requestBody *bytes.Buffer, contentType string, fileSize int64) (*TranscriptionResponse, error) { +func (t *GroqTranscriber) doRequest( + ctx context.Context, + requestBody *bytes.Buffer, + contentType string, + fileSize int64, +) (*TranscriptionResponse, error) { url := t.apiBase + "/audio/transcriptions" req, err := http.NewRequestWithContext(ctx, "POST", url, requestBody) if err != nil { @@ -184,7 +193,8 @@ func DetectTranscriber(cfg *config.Config) Transcriber { } // Fall back to any model-list entry that uses the groq/ protocol or is explicitly named groq. for _, mc := range cfg.ModelList { - if (strings.HasPrefix(mc.Model, "groq/") || mc.ModelName == "groq" || mc.Model == "whisper-large-v3-turbo") && mc.APIKey != "" { + if (strings.HasPrefix(mc.Model, "groq/") || mc.ModelName == "groq" || mc.Model == "whisper-large-v3-turbo") && + mc.APIKey != "" { return NewGroqTranscriber(mc.APIKey) } } diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index 4c37e952f..7a6be5e7c 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -8,6 +8,7 @@ import ( "time" "github.com/bwmarrin/discordgo" + "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/logger" ) diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index 6a330d3d4..5ab19bc01 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -11,6 +11,7 @@ import ( "github.com/pion/rtp" "github.com/pion/webrtc/v3/pkg/media/oggwriter" + "github.com/sipeed/picoclaw/pkg/asr" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/logger" @@ -187,7 +188,8 @@ func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) { channelType := "discord" text := strings.ToLower(strings.TrimSpace(res.Text)) - if strings.Contains(text, "leave the voice channel") || strings.Contains(text, "leave voice") || strings.Contains(text, "disconnect voice") { + if strings.Contains(text, "leave the voice channel") || strings.Contains(text, "leave voice") || + strings.Contains(text, "disconnect voice") { logger.InfoCF("voice-agent", "Voice command triggered: leave", nil) a.bus.PublishVoiceControl(ctx, bus.VoiceControl{ SessionID: acc.sessionID, From 192e53cab1662f9a1394d714d9344017231e513c Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 08:59:02 +0100 Subject: [PATCH 09/51] lint fix --- .golangci.yaml | 3 +++ pkg/asr/asr.go | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index ea3107ec8..b2b772406 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -61,6 +61,9 @@ linters: - usestdlibvars - usetesting settings: + gomoddirectives: + replace-allow-list: + - github.com/bwmarrin/discordgo errcheck: check-type-assertions: true check-blank: true diff --git a/pkg/asr/asr.go b/pkg/asr/asr.go index d30ea3fef..67029e755 100644 --- a/pkg/asr/asr.go +++ b/pkg/asr/asr.go @@ -65,9 +65,9 @@ func (t *GroqTranscriber) TranscribeData( return nil, fmt.Errorf("failed to create form file: %w", err) } - if _, err := io.Copy(part, bytes.NewReader(data)); err != nil { - logger.ErrorCF("voice", "Failed to copy file content", map[string]any{"error": err}) - return nil, fmt.Errorf("failed to copy file content: %w", err) + if _, copyErr := io.Copy(part, bytes.NewReader(data)); copyErr != nil { + logger.ErrorCF("voice", "Failed to copy file content", map[string]any{"error": copyErr}) + return nil, fmt.Errorf("failed to copy file content: %w", copyErr) } if err = writer.WriteField("model", "whisper-large-v3-turbo"); err != nil { From 9ebdb0ad57ef9cb05b5f165115f574b6a931435f Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 09:03:46 +0100 Subject: [PATCH 10/51] fix tts panic --- pkg/channels/discord/voice.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index 7a6be5e7c..8c4b84824 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -48,7 +48,14 @@ func VoiceReceiveActive(vc *discordgo.VoiceConnection) bool { return vc != nil && vc.OpusRecv != nil } -func streamOggOpusToDiscord(ctx context.Context, vc *discordgo.VoiceConnection, r io.Reader) error { +func streamOggOpusToDiscord(ctx context.Context, vc *discordgo.VoiceConnection, r io.Reader) (retErr error) { + // Recover from panic if vc.OpusSend is closed mid-send (e.g. on disconnect) + defer func() { + if rec := recover(); rec != nil { + retErr = fmt.Errorf("voice connection closed during playback") + } + }() + // Wait for the speaking transition to register vc.Speaking(true) defer vc.Speaking(false) @@ -140,6 +147,13 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str case p, ok := <-vc.OpusRecv: if !ok { logger.InfoCF("discord", "Voice channel closed", map[string]any{"guild": guildID}) + // Cancel any TTS that may still be playing + c.ttsMu.Lock() + if c.cancelTTS != nil { + c.cancelTTS() + c.cancelTTS = nil + } + c.ttsMu.Unlock() return } From 2d065f9ab335b3903baa67ddc2f4d1bc0834417d Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 09:31:16 +0100 Subject: [PATCH 11/51] Edit from coplilot review --- pkg/bus/types.go | 1 + pkg/channels/discord/discord.go | 7 +++-- pkg/channels/discord/voice.go | 55 ++++----------------------------- pkg/voice/agent.go | 7 ++++- 4 files changed, 18 insertions(+), 52 deletions(-) diff --git a/pkg/bus/types.go b/pkg/bus/types.go index e036b4ede..794db5b0f 100644 --- a/pkg/bus/types.go +++ b/pkg/bus/types.go @@ -57,6 +57,7 @@ type AudioChunk struct { SessionID string `json:"session_id"` SpeakerID string `json:"speaker_id"` // User ID or SSRC ChatID string `json:"chat_id"` // Where to respond + Channel string `json:"channel"` // Source channel type (e.g. "discord") Sequence uint64 `json:"sequence"` Timestamp uint32 `json:"timestamp"` SampleRate int `json:"sample_rate"` diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index c32a6409c..1db63c014 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -162,7 +162,7 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) erro if c.cancelTTS != nil { c.cancelTTS() } - ttsCtx, ttsCancel := context.WithCancel(context.Background()) + ttsCtx, ttsCancel := context.WithCancel(c.ctx) c.cancelTTS = ttsCancel c.ttsMu.Unlock() @@ -651,7 +651,10 @@ func (c *DiscordChannel) listenVoiceControl(ctx context.Context) { select { case <-ctx.Done(): return - case ctrl := <-c.bus.VoiceControlsChan(): + case ctrl, ok := <-c.bus.VoiceControlsChan(): + if !ok { + return + } if ctrl.Type == "command" && ctrl.Action == "leave" { if strings.HasPrefix(ctrl.SessionID, "discord_vc_") { guildID := strings.TrimPrefix(ctrl.SessionID, "discord_vc_") diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index 8c4b84824..8dd1a6b55 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -1,7 +1,6 @@ package discord import ( - "bytes" "context" "fmt" "io" @@ -9,6 +8,7 @@ import ( "github.com/bwmarrin/discordgo" + "github.com/sipeed/picoclaw/pkg/audio" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/logger" ) @@ -60,58 +60,14 @@ func streamOggOpusToDiscord(ctx context.Context, vc *discordgo.VoiceConnection, vc.Speaking(true) defer vc.Speaking(false) - var packet []byte - header := make([]byte, 27) - - for { - // Check for interruption + return audio.DecodeOggOpus(r, func(frame []byte) error { select { case <-ctx.Done(): return ctx.Err() - default: + case vc.OpusSend <- frame: + return nil } - - if _, err := io.ReadFull(r, header); err != nil { - if err == io.EOF || err == io.ErrUnexpectedEOF { - return nil - } - return fmt.Errorf("failed to read ogg header: %w", err) - } - if string(header[:4]) != "OggS" { - return fmt.Errorf("invalid ogg magic string") - } - - pageSegments := int(header[26]) - segmentTable := make([]byte, pageSegments) - if _, err := io.ReadFull(r, segmentTable); err != nil { - return fmt.Errorf("failed to read segment table: %w", err) - } - - for _, lacing := range segmentTable { - segment := make([]byte, lacing) - if _, err := io.ReadFull(r, segment); err != nil { - return fmt.Errorf("failed to read segment data: %w", err) - } - - packet = append(packet, segment...) - - // If lacing is less than 255, the packet is complete - if lacing < 255 { - if len(packet) > 0 { - // Ignore Ogg Opus headers - if !bytes.HasPrefix(packet, []byte("OpusHead")) && !bytes.HasPrefix(packet, []byte("OpusTags")) { - select { - case <-ctx.Done(): - return ctx.Err() - case vc.OpusSend <- packet: - } - } - // Start new packet - packet = nil - } - } - } - } + }) } func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID string, chatID string) { @@ -193,6 +149,7 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str SessionID: sessionID, SpeakerID: fmt.Sprintf("%d", p.SSRC), ChatID: chatID, + Channel: "discord", Sequence: sequence, Timestamp: p.Timestamp, SampleRate: 48000, diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index 5ab19bc01..ceb14540e 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -26,6 +26,7 @@ type speechAccumulator struct { chatID string speakerID string sessionID string + channel string } func (a *speechAccumulator) Push(chunk bus.AudioChunk) { @@ -117,6 +118,7 @@ func (a *Agent) handleChunk(chunk bus.AudioChunk) { chatID: chunk.ChatID, speakerID: chunk.SpeakerID, sessionID: chunk.SessionID, + channel: chunk.Channel, } a.sessions[key] = acc logger.DebugCF("voice-agent", "Started accumulating voice", map[string]any{"key": key, "file": filename}) @@ -185,7 +187,10 @@ func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) { logger.InfoCF("voice-agent", "Transcription result", map[string]any{"text": res.Text, "duration": res.Duration}) - channelType := "discord" + channelType := acc.channel + if channelType == "" { + channelType = "discord" // fallback for legacy chunks + } text := strings.ToLower(strings.TrimSpace(res.Text)) if strings.Contains(text, "leave the voice channel") || strings.Contains(text, "leave voice") || From 665ccf125945759963b510dfda940aa5f9799f85 Mon Sep 17 00:00:00 2001 From: Hua Audio Date: Sat, 21 Mar 2026 09:33:15 +0100 Subject: [PATCH 12/51] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/channels/discord/voice.go | 30 ++++++++++++++++++++++++------ pkg/voice/agent.go | 5 ++++- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index 8dd1a6b55..d5d5d303f 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -113,16 +113,24 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str return } + if p == nil { + logger.DebugCF("discord", "Received nil Opus packet", nil) + continue + } + + if len(p.Opus) == 0 { + logger.DebugCF("discord", "Received empty Opus packet", map[string]any{ + "seq": p.Sequence, + "ssrc": p.SSRC, + }) + continue + } + logger.DebugCF("discord", "Received Opus packet", map[string]any{ "seq": p.Sequence, "len": len(p.Opus), "ssrc": p.SSRC, }) - - if p == nil || len(p.Opus) == 0 { - continue - } - // Interruption detection: if user sends voice while TTS is playing, // cancel TTS after a short debounce (3 packets in 200ms) now := time.Now() @@ -158,7 +166,17 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str Data: p.Opus, } - c.bus.PublishAudioChunk(c.ctx, chunk) + ctx, cancel := context.WithTimeout(c.ctx, 100*time.Millisecond) + err := c.bus.PublishAudioChunk(ctx, chunk) + cancel() + if err != nil { + logger.ErrorCF("discord", "Failed to publish audio chunk", map[string]any{ + "guild": guildID, + "sessionID": sessionID, + "sequence": sequence, + "error": err.Error(), + }) + } } } } diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index ceb14540e..8e978487a 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -90,7 +90,10 @@ func (a *Agent) listenChunks(ctx context.Context) { select { case <-ctx.Done(): return - case chunk := <-chunks: + case chunk, ok := <-chunks: + if !ok { + return + } a.handleChunk(chunk) } } From aa6fdff11ebb4f674e3f4d2137518a7de1a2a41d Mon Sep 17 00:00:00 2001 From: Hua Audio Date: Sat, 21 Mar 2026 09:42:09 +0100 Subject: [PATCH 13/51] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/bus/types.go | 2 +- pkg/channels/discord/voice.go | 39 +++++++++++++++++++++++++++---- pkg/tts/tts.go | 44 +++++++++++++++++++++++++++++++---- 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/pkg/bus/types.go b/pkg/bus/types.go index 794db5b0f..c648fb0ce 100644 --- a/pkg/bus/types.go +++ b/pkg/bus/types.go @@ -70,5 +70,5 @@ type AudioChunk struct { type VoiceControl struct { SessionID string `json:"session_id"` Type string `json:"type"` // "state", "command" - Action string `json:"action"` // "idle", "listening", "start", "stop" + Action string `json:"action"` // "idle", "listening", "start", "stop", "leave" } diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index d5d5d303f..37c882b25 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -73,17 +73,46 @@ func streamOggOpusToDiscord(ctx context.Context, vc *discordgo.VoiceConnection, func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID string, chatID string) { logger.InfoCF("discord", "Started listening for voice", map[string]any{"guild": guildID}) - go func() { + go func(ctx context.Context, vc *discordgo.VoiceConnection) { + // Recover from potential panics if OpusSend is closed mid-send. + defer func() { + if rec := recover(); rec != nil { + logger.WarnCF("discord", "Recovered from panic while sending wake-up frames", map[string]any{ + "error": rec, + "guild": guildID, + }) + } + }() + + // If the voice connection or OpusSend are not available, nothing to do. + if vc == nil || vc.OpusSend == nil { + return + } + time.Sleep(250 * time.Millisecond) // Wait a bit for connection to settle + + // Abort if the context has already been cancelled. + select { + case <-ctx.Done(): + return + default: + } + vc.Speaking(true) + defer vc.Speaking(false) + + silenceFrame := []byte{0xF8, 0xFF, 0xFE} for i := 0; i < 5; i++ { - vc.OpusSend <- []byte{0xF8, 0xFF, 0xFE} + select { + case <-ctx.Done(): + return + case vc.OpusSend <- silenceFrame: + } time.Sleep(20 * time.Millisecond) } - vc.Speaking(false) - logger.DebugCF("discord", "Sent wake-up silence frames", nil) - }() + logger.DebugCF("discord", "Sent wake-up silence frames", map[string]any{"guild": guildID}) + }(c.ctx, vc) sessionID := fmt.Sprintf("discord_vc_%s", guildID) c.bus.PublishVoiceControl(c.ctx, bus.VoiceControl{ diff --git a/pkg/tts/tts.go b/pkg/tts/tts.go index 63b4ecd24..25b533326 100644 --- a/pkg/tts/tts.go +++ b/pkg/tts/tts.go @@ -29,11 +29,47 @@ type OpenAITTSProvider struct { } func NewOpenAITTSProvider(apiKey string, apiBase string, proxyURL string) *OpenAITTSProvider { - if apiBase == "" || apiBase == "https://api.openai.com/v1" { + // Normalize apiBase to avoid malformed endpoints like + // "https://api.openai.com/audio/speech" when "/v1" is required. + if apiBase == "" { apiBase = "https://api.openai.com/v1/audio/speech" - } else if !strings.HasSuffix(apiBase, "/audio/speech") { - // Just in case they provide openrouter base or standard base - apiBase = strings.TrimSuffix(apiBase, "/") + "/audio/speech" + } else { + if u, err := url.Parse(apiBase); err == nil && u.Scheme != "" && u.Host != "" { + path := u.Path + if u.Host == "api.openai.com" { + // For the official OpenAI host, ensure exactly one /v1 prefix and + // that the path ends with /audio/speech. + if path == "" || path == "/" || path == "/v1" { + path = "/v1/audio/speech" + } else { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + if !strings.HasPrefix(path, "/v1/") { + path = "/v1" + strings.TrimSuffix(path, "/") + } + if !strings.HasSuffix(path, "/audio/speech") { + path = strings.TrimSuffix(path, "/") + "/audio/speech" + } + } + } else { + // For non-OpenAI hosts (e.g., proxies), preserve the existing base + // path and only ensure it ends with /audio/speech. + if !strings.HasSuffix(path, "/audio/speech") { + path = strings.TrimSuffix(path, "/") + "/audio/speech" + } + } + u.Path = path + apiBase = u.String() + } else { + // Fallback to the previous string-based behavior if parsing fails. + if apiBase == "https://api.openai.com/v1" { + apiBase = "https://api.openai.com/v1/audio/speech" + } else if !strings.HasSuffix(apiBase, "/audio/speech") { + // Just in case they provide openrouter base or standard base + apiBase = strings.TrimSuffix(apiBase, "/") + "/audio/speech" + } + } } client := &http.Client{ From f8b9b299d9adce8d8cd9bf037caae75e93d3084d Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 09:54:45 +0100 Subject: [PATCH 14/51] Code review --- pkg/asr/asr.go | 27 ++++++++++++++++------ pkg/channels/discord/discord.go | 25 ++++++++++++++------- pkg/channels/discord/voice.go | 2 +- pkg/voice/agent.go | 40 +++++++++++++++++++++++++++------ 4 files changed, 71 insertions(+), 23 deletions(-) diff --git a/pkg/asr/asr.go b/pkg/asr/asr.go index 67029e755..fe2e3e11b 100644 --- a/pkg/asr/asr.go +++ b/pkg/asr/asr.go @@ -93,25 +93,38 @@ func (t *GroqTranscriber) Transcribe(ctx context.Context, audioFilePath string) audioFile, err := os.Open(audioFilePath) if err != nil { - return nil, fmt.Errorf("failed to open audio file: %w", err) + return nil, fmt.Errorf("failed to open audio file %s: %w", audioFilePath, err) } defer audioFile.Close() fileInfo, err := audioFile.Stat() if err != nil { - return nil, err + return nil, fmt.Errorf("failed to stat audio file %s: %w", audioFilePath, err) } var requestBody bytes.Buffer writer := multipart.NewWriter(&requestBody) + part, err := writer.CreateFormFile("file", filepath.Base(audioFilePath)) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create form file: %w", err) + } + + if _, copyErr := io.Copy(part, audioFile); copyErr != nil { + return nil, fmt.Errorf("failed to copy audio data: %w", copyErr) + } + + if err = writer.WriteField("model", "whisper-large-v3-turbo"); err != nil { + return nil, fmt.Errorf("failed to write model field: %w", err) + } + + if err = writer.WriteField("response_format", "json"); err != nil { + return nil, fmt.Errorf("failed to write response_format field: %w", err) + } + + if err = writer.Close(); err != nil { + return nil, fmt.Errorf("failed to close multipart writer: %w", err) } - io.Copy(part, audioFile) - writer.WriteField("model", "whisper-large-v3") - writer.WriteField("response_format", "json") - writer.Close() return t.doRequest(ctx, &requestBody, writer.FormDataContentType(), fileInfo.Size()) } diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index 1db63c014..2c7407171 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -669,6 +669,13 @@ func (c *DiscordChannel) listenVoiceControl(ctx context.Context) { } func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnection, text string) { + // Clear cancelTTS when playback finishes (normal or interrupted) + defer func() { + c.ttsMu.Lock() + c.cancelTTS = nil + c.ttsMu.Unlock() + }() + sentences := audio.SplitSentences(text) if len(sentences) == 0 { return @@ -684,6 +691,16 @@ func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnect var prefetch chan ttResult + // Ensure any in-flight prefetch is drained on exit to prevent stream leaks + defer func() { + if prefetch != nil { + result := <-prefetch + if result.stream != nil { + result.stream.Close() + } + } + }() + for i, sentence := range sentences { // Check for cancellation (interruption) select { @@ -730,12 +747,4 @@ func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnect prefetch = nextPrefetch } - - // Drain any leftover prefetch - if prefetch != nil { - result := <-prefetch - if result.stream != nil { - result.stream.Close() - } - } } diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index 37c882b25..233cfcccb 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -91,7 +91,7 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str time.Sleep(250 * time.Millisecond) // Wait a bit for connection to settle - // Abort if the context has already been cancelled. + // Abort if the context has already been canceled. select { case <-ctx.Done(): return diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index 8e978487a..0885e6b2e 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -82,6 +82,19 @@ func (a *Agent) Start(ctx context.Context) { logger.InfoCF("voice-agent", "Started Voice Agent orchestrator", nil) go a.listenChunks(ctx) go a.vadTick(ctx) + + // Cleanup sessions on shutdown + go func() { + <-ctx.Done() + a.mu.Lock() + for key, acc := range a.sessions { + acc.Close() + os.Remove(acc.file) + delete(a.sessions, key) + } + a.mu.Unlock() + logger.InfoCF("voice-agent", "Cleaned up voice sessions on shutdown", nil) + }() } func (a *Agent) listenChunks(ctx context.Context) { @@ -100,6 +113,12 @@ func (a *Agent) listenChunks(ctx context.Context) { } func (a *Agent) handleChunk(chunk bus.AudioChunk) { + // Only accept Opus-encoded audio + if chunk.Format != "opus" { + logger.DebugCF("voice-agent", "Ignoring unsupported audio format", map[string]any{"format": chunk.Format}) + return + } + a.mu.Lock() defer a.mu.Unlock() @@ -197,24 +216,29 @@ func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) { text := strings.ToLower(strings.TrimSpace(res.Text)) if strings.Contains(text, "leave the voice channel") || strings.Contains(text, "leave voice") || - strings.Contains(text, "disconnect voice") { + strings.Contains(text, "disconnect voice") || strings.Contains(text, "leave the channel") || + strings.Contains(text, "leave channel") { logger.InfoCF("voice-agent", "Voice command triggered: leave", nil) - a.bus.PublishVoiceControl(ctx, bus.VoiceControl{ + if err := a.bus.PublishVoiceControl(ctx, bus.VoiceControl{ SessionID: acc.sessionID, Type: "command", Action: "leave", - }) - a.bus.PublishOutbound(ctx, bus.OutboundMessage{ + }); err != nil { + logger.ErrorCF("voice-agent", "Failed to publish leave control", map[string]any{"error": err}) + } + if err := a.bus.PublishOutbound(ctx, bus.OutboundMessage{ Channel: channelType, ChatID: acc.chatID, Content: "Goodbye! Leaving the voice channel.", - }) + }); err != nil { + logger.ErrorCF("voice-agent", "Failed to publish goodbye message", map[string]any{"error": err}) + } return } oralPrompt := "\n\n[SYSTEM]: The user just spoke this to you over voice chat. Please reply in a highly concise, conversational, oral style suitable for text-to-speech. Do not use markdown, emojis, asterisks, or code blocks. Speak naturally." - a.bus.PublishInbound(ctx, bus.InboundMessage{ + if err := a.bus.PublishInbound(ctx, bus.InboundMessage{ Channel: channelType, SenderID: acc.speakerID, ChatID: acc.chatID, @@ -223,5 +247,7 @@ func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) { Metadata: map[string]string{ "is_voice": "true", }, - }) + }); err != nil { + logger.ErrorCF("voice-agent", "Failed to publish inbound message", map[string]any{"error": err}) + } } From 21c6e7808cb0de403e77000d3f9a559f1c4414ca Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 18:31:15 +0100 Subject: [PATCH 15/51] fix tool_call tts --- pkg/agent/loop.go | 9 +++++++++ pkg/bus/types.go | 9 +++++---- pkg/channels/discord/discord.go | 9 ++++++++- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 54a2bfb96..4c44f2ea5 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -1379,6 +1379,9 @@ func (al *AgentLoop) runLLMIteration( Channel: opts.Channel, ChatID: opts.ChatID, Content: feedbackMsg, + Metadata: map[string]string{ + "is_tool_call": "true", + }, }) fbCancel() } @@ -1397,6 +1400,9 @@ func (al *AgentLoop) runLLMIteration( Channel: opts.Channel, ChatID: opts.ChatID, Content: result.ForUser, + Metadata: map[string]string{ + "is_tool_call": "true", + }, }) } @@ -1447,6 +1453,9 @@ func (al *AgentLoop) runLLMIteration( Channel: opts.Channel, ChatID: opts.ChatID, Content: r.result.ForUser, + Metadata: map[string]string{ + "is_tool_call": "true", + }, }) logger.DebugCF("agent", "Sent tool result to user", map[string]any{ diff --git a/pkg/bus/types.go b/pkg/bus/types.go index c648fb0ce..15aebc345 100644 --- a/pkg/bus/types.go +++ b/pkg/bus/types.go @@ -30,10 +30,11 @@ type InboundMessage struct { } type OutboundMessage struct { - Channel string `json:"channel"` - ChatID string `json:"chat_id"` - Content string `json:"content"` - ReplyToMessageID string `json:"reply_to_message_id,omitempty"` + Channel string `json:"channel"` + ChatID string `json:"chat_id"` + Content string `json:"content"` + ReplyToMessageID string `json:"reply_to_message_id,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` } // MediaPart describes a single media attachment to send. diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index 2c7407171..cff8b7a16 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -154,7 +154,14 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) erro return nil } - if c.tts != nil { + isToolCall := false + if msg.Metadata != nil { + if val, ok := msg.Metadata["is_tool_call"]; ok && val == "true" { + isToolCall = true + } + } + + if c.tts != nil && !isToolCall { if ch, err := c.session.State.Channel(channelID); err == nil && ch.GuildID != "" { if vc, ok := c.session.VoiceConnections[ch.GuildID]; ok && vc != nil { // Cancel any previous TTS playback From 87281a6cce26711070c75ebd2ab899d17b5913c7 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 03:35:03 +0100 Subject: [PATCH 16/51] fix asr --- pkg/{voice => asr}/audio_model_transcriber.go | 2 +- pkg/{voice => asr}/audio_model_transcriber_test.go | 2 +- pkg/{voice => asr}/groq_transcriber.go | 2 +- pkg/{voice => asr}/groq_transcriber_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename pkg/{voice => asr}/audio_model_transcriber.go (99%) rename pkg/{voice => asr}/audio_model_transcriber_test.go (99%) rename pkg/{voice => asr}/groq_transcriber.go (99%) rename pkg/{voice => asr}/groq_transcriber_test.go (99%) diff --git a/pkg/voice/audio_model_transcriber.go b/pkg/asr/audio_model_transcriber.go similarity index 99% rename from pkg/voice/audio_model_transcriber.go rename to pkg/asr/audio_model_transcriber.go index 94486b5e4..35f1c2667 100644 --- a/pkg/voice/audio_model_transcriber.go +++ b/pkg/asr/audio_model_transcriber.go @@ -1,4 +1,4 @@ -package voice +package asr import ( "context" diff --git a/pkg/voice/audio_model_transcriber_test.go b/pkg/asr/audio_model_transcriber_test.go similarity index 99% rename from pkg/voice/audio_model_transcriber_test.go rename to pkg/asr/audio_model_transcriber_test.go index c33e3bf97..5aaa82061 100644 --- a/pkg/voice/audio_model_transcriber_test.go +++ b/pkg/asr/audio_model_transcriber_test.go @@ -1,4 +1,4 @@ -package voice +package asr import ( "context" diff --git a/pkg/voice/groq_transcriber.go b/pkg/asr/groq_transcriber.go similarity index 99% rename from pkg/voice/groq_transcriber.go rename to pkg/asr/groq_transcriber.go index b42e598f7..ca6a5eb5b 100644 --- a/pkg/voice/groq_transcriber.go +++ b/pkg/asr/groq_transcriber.go @@ -1,4 +1,4 @@ -package voice +package asr import ( "bytes" diff --git a/pkg/voice/groq_transcriber_test.go b/pkg/asr/groq_transcriber_test.go similarity index 99% rename from pkg/voice/groq_transcriber_test.go rename to pkg/asr/groq_transcriber_test.go index fdcaa7580..b05700d80 100644 --- a/pkg/voice/groq_transcriber_test.go +++ b/pkg/asr/groq_transcriber_test.go @@ -1,4 +1,4 @@ -package voice +package asr import ( "context" From bafbf699bdf4c1eef153517b955d84792dc5f216 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 04:02:55 +0100 Subject: [PATCH 17/51] resolve conflicts --- pkg/agent/loop.go | 206 +++++++++++++++++++++++++++++++--------------- 1 file changed, 138 insertions(+), 68 deletions(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index f8077e0e5..2aa1775b9 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -18,7 +18,6 @@ import ( "sync/atomic" "time" - "github.com/sipeed/picoclaw/pkg/asr" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/commands" @@ -32,6 +31,7 @@ import ( "github.com/sipeed/picoclaw/pkg/state" "github.com/sipeed/picoclaw/pkg/tools" "github.com/sipeed/picoclaw/pkg/utils" + "github.com/sipeed/picoclaw/pkg/asr" ) type AgentLoop struct { @@ -2159,73 +2159,146 @@ turnLoop: }) } messages = append(messages, assistantMsg) - - // Save assistant message with tool calls to session - agent.Sessions.AddFullMessage(opts.SessionKey, assistantMsg) - - // Execute tool calls in parallel - type indexedAgentResult struct { - result *tools.ToolResult - tc providers.ToolCall + if !ts.opts.NoHistory { + ts.agent.Sessions.AddFullMessage(ts.sessionKey, assistantMsg) + ts.recordPersistedMessage(assistantMsg) } - agentResults := make([]indexedAgentResult, len(normalizedToolCalls)) - var wg sync.WaitGroup - + ts.setPhase(TurnPhaseTools) for i, tc := range normalizedToolCalls { - agentResults[i].tc = tc + if ts.hardAbortRequested() { + turnStatus = TurnEndStatusAborted + return al.abortTurn(ts) + } - wg.Add(1) - go func(idx int, tc providers.ToolCall) { - defer wg.Done() + toolName := tc.Name + toolArgs := cloneStringAnyMap(tc.Arguments) - argsJSON, _ := json.Marshal(tc.Arguments) - argsPreview := utils.Truncate(string(argsJSON), 200) - logger.InfoCF("agent", fmt.Sprintf("Tool call: %s(%s)", tc.Name, argsPreview), - map[string]any{ - "agent_id": agent.ID, - "tool": tc.Name, - "iteration": iteration, - }) - - // Send tool feedback to chat channel if enabled - if al.cfg.Agents.Defaults.IsToolFeedbackEnabled() && opts.Channel != "" { - feedbackPreview := utils.Truncate( - string(argsJSON), - al.cfg.Agents.Defaults.GetToolFeedbackMaxArgsLength(), - ) - feedbackMsg := fmt.Sprintf("\U0001f527 `%s`\n```\n%s\n```", tc.Name, feedbackPreview) - fbCtx, fbCancel := context.WithTimeout(ctx, 3*time.Second) - _ = al.bus.PublishOutbound(fbCtx, bus.OutboundMessage{ - Channel: opts.Channel, - ChatID: opts.ChatID, - Content: feedbackMsg, - Metadata: map[string]string{ - "is_tool_call": "true", - }, - }) - fbCancel() - } - - // Create async callback for tools that implement AsyncExecutor. - // When the background work completes, this publishes the result - // as an inbound system message so processSystemMessage routes it - // back to the user via the normal agent loop. - asyncCallback := func(_ context.Context, result *tools.ToolResult) { - // Send ForUser content directly to the user (immediate feedback), - // mirroring the synchronous tool execution path. - if !result.Silent && result.ForUser != "" { - outCtx, outCancel := context.WithTimeout(context.Background(), 5*time.Second) - defer outCancel() - _ = al.bus.PublishOutbound(outCtx, bus.OutboundMessage{ - Channel: opts.Channel, - ChatID: opts.ChatID, - Content: result.ForUser, - Metadata: map[string]string{ - "is_tool_call": "true", - }, - }) + if al.hooks != nil { + toolReq, decision := al.hooks.BeforeTool(turnCtx, &ToolCallHookRequest{ + Meta: ts.eventMeta("runTurn", "turn.tool.before"), + Tool: toolName, + Arguments: toolArgs, + Channel: ts.channel, + ChatID: ts.chatID, + }) + switch decision.normalizedAction() { + case HookActionContinue, HookActionModify: + if toolReq != nil { + toolName = toolReq.Tool + toolArgs = toolReq.Arguments } + case HookActionDenyTool: + denyContent := hookDeniedToolContent("Tool execution denied by hook", decision.Reason) + al.emitEvent( + EventKindToolExecSkipped, + ts.eventMeta("runTurn", "turn.tool.skipped"), + ToolExecSkippedPayload{ + Tool: toolName, + Reason: denyContent, + }, + ) + deniedMsg := providers.Message{ + Role: "tool", + Content: denyContent, + ToolCallID: tc.ID, + } + messages = append(messages, deniedMsg) + if !ts.opts.NoHistory { + ts.agent.Sessions.AddFullMessage(ts.sessionKey, deniedMsg) + ts.recordPersistedMessage(deniedMsg) + } + continue + case HookActionAbortTurn: + turnStatus = TurnEndStatusError + return turnResult{}, al.hookAbortError(ts, "before_tool", decision) + case HookActionHardAbort: + _ = ts.requestHardAbort() + turnStatus = TurnEndStatusAborted + return al.abortTurn(ts) + } + } + + if al.hooks != nil { + approval := al.hooks.ApproveTool(turnCtx, &ToolApprovalRequest{ + Meta: ts.eventMeta("runTurn", "turn.tool.approve"), + Tool: toolName, + Arguments: toolArgs, + Channel: ts.channel, + ChatID: ts.chatID, + }) + if !approval.Approved { + denyContent := hookDeniedToolContent("Tool execution denied by approval hook", approval.Reason) + al.emitEvent( + EventKindToolExecSkipped, + ts.eventMeta("runTurn", "turn.tool.skipped"), + ToolExecSkippedPayload{ + Tool: toolName, + Reason: denyContent, + }, + ) + deniedMsg := providers.Message{ + Role: "tool", + Content: denyContent, + ToolCallID: tc.ID, + } + messages = append(messages, deniedMsg) + if !ts.opts.NoHistory { + ts.agent.Sessions.AddFullMessage(ts.sessionKey, deniedMsg) + ts.recordPersistedMessage(deniedMsg) + } + continue + } + } + + argsJSON, _ := json.Marshal(toolArgs) + argsPreview := utils.Truncate(string(argsJSON), 200) + logger.InfoCF("agent", fmt.Sprintf("Tool call: %s(%s)", toolName, argsPreview), + map[string]any{ + "agent_id": ts.agent.ID, + "tool": toolName, + "iteration": iteration, + }) + al.emitEvent( + EventKindToolExecStart, + ts.eventMeta("runTurn", "turn.tool.start"), + ToolExecStartPayload{ + Tool: toolName, + Arguments: cloneEventArguments(toolArgs), + }, + ) + + // Send tool feedback to chat channel if enabled (from HEAD) + if al.cfg.Agents.Defaults.IsToolFeedbackEnabled() && ts.channel != "" { + feedbackPreview := utils.Truncate( + string(argsJSON), + al.cfg.Agents.Defaults.GetToolFeedbackMaxArgsLength(), + ) + feedbackMsg := fmt.Sprintf("\U0001f527 `%s`\n```\n%s\n```", tc.Name, feedbackPreview) + fbCtx, fbCancel := context.WithTimeout(turnCtx, 3*time.Second) + _ = al.bus.PublishOutbound(fbCtx, bus.OutboundMessage{ + Channel: ts.channel, + ChatID: ts.chatID, + Content: feedbackMsg, + }) + fbCancel() + } + + toolCallID := tc.ID + toolIteration := iteration + asyncToolName := toolName + asyncCallback := func(_ context.Context, result *tools.ToolResult) { + // Send ForUser content directly to the user (immediate feedback), + // mirroring the synchronous tool execution path. + if !result.Silent && result.ForUser != "" { + outCtx, outCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer outCancel() + _ = al.bus.PublishOutbound(outCtx, bus.OutboundMessage{ + Channel: ts.channel, + ChatID: ts.chatID, + Content: result.ForUser, + }) + } // Determine content for the agent loop (ForLLM or error). content := result.ForLLM @@ -2315,12 +2388,9 @@ turnLoop: if !toolResult.Silent && toolResult.ForUser != "" && ts.opts.SendResponse { al.bus.PublishOutbound(ctx, bus.OutboundMessage{ - Channel: opts.Channel, - ChatID: opts.ChatID, - Content: r.result.ForUser, - Metadata: map[string]string{ - "is_tool_call": "true", - }, + Channel: ts.channel, + ChatID: ts.chatID, + Content: toolResult.ForUser, }) logger.DebugCF("agent", "Sent tool result to user", map[string]any{ From 47c5956e249ac62f036aad34906fe1cf51ac371a Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 04:10:07 +0100 Subject: [PATCH 18/51] fix lint --- pkg/agent/loop.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 2aa1775b9..ed1dea65b 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -18,6 +18,7 @@ import ( "sync/atomic" "time" + "github.com/sipeed/picoclaw/pkg/asr" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/commands" @@ -31,7 +32,6 @@ import ( "github.com/sipeed/picoclaw/pkg/state" "github.com/sipeed/picoclaw/pkg/tools" "github.com/sipeed/picoclaw/pkg/utils" - "github.com/sipeed/picoclaw/pkg/asr" ) type AgentLoop struct { From f237aba2a30eb77d50aec0450fed35fffb7ec70e Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 04:57:31 +0100 Subject: [PATCH 19/51] update voice system prompt override --- pkg/agent/loop.go | 81 +++++++++++++++++++++++++++++++++------------ pkg/utils/string.go | 5 +++ 2 files changed, 64 insertions(+), 22 deletions(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index ed1dea65b..f8f010111 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -79,6 +79,7 @@ type processOptions struct { UserMessage string // User message content (may include prefix) SystemPromptOverride string // Override the default system prompt (Used by SubTurns) Media []string // media:// refs from inbound message + IsVoice bool // True if this message comes from an audio/voice call InitialSteeringMessages []providers.Message // Steering messages from refactor/agent DefaultResponse string // Response when LLM returns empty EnableSummary bool // Whether to trigger summarization @@ -1295,17 +1296,25 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage) "route_channel": route.Channel, }) + isVoice := msg.Metadata != nil && msg.Metadata["is_voice"] == "true" + var systemPromptOverride string + if isVoice { + systemPromptOverride = "You are a helpful AI assistant. The user is speaking to you over voice chat. Reply in a concise, conversational, and natural oral style suitable for text-to-speech. Get straight to the point. Do not use Markdown, emojis, asterisks, or code blocks." + } + opts := processOptions{ - SessionKey: sessionKey, - Channel: msg.Channel, - ChatID: msg.ChatID, - SenderID: msg.SenderID, - SenderDisplayName: msg.Sender.DisplayName, - UserMessage: msg.Content, - Media: msg.Media, - DefaultResponse: defaultResponse, - EnableSummary: true, - SendResponse: false, + SessionKey: sessionKey, + Channel: msg.Channel, + ChatID: msg.ChatID, + SenderID: msg.SenderID, + SenderDisplayName: msg.Sender.DisplayName, + UserMessage: msg.Content, + SystemPromptOverride: systemPromptOverride, + Media: msg.Media, + IsVoice: isVoice, + DefaultResponse: defaultResponse, + EnableSummary: true, + SendResponse: false, } // context-dependent commands check their own Runtime fields and report @@ -1598,23 +1607,48 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState) (turnResult, er } ts.captureRestorePoint(history, summary) - messages := ts.agent.ContextBuilder.BuildMessages( - history, - summary, - ts.userMessage, - ts.media, - ts.channel, - ts.chatID, - ts.opts.SenderID, - ts.opts.SenderDisplayName, - ) + var messages []providers.Message + if ts.opts.SystemPromptOverride != "" { + messages = append(messages, providers.Message{ + Role: "system", + Content: ts.opts.SystemPromptOverride, + }) + messages = append(messages, history...) + if summary != "" { + messages = append(messages, providers.Message{ + Role: "system", + Content: "CONTEXT_SUMMARY: " + summary, + }) + } + if ts.userMessage != "" || len(ts.media) > 0 { + messages = append(messages, providers.Message{ + Role: "user", + Content: ts.userMessage, + Media: ts.media, + }) + } + } else { + messages = ts.agent.ContextBuilder.BuildMessages( + history, + summary, + ts.userMessage, + ts.media, + ts.channel, + ts.chatID, + ts.opts.SenderID, + ts.opts.SenderDisplayName, + ) + } cfg := al.GetConfig() maxMediaSize := cfg.Agents.Defaults.GetMaxMediaSize() messages = resolveMediaRefs(messages, al.mediaStore, maxMediaSize) if !ts.opts.NoHistory { - toolDefs := ts.agent.Tools.ToProviderDefs() + var toolDefs []providers.ToolDefinition + if !ts.opts.IsVoice { + toolDefs = ts.agent.Tools.ToProviderDefs() + } if isOverContextBudget(ts.agent.ContextWindow, messages, toolDefs, ts.agent.MaxTokens) { logger.WarnCF("agent", "Proactive compression: context budget exceeded before LLM call", map[string]any{"session_key": ts.sessionKey}) @@ -1752,7 +1786,10 @@ turnLoop: }) gracefulTerminal, _ := ts.gracefulInterruptRequested() - providerToolDefs := ts.agent.Tools.ToProviderDefs() + var providerToolDefs []providers.ToolDefinition + if !ts.opts.IsVoice { + providerToolDefs = ts.agent.Tools.ToProviderDefs() + } // Native web search support (from HEAD) _, hasWebSearch := ts.agent.Tools.Get("web_search") diff --git a/pkg/utils/string.go b/pkg/utils/string.go index dbaafdb7f..99337b557 100644 --- a/pkg/utils/string.go +++ b/pkg/utils/string.go @@ -65,3 +65,8 @@ func DerefStr(s *string, fallback string) string { } return *s } + +// IsTruncationDisabled returns whether truncation is disabled globally +func IsTruncationDisabled() bool { +return disableTruncation.Load() +} From c4827bb79bab5e1fe54af2a697a1a7aaf623fda9 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 05:11:59 +0100 Subject: [PATCH 20/51] update voice history cut --- pkg/agent/loop.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index f8f010111..1b890b33b 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -1613,7 +1613,13 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState) (turnResult, er Role: "system", Content: ts.opts.SystemPromptOverride, }) - messages = append(messages, history...) + + if ts.opts.IsVoice && len(history) > 2 { + messages = append(messages, history[len(history)-2:]...) + } else { + messages = append(messages, history...) + } + if summary != "" { messages = append(messages, providers.Message{ Role: "system", From 243cad640c2399e48b7d65ee1428d1c18d7eee20 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 13:34:56 +0100 Subject: [PATCH 21/51] Revert "update voice system prompt override" This reverts commit f237aba2a30eb77d50aec0450fed35fffb7ec70e. --- pkg/agent/loop.go | 41 +++++++++++++---------------------------- pkg/utils/string.go | 5 ----- 2 files changed, 13 insertions(+), 33 deletions(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 1b890b33b..82b20e3d1 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -79,7 +79,6 @@ type processOptions struct { UserMessage string // User message content (may include prefix) SystemPromptOverride string // Override the default system prompt (Used by SubTurns) Media []string // media:// refs from inbound message - IsVoice bool // True if this message comes from an audio/voice call InitialSteeringMessages []providers.Message // Steering messages from refactor/agent DefaultResponse string // Response when LLM returns empty EnableSummary bool // Whether to trigger summarization @@ -1296,25 +1295,17 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage) "route_channel": route.Channel, }) - isVoice := msg.Metadata != nil && msg.Metadata["is_voice"] == "true" - var systemPromptOverride string - if isVoice { - systemPromptOverride = "You are a helpful AI assistant. The user is speaking to you over voice chat. Reply in a concise, conversational, and natural oral style suitable for text-to-speech. Get straight to the point. Do not use Markdown, emojis, asterisks, or code blocks." - } - opts := processOptions{ - SessionKey: sessionKey, - Channel: msg.Channel, - ChatID: msg.ChatID, - SenderID: msg.SenderID, - SenderDisplayName: msg.Sender.DisplayName, - UserMessage: msg.Content, - SystemPromptOverride: systemPromptOverride, - Media: msg.Media, - IsVoice: isVoice, - DefaultResponse: defaultResponse, - EnableSummary: true, - SendResponse: false, + SessionKey: sessionKey, + Channel: msg.Channel, + ChatID: msg.ChatID, + SenderID: msg.SenderID, + SenderDisplayName: msg.Sender.DisplayName, + UserMessage: msg.Content, + Media: msg.Media, + DefaultResponse: defaultResponse, + EnableSummary: true, + SendResponse: false, } // context-dependent commands check their own Runtime fields and report @@ -1613,7 +1604,7 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState) (turnResult, er Role: "system", Content: ts.opts.SystemPromptOverride, }) - + if ts.opts.IsVoice && len(history) > 2 { messages = append(messages, history[len(history)-2:]...) } else { @@ -1651,10 +1642,7 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState) (turnResult, er messages = resolveMediaRefs(messages, al.mediaStore, maxMediaSize) if !ts.opts.NoHistory { - var toolDefs []providers.ToolDefinition - if !ts.opts.IsVoice { - toolDefs = ts.agent.Tools.ToProviderDefs() - } + toolDefs := ts.agent.Tools.ToProviderDefs() if isOverContextBudget(ts.agent.ContextWindow, messages, toolDefs, ts.agent.MaxTokens) { logger.WarnCF("agent", "Proactive compression: context budget exceeded before LLM call", map[string]any{"session_key": ts.sessionKey}) @@ -1792,10 +1780,7 @@ turnLoop: }) gracefulTerminal, _ := ts.gracefulInterruptRequested() - var providerToolDefs []providers.ToolDefinition - if !ts.opts.IsVoice { - providerToolDefs = ts.agent.Tools.ToProviderDefs() - } + providerToolDefs := ts.agent.Tools.ToProviderDefs() // Native web search support (from HEAD) _, hasWebSearch := ts.agent.Tools.Get("web_search") diff --git a/pkg/utils/string.go b/pkg/utils/string.go index 99337b557..dbaafdb7f 100644 --- a/pkg/utils/string.go +++ b/pkg/utils/string.go @@ -65,8 +65,3 @@ func DerefStr(s *string, fallback string) string { } return *s } - -// IsTruncationDisabled returns whether truncation is disabled globally -func IsTruncationDisabled() bool { -return disableTruncation.Load() -} From e9347c522aaebb6573ee6d5d284b150a1611c1af Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 13:35:07 +0100 Subject: [PATCH 22/51] fix isVoice after revert --- pkg/agent/loop.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 82b20e3d1..8843cfc11 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -85,6 +85,7 @@ type processOptions struct { SendResponse bool // Whether to send response via bus NoHistory bool // If true, don't load session history (for heartbeat) SkipInitialSteeringPoll bool // If true, skip the steering poll at loop start (used by Continue) + IsVoice bool // If true, treat the message as voice input (transcribe before steering) } type continuationTarget struct { From d5b5b2de49768f5e61eea0df2062afa4ce324fc1 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 14:15:51 +0100 Subject: [PATCH 23/51] add tts tool --- config/config.example.json | 3 + pkg/agent/loop.go | 17 ++++++ pkg/config/config.go | 3 + pkg/config/defaults.go | 3 + pkg/tools/tts_send.go | 115 +++++++++++++++++++++++++++++++++++++ 5 files changed, 141 insertions(+) create mode 100644 pkg/tools/tts_send.go diff --git a/config/config.example.json b/config/config.example.json index 29655b594..8bcd9491a 100644 --- a/config/config.example.json +++ b/config/config.example.json @@ -523,6 +523,9 @@ "read_file": { "enabled": true }, + "send_tts": { + "enabled": false + }, "spawn": { "enabled": true }, diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 8843cfc11..d03d42244 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -31,6 +31,7 @@ import ( "github.com/sipeed/picoclaw/pkg/skills" "github.com/sipeed/picoclaw/pkg/state" "github.com/sipeed/picoclaw/pkg/tools" + "github.com/sipeed/picoclaw/pkg/tts" "github.com/sipeed/picoclaw/pkg/utils" ) @@ -153,6 +154,13 @@ func registerSharedTools( provider providers.LLMProvider, ) { allowReadPaths := buildAllowReadPatterns(cfg) + var ttsProvider tts.TTSProvider + if cfg.Tools.IsToolEnabled("send_tts") { + ttsProvider = tts.DetectTTS(cfg) + if ttsProvider == nil { + logger.WarnCF("voice-tts", "send_tts enabled but no TTS provider configured", nil) + } + } for _, agentID := range registry.ListAgentIDs() { agent, ok := registry.GetAgent(agentID) @@ -246,6 +254,10 @@ func registerSharedTools( agent.Tools.Register(sendFileTool) } + if ttsProvider != nil { + agent.Tools.Register(tools.NewSendTTSTool(ttsProvider, nil)) + } + // Skill discovery and installation tools skills_enabled := cfg.Tools.IsToolEnabled("skills") find_skills_enable := cfg.Tools.IsToolEnabled("find_skills") @@ -1022,6 +1034,11 @@ func (al *AgentLoop) SetMediaStore(s media.MediaStore) { sf.SetMediaStore(s) } }) + registry.ForEachTool("send_tts", func(t tools.Tool) { + if st, ok := t.(*tools.SendTTSTool); ok { + st.SetMediaStore(s) + } + }) } // SetTranscriber injects a voice transcriber for agent-level audio transcription. diff --git a/pkg/config/config.go b/pkg/config/config.go index cbed31ded..d3a032ac6 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -875,6 +875,7 @@ type ToolsConfig struct { Message ToolConfig `json:"message" envPrefix:"PICOCLAW_TOOLS_MESSAGE_"` ReadFile ReadFileToolConfig `json:"read_file" envPrefix:"PICOCLAW_TOOLS_READ_FILE_"` SendFile ToolConfig `json:"send_file" envPrefix:"PICOCLAW_TOOLS_SEND_FILE_"` + SendTTS ToolConfig `json:"send_tts" envPrefix:"PICOCLAW_TOOLS_SEND_TTS_"` Spawn ToolConfig `json:"spawn" envPrefix:"PICOCLAW_TOOLS_SPAWN_"` SpawnStatus ToolConfig `json:"spawn_status" envPrefix:"PICOCLAW_TOOLS_SPAWN_STATUS_"` SPI ToolConfig `json:"spi" envPrefix:"PICOCLAW_TOOLS_SPI_"` @@ -1368,6 +1369,8 @@ func (t *ToolsConfig) IsToolEnabled(name string) bool { return t.WebFetch.Enabled case "send_file": return t.SendFile.Enabled + case "send_tts": + return t.SendTTS.Enabled case "write_file": return t.WriteFile.Enabled case "mcp": diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index 2ec2b249d..3f3c09ddd 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -516,6 +516,9 @@ func DefaultConfig() *Config { SendFile: ToolConfig{ Enabled: true, }, + SendTTS: ToolConfig{ + Enabled: false, + }, MCP: MCPConfig{ ToolConfig: ToolConfig{ Enabled: false, diff --git a/pkg/tools/tts_send.go b/pkg/tools/tts_send.go new file mode 100644 index 000000000..4fae77a50 --- /dev/null +++ b/pkg/tools/tts_send.go @@ -0,0 +1,115 @@ +package tools + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/sipeed/picoclaw/pkg/media" + "github.com/sipeed/picoclaw/pkg/tts" +) + +type SendTTSTool struct { + provider tts.TTSProvider + mediaStore media.MediaStore +} + +func NewSendTTSTool(provider tts.TTSProvider, store media.MediaStore) *SendTTSTool { + return &SendTTSTool{ + provider: provider, + mediaStore: store, + } +} + +func (t *SendTTSTool) Name() string { return "send_tts" } + +func (t *SendTTSTool) Description() string { + return "Synthesize speech from text and send it as an audio file to the user." +} + +func (t *SendTTSTool) Parameters() map[string]any { + return map[string]any{ + "type": "object", + "properties": map[string]any{ + "text": map[string]any{ + "type": "string", + "description": "The text to synthesize into speech.", + }, + "filename": map[string]any{ + "type": "string", + "description": "Optional filename for the audio file (e.g., response.ogg).", + }, + }, + "required": []string{"text"}, + } +} + +func (t *SendTTSTool) SetMediaStore(store media.MediaStore) { + t.mediaStore = store +} + +func (t *SendTTSTool) Execute(ctx context.Context, args map[string]any) *ToolResult { + text, _ := args["text"].(string) + text = strings.TrimSpace(text) + if text == "" { + return ErrorResult("text is required") + } + + if t.provider == nil { + return ErrorResult("tts provider is not configured") + } + if t.mediaStore == nil { + return ErrorResult("media store not configured") + } + + channel := ToolChannel(ctx) + chatID := ToolChatID(ctx) + if channel == "" || chatID == "" { + return ErrorResult("no target channel/chat available") + } + + stream, err := t.provider.Synthesize(ctx, text) + if err != nil { + return ErrorResult(fmt.Sprintf("tts synthesize failed: %v", err)).WithError(err) + } + defer stream.Close() + + if err := os.MkdirAll(media.TempDir(), 0o755); err != nil { + return ErrorResult(fmt.Sprintf("failed to create media temp dir: %v", err)).WithError(err) + } + + file, err := os.CreateTemp(media.TempDir(), "tts-*.ogg") + if err != nil { + return ErrorResult(fmt.Sprintf("failed to create temp file: %v", err)).WithError(err) + } + defer file.Close() + + if _, err := io.Copy(file, stream); err != nil { + return ErrorResult(fmt.Sprintf("failed to write tts audio: %v", err)).WithError(err) + } + + filename, _ := args["filename"].(string) + filename = strings.TrimSpace(filename) + if filename == "" { + filename = fmt.Sprintf("tts-%d.ogg", time.Now().Unix()) + } + if filepath.Ext(filename) == "" { + filename += ".ogg" + } + + scope := fmt.Sprintf("tool:send_tts:%s:%s:%d", channel, chatID, time.Now().UnixNano()) + ref, err := t.mediaStore.Store(file.Name(), media.MediaMeta{ + Filename: filename, + ContentType: "audio/ogg", + Source: "tool:send_tts", + }, scope) + if err != nil { + return ErrorResult(fmt.Sprintf("failed to register audio: %v", err)).WithError(err) + } + + return MediaResult("TTS audio sent", []string{ref}) +} From c4d759c80d43fad1f5b1423f00fe3d2b2960a049 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 15:03:10 +0100 Subject: [PATCH 24/51] remove voice history cut and fix lint --- pkg/agent/loop.go | 8 +------- pkg/tools/tts_send.go | 6 ++++-- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index d03d42244..0ec42b683 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -86,7 +86,6 @@ type processOptions struct { SendResponse bool // Whether to send response via bus NoHistory bool // If true, don't load session history (for heartbeat) SkipInitialSteeringPoll bool // If true, skip the steering poll at loop start (used by Continue) - IsVoice bool // If true, treat the message as voice input (transcribe before steering) } type continuationTarget struct { @@ -1622,12 +1621,7 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState) (turnResult, er Role: "system", Content: ts.opts.SystemPromptOverride, }) - - if ts.opts.IsVoice && len(history) > 2 { - messages = append(messages, history[len(history)-2:]...) - } else { - messages = append(messages, history...) - } + messages = append(messages, history...) if summary != "" { messages = append(messages, providers.Message{ diff --git a/pkg/tools/tts_send.go b/pkg/tools/tts_send.go index 4fae77a50..71dd7e593 100644 --- a/pkg/tools/tts_send.go +++ b/pkg/tools/tts_send.go @@ -78,7 +78,8 @@ func (t *SendTTSTool) Execute(ctx context.Context, args map[string]any) *ToolRes } defer stream.Close() - if err := os.MkdirAll(media.TempDir(), 0o755); err != nil { + err = os.MkdirAll(media.TempDir(), 0o755) + if err != nil { return ErrorResult(fmt.Sprintf("failed to create media temp dir: %v", err)).WithError(err) } @@ -88,7 +89,8 @@ func (t *SendTTSTool) Execute(ctx context.Context, args map[string]any) *ToolRes } defer file.Close() - if _, err := io.Copy(file, stream); err != nil { + _, err = io.Copy(file, stream) + if err != nil { return ErrorResult(fmt.Sprintf("failed to write tts audio: %v", err)).WithError(err) } From 2d14da50a3c3455c71c34fd012285893d5b934b1 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 05:52:21 +0100 Subject: [PATCH 25/51] init commit --- go.mod | 3 + go.sum | 6 + pkg/bus/bus.go | 28 +++++ pkg/bus/types.go | 20 ++++ pkg/channels/discord/discord.go | 6 + pkg/channels/discord/voice.go | 86 ++++++++++++++ pkg/gateway/gateway.go | 16 +++ pkg/voice/agent.go | 195 ++++++++++++++++++++++++++++++++ pkg/voice/transcriber.go | 171 ++++++++++++++++++++++++---- 9 files changed, 512 insertions(+), 19 deletions(-) create mode 100644 pkg/channels/discord/voice.go create mode 100644 pkg/voice/agent.go diff --git a/go.mod b/go.mod index e4b6f37fd..d34f64c30 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,8 @@ require ( github.com/mymmrac/telego v1.7.0 github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1 github.com/openai/openai-go/v3 v3.22.0 + github.com/pion/rtp v1.8.7 + github.com/pion/webrtc/v3 v3.3.6 github.com/rivo/tview v0.42.0 github.com/rs/zerolog v1.34.0 github.com/slack-go/slack v0.17.3 @@ -53,6 +55,7 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 // indirect + github.com/pion/randutil v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect diff --git a/go.sum b/go.sum index f24b997d4..f95643290 100644 --- a/go.sum +++ b/go.sum @@ -162,6 +162,12 @@ github.com/openai/openai-go/v3 v3.22.0 h1:6MEoNoV8sbjOVmXdvhmuX3BjVbVdcExbVyGixi github.com/openai/openai-go/v3 v3.22.0/go.mod h1:cdufnVK14cWcT9qA1rRtrXx4FTRsgbDPW7Ia7SS5cZo= github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 h1:rh2lKw/P/EqHa724vYH2+VVQ1YnW4u6EOXl0PMAovZE= github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= +github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= +github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= +github.com/pion/rtp v1.8.7 h1:qslKkG8qxvQ7hqaxkmL7Pl0XcUm+/Er7nMnu6Vq+ZxM= +github.com/pion/rtp v1.8.7/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/webrtc/v3 v3.3.6 h1:7XAh4RPtlY1Vul6/GmZrv7z+NnxKA6If0KStXBI2ZLE= +github.com/pion/webrtc/v3 v3.3.6/go.mod h1:zyN7th4mZpV27eXybfR/cnUf3J2DRy8zw/mdjD9JTNM= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/pkg/bus/bus.go b/pkg/bus/bus.go index 37fcb74c5..a9c74ef90 100644 --- a/pkg/bus/bus.go +++ b/pkg/bus/bus.go @@ -34,6 +34,8 @@ type MessageBus struct { inbound chan InboundMessage outbound chan OutboundMessage outboundMedia chan OutboundMediaMessage + audioChunks chan AudioChunk + voiceControls chan VoiceControl closeOnce sync.Once done chan struct{} @@ -47,6 +49,8 @@ func NewMessageBus() *MessageBus { inbound: make(chan InboundMessage, defaultBusBufferSize), outbound: make(chan OutboundMessage, defaultBusBufferSize), outboundMedia: make(chan OutboundMediaMessage, defaultBusBufferSize), + audioChunks: make(chan AudioChunk, defaultBusBufferSize*4), // Audio chunks need more buffer + voiceControls: make(chan VoiceControl, defaultBusBufferSize), done: make(chan struct{}), } } @@ -103,6 +107,22 @@ func (mb *MessageBus) OutboundMediaChan() <-chan OutboundMediaMessage { return mb.outboundMedia } +func (mb *MessageBus) PublishAudioChunk(ctx context.Context, chunk AudioChunk) error { + return publish(ctx, mb, mb.audioChunks, chunk) +} + +func (mb *MessageBus) AudioChunksChan() <-chan AudioChunk { + return mb.audioChunks +} + +func (mb *MessageBus) PublishVoiceControl(ctx context.Context, ctrl VoiceControl) error { + return publish(ctx, mb, mb.voiceControls, ctrl) +} + +func (mb *MessageBus) VoiceControlsChan() <-chan VoiceControl { + return mb.voiceControls +} + // SetStreamDelegate registers a StreamDelegate (typically the channel Manager). func (mb *MessageBus) SetStreamDelegate(d StreamDelegate) { mb.streamDelegate.Store(d) @@ -132,6 +152,8 @@ func (mb *MessageBus) Close() { close(mb.inbound) close(mb.outbound) close(mb.outboundMedia) + close(mb.audioChunks) + close(mb.voiceControls) // clean up any remaining messages in channels drained := 0 @@ -144,6 +166,12 @@ func (mb *MessageBus) Close() { for range mb.outboundMedia { drained++ } + for range mb.audioChunks { + drained++ + } + for range mb.voiceControls { + drained++ + } if drained > 0 { logger.DebugCF("bus", "Drained buffered messages during close", map[string]any{ diff --git a/pkg/bus/types.go b/pkg/bus/types.go index 12da3f1dd..9c637f3e7 100644 --- a/pkg/bus/types.go +++ b/pkg/bus/types.go @@ -51,3 +51,23 @@ type OutboundMediaMessage struct { ChatID string `json:"chat_id"` Parts []MediaPart `json:"parts"` } + +// AudioChunk represents a chunk of streaming voice data. +type AudioChunk struct { + SessionID string `json:"session_id"` + SpeakerID string `json:"speaker_id"` // User ID or SSRC + ChatID string `json:"chat_id"` // Where to respond + Sequence uint64 `json:"sequence"` + Timestamp uint32 `json:"timestamp"` + SampleRate int `json:"sample_rate"` + Channels int `json:"channels"` + Format string `json:"format"` // "opus", "pcm", etc + Data []byte `json:"data"` +} + +// VoiceControl represents state or commands for voice sessions. +type VoiceControl struct { + SessionID string `json:"session_id"` + Type string `json:"type"` // "state", "command" + Action string `json:"action"` // "idle", "listening", "start", "stop" +} diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index 3b5b4f8bb..49121d3ad 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -42,6 +42,7 @@ type DiscordChannel struct { typingMu sync.Mutex typingStop map[string]chan struct{} // chatID → stop signal botUserID string // stored for mention checking + bus *bus.MessageBus } func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordChannel, error) { @@ -73,6 +74,7 @@ func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordC config: cfg, ctx: context.Background(), typingStop: make(map[string]chan struct{}), + bus: bus, }, nil } @@ -321,6 +323,10 @@ func (c *DiscordChannel) handleMessage(s *discordgo.Session, m *discordgo.Messag return } + if c.handleVoiceCommand(s, m) { + return + } + // Check allowlist first to avoid downloading attachments for rejected users sender := bus.SenderInfo{ Platform: "discord", diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go new file mode 100644 index 000000000..e2aae52ed --- /dev/null +++ b/pkg/channels/discord/voice.go @@ -0,0 +1,86 @@ +package discord + +import ( + "fmt" + + "github.com/bwmarrin/discordgo" + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/logger" +) + +func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.MessageCreate) bool { + if m.Content == "!vc join" { + vs, err := s.State.VoiceState(m.GuildID, m.Author.ID) + if err != nil || vs == nil { + s.ChannelMessageSend(m.ChannelID, "You need to be in a voice channel first!") + return true + } + + logger.InfoCF("discord", "Joining voice channel", map[string]any{"channel": vs.ChannelID}) + vc, err := s.ChannelVoiceJoin(m.GuildID, vs.ChannelID, false, false) + if err != nil { + s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("Failed to join voice channel: %v", err)) + return true + } + + go c.receiveVoice(vc, m.GuildID, m.ChannelID) + s.ChannelMessageSend(m.ChannelID, "Joined Voice Channel! Listening for audio...") + return true + } else if m.Content == "!vc leave" { + vc, exists := s.VoiceConnections[m.GuildID] + if exists && vc != nil { + vc.Disconnect() + s.ChannelMessageSend(m.ChannelID, "Left Voice Channel.") + } else { + s.ChannelMessageSend(m.ChannelID, "Not in a voice channel.") + } + return true + } + return false +} + +func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID string, chatID string) { + logger.InfoCF("discord", "Started listening for voice", map[string]any{"guild": guildID}) + + sessionID := fmt.Sprintf("discord_vc_%s", guildID) + + c.bus.PublishVoiceControl(c.ctx, bus.VoiceControl{ + SessionID: sessionID, + Type: "state", + Action: "listening", + }) + + var sequence uint64 = 0 + + for { + select { + case <-c.ctx.Done(): + return + case p, ok := <-vc.OpusRecv: + if !ok { + logger.InfoCF("discord", "Voice channel closed", map[string]any{"guild": guildID}) + return + } + + if p == nil { + continue + } + + sequence++ + + chunk := bus.AudioChunk{ + SessionID: sessionID, + SpeakerID: fmt.Sprintf("%d", p.SSRC), + ChatID: chatID, + Sequence: sequence, + Timestamp: p.Timestamp, + SampleRate: 48000, + Channels: 2, + Format: "opus", + Data: p.Opus, + } + + c.bus.PublishAudioChunk(c.ctx, chunk) + } + } +} diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go index fc2465747..f9d1c824b 100644 --- a/pkg/gateway/gateway.go +++ b/pkg/gateway/gateway.go @@ -60,6 +60,7 @@ type services struct { ChannelManager *channels.Manager DeviceService *devices.Service HealthServer *health.Server + VoiceAgentCancel context.CancelFunc manualReloadChan chan struct{} reloading atomic.Bool } @@ -305,6 +306,12 @@ func setupAndStartServices( if transcriber := voice.DetectTranscriber(cfg); transcriber != nil { agentLoop.SetTranscriber(transcriber) logger.InfoCF("voice", "Transcription enabled (agent-level)", map[string]any{"provider": transcriber.Name()}) + + // Start Voice Agent Orchestrator + vaCtx, vaCancel := context.WithCancel(context.Background()) + runningServices.VoiceAgentCancel = vaCancel + voiceAgent := voice.NewAgent(msgBus, transcriber) + voiceAgent.Start(vaCtx) } enabledChannels := runningServices.ChannelManager.GetEnabledChannels() @@ -351,6 +358,9 @@ func stopAndCleanupServices(runningServices *services, shutdownTimeout time.Dura if !isReload && runningServices.ChannelManager != nil { runningServices.ChannelManager.StopAll(shutdownCtx) } + if runningServices.VoiceAgentCancel != nil { + runningServices.VoiceAgentCancel() + } if runningServices.DeviceService != nil { runningServices.DeviceService.Stop() } @@ -532,6 +542,12 @@ func restartServices( al.SetTranscriber(transcriber) if transcriber != nil { logger.InfoCF("voice", "Transcription re-enabled (agent-level)", map[string]any{"provider": transcriber.Name()}) + + // Start Voice Agent Orchestrator on reload + vaCtx, vaCancel := context.WithCancel(context.Background()) + runningServices.VoiceAgentCancel = vaCancel + voiceAgent := voice.NewAgent(msgBus, transcriber) + voiceAgent.Start(vaCtx) } else { logger.InfoCF("voice", "Transcription disabled", nil) } diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go new file mode 100644 index 000000000..4362eb863 --- /dev/null +++ b/pkg/voice/agent.go @@ -0,0 +1,195 @@ +package voice + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/pion/rtp" + "github.com/pion/webrtc/v3/pkg/media/oggwriter" + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/logger" +) + +type speechAccumulator struct { + writer *oggwriter.OggWriter + file string + lastAudioAt time.Time + mu sync.Mutex + closed bool + chatID string + speakerID string +} + +func (a *speechAccumulator) Push(chunk bus.AudioChunk) { + a.mu.Lock() + defer a.mu.Unlock() + + if a.closed { + return + } + + a.lastAudioAt = time.Now() + + pkt := &rtp.Packet{ + Header: rtp.Header{ + SequenceNumber: uint16(chunk.Sequence), + Timestamp: chunk.Timestamp, + SSRC: uint32(chunk.Sequence), // Arbitrary dummy + }, + Payload: chunk.Data, + } + + if err := a.writer.WriteRTP(pkt); err != nil { + logger.ErrorCF("voice-agent", "Failed to write RTP", map[string]any{"error": err}) + } +} + +func (a *speechAccumulator) Close() { + a.mu.Lock() + defer a.mu.Unlock() + if !a.closed { + a.writer.Close() + a.closed = true + } +} + +type Agent struct { + bus *bus.MessageBus + transcriber Transcriber + + mu sync.Mutex + sessions map[string]*speechAccumulator // keyed by sessionID_speakerID +} + +func NewAgent(mb *bus.MessageBus, t Transcriber) *Agent { + return &Agent{ + bus: mb, + transcriber: t, + sessions: make(map[string]*speechAccumulator), + } +} + +func (a *Agent) Start(ctx context.Context) { + logger.InfoCF("voice-agent", "Started Voice Agent orchestrator", nil) + go a.listenChunks(ctx) + go a.vadTick(ctx) +} + +func (a *Agent) listenChunks(ctx context.Context) { + chunks := a.bus.AudioChunksChan() + for { + select { + case <-ctx.Done(): + return + case chunk := <-chunks: + a.handleChunk(chunk) + } + } +} + +func (a *Agent) handleChunk(chunk bus.AudioChunk) { + a.mu.Lock() + defer a.mu.Unlock() + + key := fmt.Sprintf("%s_%s", chunk.SessionID, chunk.SpeakerID) + + acc, exists := a.sessions[key] + if !exists { + filename := filepath.Join(os.TempDir(), fmt.Sprintf("voice_%s_%d.ogg", key, time.Now().UnixNano())) + writer, err := oggwriter.New(filename, uint32(chunk.SampleRate), uint16(chunk.Channels)) + if err != nil { + logger.ErrorCF("voice-agent", "Failed to create OggWriter", map[string]any{"error": err}) + return + } + + acc = &speechAccumulator{ + writer: writer, + file: filename, + lastAudioAt: time.Now(), + chatID: chunk.ChatID, + speakerID: chunk.SpeakerID, + } + a.sessions[key] = acc + logger.DebugCF("voice-agent", "Started accumulating voice", map[string]any{"key": key, "file": filename}) + } + + acc.Push(chunk) +} + +func (a *Agent) vadTick(ctx context.Context) { + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + a.checkSilence(ctx) + } + } +} + +func (a *Agent) checkSilence(ctx context.Context) { + a.mu.Lock() + now := time.Now() + var finished []*speechAccumulator + + for key, acc := range a.sessions { + acc.mu.Lock() + last := acc.lastAudioAt + acc.mu.Unlock() + + if now.Sub(last) > 1500*time.Millisecond { + acc.Close() + delete(a.sessions, key) + finished = append(finished, acc) + } + } + a.mu.Unlock() + + for _, acc := range finished { + go a.processUtterance(ctx, acc) + } +} + +func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) { + defer os.Remove(acc.file) + + logger.InfoCF("voice-agent", "User finished speaking, transcribing...", map[string]any{"file": acc.file}) + + if a.transcriber == nil { + logger.ErrorCF("voice-agent", "No STT configured!", nil) + return + } + + res, err := a.transcriber.Transcribe(ctx, acc.file) + if err != nil { + logger.ErrorCF("voice-agent", "Transcription failed", map[string]any{"error": err}) + return + } + + if res.Text == "" { + logger.DebugCF("voice-agent", "Ignored empty transcription", map[string]any{"file": acc.file}) + return + } + + logger.InfoCF("voice-agent", "Transcription result", map[string]any{"text": res.Text, "duration": res.Duration}) + + channelType := "discord" + + a.bus.PublishInbound(ctx, bus.InboundMessage{ + Channel: channelType, + SenderID: acc.speakerID, + ChatID: acc.chatID, + Content: res.Text, + Peer: bus.Peer{Kind: "channel", ID: acc.chatID}, + Metadata: map[string]string{ + "is_voice": "true", + }, + }) +} diff --git a/pkg/voice/transcriber.go b/pkg/voice/transcriber.go index a50fba8f8..1a089fd3c 100644 --- a/pkg/voice/transcriber.go +++ b/pkg/voice/transcriber.go @@ -1,11 +1,21 @@ package voice import ( + "bytes" "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + "path/filepath" "strings" + "time" "github.com/sipeed/picoclaw/pkg/config" - "github.com/sipeed/picoclaw/pkg/providers" + "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/utils" ) type Transcriber interface { @@ -13,34 +23,157 @@ type Transcriber interface { Transcribe(ctx context.Context, audioFilePath string) (*TranscriptionResponse, error) } +type GroqTranscriber struct { + apiKey string + apiBase string + httpClient *http.Client +} + type TranscriptionResponse struct { Text string `json:"text"` Language string `json:"language,omitempty"` Duration float64 `json:"duration,omitempty"` } -func supportsAudioTranscription(model string) bool { - protocol, _ := providers.ExtractProtocol(model) +func NewGroqTranscriber(apiKey string) *GroqTranscriber { + logger.DebugCF("voice", "Creating Groq transcriber", map[string]any{"has_api_key": apiKey != ""}) - switch protocol { - case "openai", "azure", "azure-openai", - "litellm", "openrouter", "groq", "zhipu", "gemini", "nvidia", - "ollama", "moonshot", "shengsuanyun", "deepseek", "cerebras", - "vivgrid", "volcengine", "vllm", "qwen", "qwen-intl", "qwen-international", "dashscope-intl", - "qwen-us", "dashscope-us", "mistral", "avian", "minimax", "longcat", "modelscope", "novita", - "coding-plan", "alibaba-coding", "qwen-coding": - // These protocols all go through the OpenAI-compatible or Azure provider path in - // providers.CreateProviderFromConfig, so they are the only ones that can supply - // the audio media payload shape expected by NewAudioModelTranscriber. - - // TODO: Further restrict this by modelID, since not every model under these - // protocols supports audio transcription. - return true - default: - return false + apiBase := "https://api.groq.com/openai/v1" + return &GroqTranscriber{ + apiKey: apiKey, + apiBase: apiBase, + httpClient: &http.Client{ + Timeout: 60 * time.Second, + }, } } +func (t *GroqTranscriber) TranscribeData(ctx context.Context, data []byte, filename string) (*TranscriptionResponse, error) { + logger.InfoCF("voice", "Starting memory transcription", map[string]any{"filename": filename, "bytes": len(data)}) + + var requestBody bytes.Buffer + writer := multipart.NewWriter(&requestBody) + + part, err := writer.CreateFormFile("file", filename) + if err != nil { + logger.ErrorCF("voice", "Failed to create form file", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to create form file: %w", err) + } + + if _, err := io.Copy(part, bytes.NewReader(data)); err != nil { + logger.ErrorCF("voice", "Failed to copy file content", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to copy file content: %w", err) + } + + if err = writer.WriteField("model", "whisper-large-v3-turbo"); err != nil { + logger.ErrorCF("voice", "Failed to write model field", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to write model field: %w", err) + } + + if err = writer.WriteField("response_format", "json"); err != nil { + logger.ErrorCF("voice", "Failed to write response_format field", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to write response_format field: %w", err) + } + + if err = writer.Close(); err != nil { + logger.ErrorCF("voice", "Failed to close multipart writer", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to close multipart writer: %w", err) + } + + return t.doRequest(ctx, &requestBody, writer.FormDataContentType(), int64(len(data))) +} + +func (t *GroqTranscriber) Transcribe(ctx context.Context, audioFilePath string) (*TranscriptionResponse, error) { + logger.InfoCF("voice", "Starting transcription", map[string]any{"audio_file": audioFilePath}) + + audioFile, err := os.Open(audioFilePath) + if err != nil { + return nil, fmt.Errorf("failed to open audio file: %w", err) + } + defer audioFile.Close() + + fileInfo, err := audioFile.Stat() + if err != nil { + return nil, err + } + + var requestBody bytes.Buffer + writer := multipart.NewWriter(&requestBody) + part, err := writer.CreateFormFile("file", filepath.Base(audioFilePath)) + if err != nil { + return nil, err + } + io.Copy(part, audioFile) + writer.WriteField("model", "whisper-large-v3") + writer.WriteField("response_format", "json") + writer.Close() + + return t.doRequest(ctx, &requestBody, writer.FormDataContentType(), fileInfo.Size()) +} + +func (t *GroqTranscriber) doRequest(ctx context.Context, requestBody *bytes.Buffer, contentType string, fileSize int64) (*TranscriptionResponse, error) { + url := t.apiBase + "/audio/transcriptions" + req, err := http.NewRequestWithContext(ctx, "POST", url, requestBody) + if err != nil { + logger.ErrorCF("voice", "Failed to create request", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", contentType) + req.Header.Set("Authorization", "Bearer "+t.apiKey) + + logger.DebugCF("voice", "Sending transcription request to Groq API", map[string]any{ + "url": url, + "request_size_bytes": requestBody.Len(), + "file_size_bytes": fileSize, + }) + + resp, err := t.httpClient.Do(req) + if err != nil { + logger.ErrorCF("voice", "Failed to send request", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + logger.ErrorCF("voice", "Failed to read response", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to read response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + logger.ErrorCF("voice", "API error", map[string]any{ + "status_code": resp.StatusCode, + "response": string(body), + }) + return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body)) + } + + logger.DebugCF("voice", "Received response from Groq API", map[string]any{ + "status_code": resp.StatusCode, + "response_size_bytes": len(body), + }) + + var result TranscriptionResponse + if err := json.Unmarshal(body, &result); err != nil { + logger.ErrorCF("voice", "Failed to unmarshal response", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to unmarshal response: %w", err) + } + + logger.InfoCF("voice", "Transcription completed successfully", map[string]any{ + "text_length": len(result.Text), + "language": result.Language, + "duration_seconds": result.Duration, + "transcription_preview": utils.Truncate(result.Text, 50), + }) + + return &result, nil +} + +func (t *GroqTranscriber) Name() string { + return "groq" +} + // DetectTranscriber inspects cfg and returns the appropriate Transcriber, or // nil if no supported transcription provider is configured. func DetectTranscriber(cfg *config.Config) Transcriber { From b444f53ce2c4d33cb01153b3de5fbfacae90d9a8 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 07:18:14 +0100 Subject: [PATCH 26/51] update with corrent transcriber and leave_tool --- go.mod | 3 ++ go.sum | 7 +++-- pkg/agent/loop.go | 3 ++ pkg/bus/types.go | 3 +- pkg/channels/discord/discord.go | 30 +++++++++++++++++++ pkg/channels/discord/voice.go | 24 +++++++++++++-- pkg/tools/voice_leave.go | 52 +++++++++++++++++++++++++++++++++ pkg/voice/agent.go | 7 ++++- pkg/voice/transcriber.go | 7 ++--- 9 files changed, 124 insertions(+), 12 deletions(-) create mode 100644 pkg/tools/voice_leave.go diff --git a/go.mod b/go.mod index d34f64c30..eede7da1a 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( require ( filippo.io/edwards25519 v1.2.0 // indirect github.com/beeper/argo-go v1.1.2 // indirect + github.com/cloudflare/circl v1.6.3 // indirect github.com/coder/websocket v1.8.14 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -101,3 +102,5 @@ require ( golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 ) + +replace github.com/bwmarrin/discordgo => github.com/yeongaori/discordgo-fork v0.0.0-20260319072544-e8e546f5d532 diff --git a/go.sum b/go.sum index f95643290..16d618f5b 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,6 @@ github.com/anthropics/anthropic-sdk-go v1.26.0 h1:oUTzFaUpAevfuELAP1sjL6CQJ9HHAf github.com/anthropics/anthropic-sdk-go v1.26.0/go.mod h1:qUKmaW+uuPB64iy1l+4kOSvaLqPXnHTTBKH6RVZ7q5Q= github.com/beeper/argo-go v1.1.2 h1:UQI2G8F+NLfGTOmTUI0254pGKx/HUU/etbUGTJv91Fs= github.com/beeper/argo-go v1.1.2/go.mod h1:M+LJAnyowKVQ6Rdj6XYGEn+qcVFkb3R/MUpqkGR0hM4= -github.com/bwmarrin/discordgo v0.29.0 h1:FmWeXFaKUwrcL3Cx65c20bTRW+vOb6k8AnaP+EgjDno= -github.com/bwmarrin/discordgo v0.29.0/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -31,6 +29,8 @@ github.com/caarlos0/env/v11 v11.4.0 h1:Kcb6t5kIIr4XkoQC9AF2j+8E1Jsrl3Wz/hhm1LtoG github.com/caarlos0/env/v11 v11.4.0/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g= @@ -236,6 +236,8 @@ github.com/vektah/gqlparser/v2 v2.5.27 h1:RHPD3JOplpk5mP5JGX8RKZkt2/Vwj/PZv0HxTd github.com/vektah/gqlparser/v2 v2.5.27/go.mod h1:D1/VCZtV3LPnQrcPBeR/q5jkSQIPti0uYCP/RI0gIeo= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yeongaori/discordgo-fork v0.0.0-20260319072544-e8e546f5d532 h1:gxFHYeUDGziRb0zXYEqBFohC+NJbIW9L0tddaXMWr2o= +github.com/yeongaori/discordgo-fork v0.0.0-20260319072544-e8e546f5d532/go.mod h1:A0FcMFJKJ9fRjgSuZ2o+pIQ6mPS81SVuiLN2vYTa7Ao= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -255,7 +257,6 @@ golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 72c78c729..1206545b0 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -238,6 +238,9 @@ func registerSharedTools( agent.Tools.Register(messageTool) } + // Always register Voice Leave Tool (it inherently checks if channel == "discord") + agent.Tools.Register(tools.NewVoiceLeaveTool(msgBus)) + // Send file tool (outbound media via MediaStore — store injected later by SetMediaStore) if cfg.Tools.IsToolEnabled("send_file") { sendFileTool := tools.NewSendFileTool( diff --git a/pkg/bus/types.go b/pkg/bus/types.go index 9c637f3e7..b4e99d955 100644 --- a/pkg/bus/types.go +++ b/pkg/bus/types.go @@ -68,6 +68,7 @@ type AudioChunk struct { // VoiceControl represents state or commands for voice sessions. type VoiceControl struct { SessionID string `json:"session_id"` - Type string `json:"type"` // "state", "command" + ChatID string `json:"chat_id"` + Type string `json:"type"` // "state", "command" Action string `json:"action"` // "idle", "listening", "start", "stop" } diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index 49121d3ad..6194685e8 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -91,6 +91,8 @@ func (c *DiscordChannel) Start(ctx context.Context) error { c.botUserID = botUser.ID c.session.AddHandler(c.handleMessage) + + go c.listenVoiceControl(c.ctx) if err := c.session.Open(); err != nil { return fmt.Errorf("failed to open discord session: %w", err) @@ -619,3 +621,31 @@ func (c *DiscordChannel) stripBotMention(text string) string { text = strings.ReplaceAll(text, fmt.Sprintf("<@!%s>", c.botUserID), "") return strings.TrimSpace(text) } + +func (c *DiscordChannel) listenVoiceControl(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case ctrl := <-c.bus.VoiceControlsChan(): + if ctrl.Type == "command" && ctrl.Action == "leave" { + var guildID string + if strings.HasPrefix(ctrl.SessionID, "discord_vc_") { + guildID = strings.TrimPrefix(ctrl.SessionID, "discord_vc_") + } else if ctrl.ChatID != "" { + ch, err := c.session.State.Channel(ctrl.ChatID) + if err == nil { + guildID = ch.GuildID + } + } + + if guildID != "" { + vc, exists := c.session.VoiceConnections[guildID] + if exists && vc != nil { + vc.Disconnect(ctx) + } + } + } + } + } +} diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index e2aae52ed..23386e642 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -2,6 +2,7 @@ package discord import ( "fmt" + "time" "github.com/bwmarrin/discordgo" "github.com/sipeed/picoclaw/pkg/bus" @@ -17,7 +18,7 @@ func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.M } logger.InfoCF("discord", "Joining voice channel", map[string]any{"channel": vs.ChannelID}) - vc, err := s.ChannelVoiceJoin(m.GuildID, vs.ChannelID, false, false) + vc, err := s.ChannelVoiceJoin(c.ctx, m.GuildID, vs.ChannelID, false, false) if err != nil { s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("Failed to join voice channel: %v", err)) return true @@ -29,7 +30,7 @@ func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.M } else if m.Content == "!vc leave" { vc, exists := s.VoiceConnections[m.GuildID] if exists && vc != nil { - vc.Disconnect() + vc.Disconnect(c.ctx) s.ChannelMessageSend(m.ChannelID, "Left Voice Channel.") } else { s.ChannelMessageSend(m.ChannelID, "Not in a voice channel.") @@ -42,6 +43,17 @@ func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.M func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID string, chatID string) { logger.InfoCF("discord", "Started listening for voice", map[string]any{"guild": guildID}) + go func() { + time.Sleep(250 * time.Millisecond) // Wait a bit for connection to settle + vc.Speaking(true) + for i := 0; i < 5; i++ { + vc.OpusSend <- []byte{0xF8, 0xFF, 0xFE} + time.Sleep(20 * time.Millisecond) + } + vc.Speaking(false) + logger.DebugCF("discord", "Sent wake-up silence frames", nil) + }() + sessionID := fmt.Sprintf("discord_vc_%s", guildID) c.bus.PublishVoiceControl(c.ctx, bus.VoiceControl{ @@ -62,7 +74,13 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str return } - if p == nil { + logger.DebugCF("discord", "Received Opus packet", map[string]any{ + "seq": p.Sequence, + "len": len(p.Opus), + "ssrc": p.SSRC, + }) + + if p == nil || len(p.Opus) == 0 { continue } diff --git a/pkg/tools/voice_leave.go b/pkg/tools/voice_leave.go new file mode 100644 index 000000000..882e7cb9d --- /dev/null +++ b/pkg/tools/voice_leave.go @@ -0,0 +1,52 @@ +package tools + +import ( + "context" + + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/logger" +) + +type VoiceLeaveTool struct { + bus *bus.MessageBus +} + +func NewVoiceLeaveTool(mb *bus.MessageBus) *VoiceLeaveTool { + return &VoiceLeaveTool{bus: mb} +} + +func (t *VoiceLeaveTool) Name() string { + return "voice_leave" +} + +func (t *VoiceLeaveTool) Description() string { + return "Disconnects the bot from the current voice channel. Use this tool when the user says goodbye or explicitly asks you to leave the voice chat." +} + +func (t *VoiceLeaveTool) Parameters() map[string]any { + return map[string]any{ + "type": "object", + "properties": map[string]any{}, + } +} + +func (t *VoiceLeaveTool) Execute(ctx context.Context, args map[string]any) *ToolResult { + channel := ToolChannel(ctx) + chatID := ToolChatID(ctx) + + if channel != "discord" { + return &ToolResult{ForLLM: "Can only leave voice channels on Discord", IsError: true} + } + + t.bus.PublishVoiceControl(ctx, bus.VoiceControl{ + ChatID: chatID, + Type: "command", + Action: "leave", + }) + + logger.InfoCF("agent", "Voice command triggered via tool: leave", map[string]any{"chat_id": chatID}) + + return &ToolResult{ + ForLLM: "Successfully sent disconnect command to voice adapter.", + } +} diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index 4362eb863..40bc02c57 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "sync" "time" @@ -22,6 +23,7 @@ type speechAccumulator struct { closed bool chatID string speakerID string + sessionID string } func (a *speechAccumulator) Push(chunk bus.AudioChunk) { @@ -112,6 +114,7 @@ func (a *Agent) handleChunk(chunk bus.AudioChunk) { lastAudioAt: time.Now(), chatID: chunk.ChatID, speakerID: chunk.SpeakerID, + sessionID: chunk.SessionID, } a.sessions[key] = acc logger.DebugCF("voice-agent", "Started accumulating voice", map[string]any{"key": key, "file": filename}) @@ -182,11 +185,13 @@ func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) { channelType := "discord" + oralPrompt := "\n\n[SYSTEM]: The user just spoke this to you over voice chat. Please reply in a highly concise, conversational, oral style suitable for text-to-speech. Do not use markdown, emojis, asterisks, or code blocks. Speak naturally. If the user expresses that they want to end the call, say goodbye and use the voice_leave tool." + a.bus.PublishInbound(ctx, bus.InboundMessage{ Channel: channelType, SenderID: acc.speakerID, ChatID: acc.chatID, - Content: res.Text, + Content: res.Text + oralPrompt, Peer: bus.Peer{Kind: "channel", ID: acc.chatID}, Metadata: map[string]string{ "is_voice": "true", diff --git a/pkg/voice/transcriber.go b/pkg/voice/transcriber.go index 1a089fd3c..15f7fc0d0 100644 --- a/pkg/voice/transcriber.go +++ b/pkg/voice/transcriber.go @@ -186,11 +186,10 @@ func DetectTranscriber(cfg *config.Config) Transcriber { return NewAudioModelTranscriber(modelCfg) } } - - // Fall back to any model-list entry that uses the groq/ protocol. + // Fall back to any model-list entry that uses the groq/ protocol or is explicitly named groq. for _, mc := range cfg.ModelList { - if strings.HasPrefix(mc.Model, "groq/") && mc.APIKey() != "" { - return NewGroqTranscriber(mc.APIKey()) + if (strings.HasPrefix(mc.Model, "groq/") || mc.ModelName == "groq" || mc.Model == "whisper-large-v3-turbo") && mc.APIKey != "" { + return NewGroqTranscriber(mc.APIKey) } } return nil From 98b337e703f7e69bc1e8fe855015a432a62f1747 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 07:18:39 +0100 Subject: [PATCH 27/51] fix lint --- pkg/voice/agent.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index 40bc02c57..233da6019 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "path/filepath" - "strings" "sync" "time" From 5ac577f0989c81acfbea5be9608b4fcf985ce334 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 07:21:33 +0100 Subject: [PATCH 28/51] remove leave tool due to delay --- pkg/agent/loop.go | 3 -- pkg/bus/types.go | 3 +- pkg/channels/discord/discord.go | 11 +------ pkg/tools/voice_leave.go | 52 --------------------------------- pkg/voice/agent.go | 19 +++++++++++- 5 files changed, 20 insertions(+), 68 deletions(-) delete mode 100644 pkg/tools/voice_leave.go diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 1206545b0..72c78c729 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -238,9 +238,6 @@ func registerSharedTools( agent.Tools.Register(messageTool) } - // Always register Voice Leave Tool (it inherently checks if channel == "discord") - agent.Tools.Register(tools.NewVoiceLeaveTool(msgBus)) - // Send file tool (outbound media via MediaStore — store injected later by SetMediaStore) if cfg.Tools.IsToolEnabled("send_file") { sendFileTool := tools.NewSendFileTool( diff --git a/pkg/bus/types.go b/pkg/bus/types.go index b4e99d955..9c637f3e7 100644 --- a/pkg/bus/types.go +++ b/pkg/bus/types.go @@ -68,7 +68,6 @@ type AudioChunk struct { // VoiceControl represents state or commands for voice sessions. type VoiceControl struct { SessionID string `json:"session_id"` - ChatID string `json:"chat_id"` - Type string `json:"type"` // "state", "command" + Type string `json:"type"` // "state", "command" Action string `json:"action"` // "idle", "listening", "start", "stop" } diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index 6194685e8..26332455a 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -629,17 +629,8 @@ func (c *DiscordChannel) listenVoiceControl(ctx context.Context) { return case ctrl := <-c.bus.VoiceControlsChan(): if ctrl.Type == "command" && ctrl.Action == "leave" { - var guildID string if strings.HasPrefix(ctrl.SessionID, "discord_vc_") { - guildID = strings.TrimPrefix(ctrl.SessionID, "discord_vc_") - } else if ctrl.ChatID != "" { - ch, err := c.session.State.Channel(ctrl.ChatID) - if err == nil { - guildID = ch.GuildID - } - } - - if guildID != "" { + guildID := strings.TrimPrefix(ctrl.SessionID, "discord_vc_") vc, exists := c.session.VoiceConnections[guildID] if exists && vc != nil { vc.Disconnect(ctx) diff --git a/pkg/tools/voice_leave.go b/pkg/tools/voice_leave.go deleted file mode 100644 index 882e7cb9d..000000000 --- a/pkg/tools/voice_leave.go +++ /dev/null @@ -1,52 +0,0 @@ -package tools - -import ( - "context" - - "github.com/sipeed/picoclaw/pkg/bus" - "github.com/sipeed/picoclaw/pkg/logger" -) - -type VoiceLeaveTool struct { - bus *bus.MessageBus -} - -func NewVoiceLeaveTool(mb *bus.MessageBus) *VoiceLeaveTool { - return &VoiceLeaveTool{bus: mb} -} - -func (t *VoiceLeaveTool) Name() string { - return "voice_leave" -} - -func (t *VoiceLeaveTool) Description() string { - return "Disconnects the bot from the current voice channel. Use this tool when the user says goodbye or explicitly asks you to leave the voice chat." -} - -func (t *VoiceLeaveTool) Parameters() map[string]any { - return map[string]any{ - "type": "object", - "properties": map[string]any{}, - } -} - -func (t *VoiceLeaveTool) Execute(ctx context.Context, args map[string]any) *ToolResult { - channel := ToolChannel(ctx) - chatID := ToolChatID(ctx) - - if channel != "discord" { - return &ToolResult{ForLLM: "Can only leave voice channels on Discord", IsError: true} - } - - t.bus.PublishVoiceControl(ctx, bus.VoiceControl{ - ChatID: chatID, - Type: "command", - Action: "leave", - }) - - logger.InfoCF("agent", "Voice command triggered via tool: leave", map[string]any{"chat_id": chatID}) - - return &ToolResult{ - ForLLM: "Successfully sent disconnect command to voice adapter.", - } -} diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index 233da6019..6a2abd24f 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "sync" "time" @@ -184,7 +185,23 @@ func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) { channelType := "discord" - oralPrompt := "\n\n[SYSTEM]: The user just spoke this to you over voice chat. Please reply in a highly concise, conversational, oral style suitable for text-to-speech. Do not use markdown, emojis, asterisks, or code blocks. Speak naturally. If the user expresses that they want to end the call, say goodbye and use the voice_leave tool." + text := strings.ToLower(strings.TrimSpace(res.Text)) + if strings.Contains(text, "leave the voice channel") || strings.Contains(text, "leave voice") || strings.Contains(text, "disconnect voice") { + logger.InfoCF("voice-agent", "Voice command triggered: leave", nil) + a.bus.PublishVoiceControl(ctx, bus.VoiceControl{ + SessionID: acc.sessionID, + Type: "command", + Action: "leave", + }) + a.bus.PublishOutbound(ctx, bus.OutboundMessage{ + Channel: channelType, + ChatID: acc.chatID, + Content: "Goodbye! Leaving the voice channel.", + }) + return + } + + oralPrompt := "\n\n[SYSTEM]: The user just spoke this to you over voice chat. Please reply in a highly concise, conversational, oral style suitable for text-to-speech. Do not use markdown, emojis, asterisks, or code blocks. Speak naturally." a.bus.PublishInbound(ctx, bus.InboundMessage{ Channel: channelType, From cf315091df1e864037c687ff6c276e0a05ecfd8b Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 07:53:28 +0100 Subject: [PATCH 29/51] add tts support --- pkg/channels/discord/discord.go | 23 +++++++ pkg/channels/discord/init.go | 7 ++- pkg/channels/discord/voice.go | 55 ++++++++++++++++ pkg/voice/tts.go | 108 ++++++++++++++++++++++++++++++++ 4 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 pkg/voice/tts.go diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index 26332455a..06a0f175f 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -21,6 +21,7 @@ import ( "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/media" "github.com/sipeed/picoclaw/pkg/utils" + "github.com/sipeed/picoclaw/pkg/voice" ) const ( @@ -43,6 +44,7 @@ type DiscordChannel struct { typingStop map[string]chan struct{} // chatID → stop signal botUserID string // stored for mention checking bus *bus.MessageBus + tts voice.TTSProvider } func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordChannel, error) { @@ -146,6 +148,14 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) erro return nil } + if c.tts != nil { + if ch, err := c.session.State.Channel(channelID); err == nil && ch.GuildID != "" { + if vc, ok := c.session.VoiceConnections[ch.GuildID]; ok && vc != nil { + go c.playTTS(context.Background(), vc, msg.Content) + } + } + } + return c.sendChunk(ctx, channelID, msg.Content, msg.ReplyToMessageID) } @@ -640,3 +650,16 @@ func (c *DiscordChannel) listenVoiceControl(ctx context.Context) { } } } + +func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnection, text string) { + stream, err := c.tts.Synthesize(ctx, text) + if err != nil { + logger.ErrorCF("discord", "TTS synthesize failed", map[string]any{"error": err.Error()}) + return + } + defer stream.Close() + + if err := streamOggOpusToDiscord(vc, stream); err != nil { + logger.ErrorCF("discord", "TTS playback failed", map[string]any{"error": err.Error()}) + } +} diff --git a/pkg/channels/discord/init.go b/pkg/channels/discord/init.go index 15a539804..13e4fbc91 100644 --- a/pkg/channels/discord/init.go +++ b/pkg/channels/discord/init.go @@ -4,10 +4,15 @@ import ( "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/voice" ) func init() { channels.RegisterFactory("discord", func(cfg *config.Config, b *bus.MessageBus) (channels.Channel, error) { - return NewDiscordChannel(cfg.Channels.Discord, b) + ch, err := NewDiscordChannel(cfg.Channels.Discord, b) + if err == nil { + ch.tts = voice.DetectTTS(cfg) + } + return ch, err }) } diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index 23386e642..72f767b85 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -1,7 +1,9 @@ package discord import ( + "bytes" "fmt" + "io" "time" "github.com/bwmarrin/discordgo" @@ -40,6 +42,59 @@ func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.M return false } +func VoiceReceiveActive(vc *discordgo.VoiceConnection) bool { + return vc != nil && vc.OpusRecv != nil +} + +func streamOggOpusToDiscord(vc *discordgo.VoiceConnection, r io.Reader) error { + // Wait for the speaking transition to register + vc.Speaking(true) + defer vc.Speaking(false) + + var packet []byte + header := make([]byte, 27) + + for { + if _, err := io.ReadFull(r, header); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + return nil + } + return fmt.Errorf("failed to read ogg header: %w", err) + } + if string(header[:4]) != "OggS" { + return fmt.Errorf("invalid ogg magic string") + } + + pageSegments := int(header[26]) + segmentTable := make([]byte, pageSegments) + if _, err := io.ReadFull(r, segmentTable); err != nil { + return fmt.Errorf("failed to read segment table: %w", err) + } + + for _, lacing := range segmentTable { + segment := make([]byte, lacing) + if _, err := io.ReadFull(r, segment); err != nil { + return fmt.Errorf("failed to read segment data: %w", err) + } + + packet = append(packet, segment...) + + // If lacing is less than 255, the packet is complete + if lacing < 255 { + if len(packet) > 0 { + // Ignore Ogg Opus headers + if !bytes.HasPrefix(packet, []byte("OpusHead")) && !bytes.HasPrefix(packet, []byte("OpusTags")) { + // Pacing is handled natively by vc.OpusSend blocking (it has an internal ticker) + vc.OpusSend <- packet + } + // Start new packet + packet = nil + } + } + } + } +} + func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID string, chatID string) { logger.InfoCF("discord", "Started listening for voice", map[string]any{"guild": guildID}) diff --git a/pkg/voice/tts.go b/pkg/voice/tts.go new file mode 100644 index 000000000..8de0bbc9c --- /dev/null +++ b/pkg/voice/tts.go @@ -0,0 +1,108 @@ +package voice + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/logger" +) + +type TTSProvider interface { + Name() string + Synthesize(ctx context.Context, text string) (io.ReadCloser, error) +} + +type OpenAITTSProvider struct { + apiKey string + apiBase string + voice string + model string + httpClient *http.Client +} + +func NewOpenAITTSProvider(apiKey string, apiBase string, proxyURL string) *OpenAITTSProvider { + if apiBase == "" || apiBase == "https://api.openai.com/v1" { + apiBase = "https://api.openai.com/v1/audio/speech" + } else if !strings.HasSuffix(apiBase, "/audio/speech") { + // Just in case they provide openrouter base or standard base + apiBase = strings.TrimSuffix(apiBase, "/") + "/audio/speech" + } + + client := &http.Client{ + Timeout: 60 * time.Second, + } + + if proxyURL != "" { + if pURL, err := url.Parse(proxyURL); err == nil { + client.Transport = &http.Transport{ + Proxy: http.ProxyURL(pURL), + } + } + } + + return &OpenAITTSProvider{ + apiKey: apiKey, + apiBase: apiBase, + voice: "alloy", + model: "tts-1", + httpClient: client, + } +} + +func (t *OpenAITTSProvider) Name() string { + return "openai-tts" +} + +func (t *OpenAITTSProvider) Synthesize(ctx context.Context, text string) (io.ReadCloser, error) { + logger.InfoCF("voice-tts", "Starting TTS synthesis", map[string]any{"text_len": len(text)}) + + reqBody := map[string]any{ + "model": t.model, + "input": text, + "voice": t.voice, + "response_format": "opus", + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", t.apiBase, bytes.NewReader(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+t.apiKey) + + resp, err := t.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + + if resp.StatusCode != http.StatusOK { + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body)) + } + + return resp.Body, nil +} + +func DetectTTS(cfg *config.Config) TTSProvider { + for _, mc := range cfg.ModelList { + if strings.Contains(strings.ToLower(mc.ModelName), "tts") && mc.APIKey != "" { + return NewOpenAITTSProvider(mc.APIKey, mc.APIBase, mc.Proxy) + } + } + return nil +} From 03b212b6d8f538b40c33705e9c87fed0a9fac3b6 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 08:05:54 +0100 Subject: [PATCH 30/51] refactor for #1648 --- pkg/agent/loop.go | 6 +- pkg/{voice/transcriber.go => asr/asr.go} | 2 +- .../transcriber_test.go => asr/asr_test.go} | 2 +- pkg/audio/ogg.go | 55 +++++++++++++++++++ pkg/bus/types.go | 2 +- pkg/channels/discord/discord.go | 6 +- pkg/channels/discord/init.go | 4 +- pkg/gateway/gateway.go | 5 +- pkg/{voice => tts}/tts.go | 2 +- pkg/voice/agent.go | 5 +- 10 files changed, 73 insertions(+), 16 deletions(-) rename pkg/{voice/transcriber.go => asr/asr.go} (99%) rename pkg/{voice/transcriber_test.go => asr/asr_test.go} (99%) create mode 100644 pkg/audio/ogg.go rename pkg/{voice => tts}/tts.go (99%) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 72c78c729..05ccbe449 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -18,6 +18,7 @@ import ( "sync/atomic" "time" + "github.com/sipeed/picoclaw/pkg/asr" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/commands" @@ -31,7 +32,6 @@ import ( "github.com/sipeed/picoclaw/pkg/state" "github.com/sipeed/picoclaw/pkg/tools" "github.com/sipeed/picoclaw/pkg/utils" - "github.com/sipeed/picoclaw/pkg/voice" ) type AgentLoop struct { @@ -51,7 +51,7 @@ type AgentLoop struct { fallback *providers.FallbackChain channelManager *channels.Manager mediaStore media.MediaStore - transcriber voice.Transcriber + transcriber asr.Transcriber cmdRegistry *commands.Registry mcp mcpRuntime hookRuntime hookRuntime @@ -1040,7 +1040,7 @@ func (al *AgentLoop) SetMediaStore(s media.MediaStore) { } // SetTranscriber injects a voice transcriber for agent-level audio transcription. -func (al *AgentLoop) SetTranscriber(t voice.Transcriber) { +func (al *AgentLoop) SetTranscriber(t asr.Transcriber) { al.transcriber = t } diff --git a/pkg/voice/transcriber.go b/pkg/asr/asr.go similarity index 99% rename from pkg/voice/transcriber.go rename to pkg/asr/asr.go index 15f7fc0d0..1e2b73500 100644 --- a/pkg/voice/transcriber.go +++ b/pkg/asr/asr.go @@ -1,4 +1,4 @@ -package voice +package asr import ( "bytes" diff --git a/pkg/voice/transcriber_test.go b/pkg/asr/asr_test.go similarity index 99% rename from pkg/voice/transcriber_test.go rename to pkg/asr/asr_test.go index 20ba5388b..c6d04c08b 100644 --- a/pkg/voice/transcriber_test.go +++ b/pkg/asr/asr_test.go @@ -1,4 +1,4 @@ -package voice +package asr import ( "testing" diff --git a/pkg/audio/ogg.go b/pkg/audio/ogg.go new file mode 100644 index 000000000..49a2b93f9 --- /dev/null +++ b/pkg/audio/ogg.go @@ -0,0 +1,55 @@ +package audio + +import ( + "bytes" + "fmt" + "io" +) + +// DecodeOggOpus reads an Ogg format stream and extracts individual Opus payloads. +// It calls onFrame for every complete Opus frame found in the stream. +func DecodeOggOpus(r io.Reader, onFrame func([]byte) error) error { + var packet []byte + header := make([]byte, 27) + + for { + if _, err := io.ReadFull(r, header); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + return nil + } + return fmt.Errorf("failed to read ogg header: %w", err) + } + if string(header[:4]) != "OggS" { + return fmt.Errorf("invalid ogg magic string") + } + + pageSegments := int(header[26]) + segmentTable := make([]byte, pageSegments) + if _, err := io.ReadFull(r, segmentTable); err != nil { + return fmt.Errorf("failed to read segment table: %w", err) + } + + for _, lacing := range segmentTable { + segment := make([]byte, lacing) + if _, err := io.ReadFull(r, segment); err != nil { + return fmt.Errorf("failed to read segment data: %w", err) + } + + packet = append(packet, segment...) + + // If lacing is less than 255, the packet is complete + if lacing < 255 { + if len(packet) > 0 { + // Ignore Ogg Opus headers + if !bytes.HasPrefix(packet, []byte("OpusHead")) && !bytes.HasPrefix(packet, []byte("OpusTags")) { + if err := onFrame(packet); err != nil { + return err + } + } + // Start new packet + packet = nil + } + } + } + } +} diff --git a/pkg/bus/types.go b/pkg/bus/types.go index 9c637f3e7..e036b4ede 100644 --- a/pkg/bus/types.go +++ b/pkg/bus/types.go @@ -68,6 +68,6 @@ type AudioChunk struct { // VoiceControl represents state or commands for voice sessions. type VoiceControl struct { SessionID string `json:"session_id"` - Type string `json:"type"` // "state", "command" + Type string `json:"type"` // "state", "command" Action string `json:"action"` // "idle", "listening", "start", "stop" } diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index 06a0f175f..ebdbd1437 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -20,8 +20,8 @@ import ( "github.com/sipeed/picoclaw/pkg/identity" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/media" + "github.com/sipeed/picoclaw/pkg/tts" "github.com/sipeed/picoclaw/pkg/utils" - "github.com/sipeed/picoclaw/pkg/voice" ) const ( @@ -44,7 +44,7 @@ type DiscordChannel struct { typingStop map[string]chan struct{} // chatID → stop signal botUserID string // stored for mention checking bus *bus.MessageBus - tts voice.TTSProvider + tts tts.TTSProvider } func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordChannel, error) { @@ -93,7 +93,7 @@ func (c *DiscordChannel) Start(ctx context.Context) error { c.botUserID = botUser.ID c.session.AddHandler(c.handleMessage) - + go c.listenVoiceControl(c.ctx) if err := c.session.Open(); err != nil { diff --git a/pkg/channels/discord/init.go b/pkg/channels/discord/init.go index 13e4fbc91..e6be8ff36 100644 --- a/pkg/channels/discord/init.go +++ b/pkg/channels/discord/init.go @@ -4,14 +4,14 @@ import ( "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/config" - "github.com/sipeed/picoclaw/pkg/voice" + "github.com/sipeed/picoclaw/pkg/tts" ) func init() { channels.RegisterFactory("discord", func(cfg *config.Config, b *bus.MessageBus) (channels.Channel, error) { ch, err := NewDiscordChannel(cfg.Channels.Discord, b) if err == nil { - ch.tts = voice.DetectTTS(cfg) + ch.tts = tts.DetectTTS(cfg) } return ch, err }) diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go index f9d1c824b..ad7591e3a 100644 --- a/pkg/gateway/gateway.go +++ b/pkg/gateway/gateway.go @@ -12,6 +12,7 @@ import ( "time" "github.com/sipeed/picoclaw/pkg/agent" + "github.com/sipeed/picoclaw/pkg/asr" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" _ "github.com/sipeed/picoclaw/pkg/channels/dingtalk" @@ -303,7 +304,7 @@ func setupAndStartServices( agentLoop.SetChannelManager(runningServices.ChannelManager) agentLoop.SetMediaStore(runningServices.MediaStore) - if transcriber := voice.DetectTranscriber(cfg); transcriber != nil { + if transcriber := asr.DetectTranscriber(cfg); transcriber != nil { agentLoop.SetTranscriber(transcriber) logger.InfoCF("voice", "Transcription enabled (agent-level)", map[string]any{"provider": transcriber.Name()}) @@ -538,7 +539,7 @@ func restartServices( fmt.Println(" ✓ Device event service restarted") } - transcriber := voice.DetectTranscriber(cfg) + transcriber := asr.DetectTranscriber(cfg) al.SetTranscriber(transcriber) if transcriber != nil { logger.InfoCF("voice", "Transcription re-enabled (agent-level)", map[string]any{"provider": transcriber.Name()}) diff --git a/pkg/voice/tts.go b/pkg/tts/tts.go similarity index 99% rename from pkg/voice/tts.go rename to pkg/tts/tts.go index 8de0bbc9c..63b4ecd24 100644 --- a/pkg/voice/tts.go +++ b/pkg/tts/tts.go @@ -1,4 +1,4 @@ -package voice +package tts import ( "bytes" diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index 6a2abd24f..6a330d3d4 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -11,6 +11,7 @@ import ( "github.com/pion/rtp" "github.com/pion/webrtc/v3/pkg/media/oggwriter" + "github.com/sipeed/picoclaw/pkg/asr" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/logger" ) @@ -61,13 +62,13 @@ func (a *speechAccumulator) Close() { type Agent struct { bus *bus.MessageBus - transcriber Transcriber + transcriber asr.Transcriber mu sync.Mutex sessions map[string]*speechAccumulator // keyed by sessionID_speakerID } -func NewAgent(mb *bus.MessageBus, t Transcriber) *Agent { +func NewAgent(mb *bus.MessageBus, t asr.Transcriber) *Agent { return &Agent{ bus: mb, transcriber: t, From 347eabd0dec9c2e943b0282fb68ba770b891f744 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 08:24:17 +0100 Subject: [PATCH 31/51] update with streaming support --- pkg/audio/sentence.go | 96 +++++++++++++++++++++++++++++++++ pkg/channels/discord/discord.go | 88 +++++++++++++++++++++++++++--- pkg/channels/discord/voice.go | 39 ++++++++++++-- 3 files changed, 213 insertions(+), 10 deletions(-) create mode 100644 pkg/audio/sentence.go diff --git a/pkg/audio/sentence.go b/pkg/audio/sentence.go new file mode 100644 index 000000000..c7a9b2f26 --- /dev/null +++ b/pkg/audio/sentence.go @@ -0,0 +1,96 @@ +package audio + +import ( + "strings" + "unicode" +) + +// SplitSentences splits text into sentence-sized chunks suitable for TTS synthesis. +// It splits on sentence-ending punctuation (.!?\n) while avoiding false splits +// on abbreviations and decimal numbers. Very short fragments are merged with +// the next sentence to prevent choppy playback. +func SplitSentences(text string) []string { + if text == "" { + return nil + } + + var sentences []string + var current strings.Builder + runes := []rune(text) + + for i := 0; i < len(runes); i++ { + r := runes[i] + current.WriteRune(r) + + if r == '\n' { + s := strings.TrimSpace(current.String()) + if s != "" { + sentences = append(sentences, s) + } + current.Reset() + continue + } + + if r == '.' || r == '!' || r == '?' { + // Avoid splitting on decimal numbers like "3.14" + if r == '.' && i > 0 && unicode.IsDigit(runes[i-1]) && + i+1 < len(runes) && unicode.IsDigit(runes[i+1]) { + continue + } + + // Consume trailing punctuation and spaces (e.g., "..." or "?!") + for i+1 < len(runes) && (runes[i+1] == '.' || runes[i+1] == '!' || runes[i+1] == '?' || runes[i+1] == ' ') { + i++ + current.WriteRune(runes[i]) + } + + s := strings.TrimSpace(current.String()) + if s != "" { + sentences = append(sentences, s) + } + current.Reset() + } + } + + // Flush remaining text + if s := strings.TrimSpace(current.String()); s != "" { + sentences = append(sentences, s) + } + + // Merge very short fragments with the next sentence + return mergeShorties(sentences, 15) +} + +// mergeShorties merges sentences shorter than minLen characters with the following sentence. +func mergeShorties(sentences []string, minLen int) []string { + if len(sentences) <= 1 { + return sentences + } + + var merged []string + var buf string + + for _, s := range sentences { + if buf != "" { + buf += " " + s + if len([]rune(buf)) >= minLen { + merged = append(merged, buf) + buf = "" + } + } else if len([]rune(s)) < minLen { + buf = s + } else { + merged = append(merged, s) + } + } + + if buf != "" { + if len(merged) > 0 { + merged[len(merged)-1] += " " + buf + } else { + merged = append(merged, buf) + } + } + + return merged +} diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index ebdbd1437..69c444afa 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -3,6 +3,7 @@ package discord import ( "context" "fmt" + "io" "net/http" "net/url" "os" @@ -14,6 +15,7 @@ import ( "github.com/bwmarrin/discordgo" "github.com/gorilla/websocket" + "github.com/sipeed/picoclaw/pkg/audio" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/config" @@ -45,6 +47,10 @@ type DiscordChannel struct { botUserID string // stored for mention checking bus *bus.MessageBus tts tts.TTSProvider + + // TTS interruption: cancel active playback when user speaks + ttsMu sync.Mutex + cancelTTS context.CancelFunc } func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordChannel, error) { @@ -151,7 +157,16 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) erro if c.tts != nil { if ch, err := c.session.State.Channel(channelID); err == nil && ch.GuildID != "" { if vc, ok := c.session.VoiceConnections[ch.GuildID]; ok && vc != nil { - go c.playTTS(context.Background(), vc, msg.Content) + // Cancel any previous TTS playback + c.ttsMu.Lock() + if c.cancelTTS != nil { + c.cancelTTS() + } + ttsCtx, ttsCancel := context.WithCancel(context.Background()) + c.cancelTTS = ttsCancel + c.ttsMu.Unlock() + + go c.playTTS(ttsCtx, vc, msg.Content) } } } @@ -652,14 +667,73 @@ func (c *DiscordChannel) listenVoiceControl(ctx context.Context) { } func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnection, text string) { - stream, err := c.tts.Synthesize(ctx, text) - if err != nil { - logger.ErrorCF("discord", "TTS synthesize failed", map[string]any{"error": err.Error()}) + sentences := audio.SplitSentences(text) + if len(sentences) == 0 { return } - defer stream.Close() - if err := streamOggOpusToDiscord(vc, stream); err != nil { - logger.ErrorCF("discord", "TTS playback failed", map[string]any{"error": err.Error()}) + logger.InfoCF("discord", "Starting streamed TTS", map[string]any{"sentences": len(sentences)}) + + // Pipeline: prefetch next sentence's audio while playing current + type ttResult struct { + stream io.ReadCloser + err error + } + + var prefetch chan ttResult + + for i, sentence := range sentences { + // Check for cancellation (interruption) + select { + case <-ctx.Done(): + logger.InfoCF("discord", "TTS interrupted", map[string]any{"at_sentence": i}) + return + default: + } + + // Start prefetching the NEXT sentence while we process the current one + var nextPrefetch chan ttResult + if i+1 < len(sentences) { + nextPrefetch = make(chan ttResult, 1) + nextSentence := sentences[i+1] + go func() { + s, e := c.tts.Synthesize(ctx, nextSentence) + nextPrefetch <- ttResult{s, e} + }() + } + + // Get the current sentence's audio + var stream io.ReadCloser + var err error + + if prefetch != nil { + // Use prefetched result from previous iteration + result := <-prefetch + stream, err = result.stream, result.err + } else { + // First sentence: synthesize directly + stream, err = c.tts.Synthesize(ctx, sentence) + } + + if err != nil { + logger.ErrorCF("discord", "TTS synthesize failed", map[string]any{"error": err.Error(), "sentence": i}) + prefetch = nextPrefetch + continue + } + + if err := streamOggOpusToDiscord(ctx, vc, stream); err != nil { + logger.ErrorCF("discord", "TTS playback failed", map[string]any{"error": err.Error(), "sentence": i}) + } + stream.Close() + + prefetch = nextPrefetch + } + + // Drain any leftover prefetch + if prefetch != nil { + result := <-prefetch + if result.stream != nil { + result.stream.Close() + } } } diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index 72f767b85..4c37e952f 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -2,6 +2,7 @@ package discord import ( "bytes" + "context" "fmt" "io" "time" @@ -46,7 +47,7 @@ func VoiceReceiveActive(vc *discordgo.VoiceConnection) bool { return vc != nil && vc.OpusRecv != nil } -func streamOggOpusToDiscord(vc *discordgo.VoiceConnection, r io.Reader) error { +func streamOggOpusToDiscord(ctx context.Context, vc *discordgo.VoiceConnection, r io.Reader) error { // Wait for the speaking transition to register vc.Speaking(true) defer vc.Speaking(false) @@ -55,6 +56,13 @@ func streamOggOpusToDiscord(vc *discordgo.VoiceConnection, r io.Reader) error { header := make([]byte, 27) for { + // Check for interruption + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + if _, err := io.ReadFull(r, header); err != nil { if err == io.EOF || err == io.ErrUnexpectedEOF { return nil @@ -84,8 +92,11 @@ func streamOggOpusToDiscord(vc *discordgo.VoiceConnection, r io.Reader) error { if len(packet) > 0 { // Ignore Ogg Opus headers if !bytes.HasPrefix(packet, []byte("OpusHead")) && !bytes.HasPrefix(packet, []byte("OpusTags")) { - // Pacing is handled natively by vc.OpusSend blocking (it has an internal ticker) - vc.OpusSend <- packet + select { + case <-ctx.Done(): + return ctx.Err() + case vc.OpusSend <- packet: + } } // Start new packet packet = nil @@ -118,6 +129,8 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str }) var sequence uint64 = 0 + var interruptCount int + var lastInterruptAt time.Time for { select { @@ -139,6 +152,26 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str continue } + // Interruption detection: if user sends voice while TTS is playing, + // cancel TTS after a short debounce (3 packets in 200ms) + now := time.Now() + if now.Sub(lastInterruptAt) > 500*time.Millisecond { + interruptCount = 0 + } + interruptCount++ + lastInterruptAt = now + + if interruptCount >= 3 { + c.ttsMu.Lock() + if c.cancelTTS != nil { + c.cancelTTS() + c.cancelTTS = nil + logger.InfoCF("discord", "TTS interrupted by user voice", nil) + } + c.ttsMu.Unlock() + interruptCount = 0 + } + sequence++ chunk := bus.AudioChunk{ From 763a774c3e57c8d5435ec576cdd22f26d6066419 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 08:46:51 +0100 Subject: [PATCH 32/51] make fmt --- pkg/asr/asr.go | 16 +++++++++++++--- pkg/channels/discord/voice.go | 1 + pkg/voice/agent.go | 4 +++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/pkg/asr/asr.go b/pkg/asr/asr.go index 1e2b73500..722ca7058 100644 --- a/pkg/asr/asr.go +++ b/pkg/asr/asr.go @@ -48,7 +48,11 @@ func NewGroqTranscriber(apiKey string) *GroqTranscriber { } } -func (t *GroqTranscriber) TranscribeData(ctx context.Context, data []byte, filename string) (*TranscriptionResponse, error) { +func (t *GroqTranscriber) TranscribeData( + ctx context.Context, + data []byte, + filename string, +) (*TranscriptionResponse, error) { logger.InfoCF("voice", "Starting memory transcription", map[string]any{"filename": filename, "bytes": len(data)}) var requestBody bytes.Buffer @@ -111,7 +115,12 @@ func (t *GroqTranscriber) Transcribe(ctx context.Context, audioFilePath string) return t.doRequest(ctx, &requestBody, writer.FormDataContentType(), fileInfo.Size()) } -func (t *GroqTranscriber) doRequest(ctx context.Context, requestBody *bytes.Buffer, contentType string, fileSize int64) (*TranscriptionResponse, error) { +func (t *GroqTranscriber) doRequest( + ctx context.Context, + requestBody *bytes.Buffer, + contentType string, + fileSize int64, +) (*TranscriptionResponse, error) { url := t.apiBase + "/audio/transcriptions" req, err := http.NewRequestWithContext(ctx, "POST", url, requestBody) if err != nil { @@ -188,7 +197,8 @@ func DetectTranscriber(cfg *config.Config) Transcriber { } // Fall back to any model-list entry that uses the groq/ protocol or is explicitly named groq. for _, mc := range cfg.ModelList { - if (strings.HasPrefix(mc.Model, "groq/") || mc.ModelName == "groq" || mc.Model == "whisper-large-v3-turbo") && mc.APIKey != "" { + if (strings.HasPrefix(mc.Model, "groq/") || mc.ModelName == "groq" || mc.Model == "whisper-large-v3-turbo") && + mc.APIKey != "" { return NewGroqTranscriber(mc.APIKey) } } diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index 4c37e952f..7a6be5e7c 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -8,6 +8,7 @@ import ( "time" "github.com/bwmarrin/discordgo" + "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/logger" ) diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index 6a330d3d4..5ab19bc01 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -11,6 +11,7 @@ import ( "github.com/pion/rtp" "github.com/pion/webrtc/v3/pkg/media/oggwriter" + "github.com/sipeed/picoclaw/pkg/asr" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/logger" @@ -187,7 +188,8 @@ func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) { channelType := "discord" text := strings.ToLower(strings.TrimSpace(res.Text)) - if strings.Contains(text, "leave the voice channel") || strings.Contains(text, "leave voice") || strings.Contains(text, "disconnect voice") { + if strings.Contains(text, "leave the voice channel") || strings.Contains(text, "leave voice") || + strings.Contains(text, "disconnect voice") { logger.InfoCF("voice-agent", "Voice command triggered: leave", nil) a.bus.PublishVoiceControl(ctx, bus.VoiceControl{ SessionID: acc.sessionID, From d010c4ea84be238594213f7688c6432712b9870f Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 08:59:02 +0100 Subject: [PATCH 33/51] lint fix --- .golangci.yaml | 3 +++ pkg/asr/asr.go | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index ea3107ec8..b2b772406 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -61,6 +61,9 @@ linters: - usestdlibvars - usetesting settings: + gomoddirectives: + replace-allow-list: + - github.com/bwmarrin/discordgo errcheck: check-type-assertions: true check-blank: true diff --git a/pkg/asr/asr.go b/pkg/asr/asr.go index 722ca7058..a73887d39 100644 --- a/pkg/asr/asr.go +++ b/pkg/asr/asr.go @@ -64,9 +64,9 @@ func (t *GroqTranscriber) TranscribeData( return nil, fmt.Errorf("failed to create form file: %w", err) } - if _, err := io.Copy(part, bytes.NewReader(data)); err != nil { - logger.ErrorCF("voice", "Failed to copy file content", map[string]any{"error": err}) - return nil, fmt.Errorf("failed to copy file content: %w", err) + if _, copyErr := io.Copy(part, bytes.NewReader(data)); copyErr != nil { + logger.ErrorCF("voice", "Failed to copy file content", map[string]any{"error": copyErr}) + return nil, fmt.Errorf("failed to copy file content: %w", copyErr) } if err = writer.WriteField("model", "whisper-large-v3-turbo"); err != nil { From a3a1fdae86c89587ffaac08c7b81406cd48cb7f5 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 09:03:46 +0100 Subject: [PATCH 34/51] fix tts panic --- pkg/channels/discord/voice.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index 7a6be5e7c..8c4b84824 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -48,7 +48,14 @@ func VoiceReceiveActive(vc *discordgo.VoiceConnection) bool { return vc != nil && vc.OpusRecv != nil } -func streamOggOpusToDiscord(ctx context.Context, vc *discordgo.VoiceConnection, r io.Reader) error { +func streamOggOpusToDiscord(ctx context.Context, vc *discordgo.VoiceConnection, r io.Reader) (retErr error) { + // Recover from panic if vc.OpusSend is closed mid-send (e.g. on disconnect) + defer func() { + if rec := recover(); rec != nil { + retErr = fmt.Errorf("voice connection closed during playback") + } + }() + // Wait for the speaking transition to register vc.Speaking(true) defer vc.Speaking(false) @@ -140,6 +147,13 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str case p, ok := <-vc.OpusRecv: if !ok { logger.InfoCF("discord", "Voice channel closed", map[string]any{"guild": guildID}) + // Cancel any TTS that may still be playing + c.ttsMu.Lock() + if c.cancelTTS != nil { + c.cancelTTS() + c.cancelTTS = nil + } + c.ttsMu.Unlock() return } From 33109e862c34c3aec1f6320931086bdeba9bd04a Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 09:31:16 +0100 Subject: [PATCH 35/51] Edit from coplilot review --- pkg/bus/types.go | 1 + pkg/channels/discord/discord.go | 7 +++-- pkg/channels/discord/voice.go | 55 ++++----------------------------- pkg/voice/agent.go | 7 ++++- 4 files changed, 18 insertions(+), 52 deletions(-) diff --git a/pkg/bus/types.go b/pkg/bus/types.go index e036b4ede..794db5b0f 100644 --- a/pkg/bus/types.go +++ b/pkg/bus/types.go @@ -57,6 +57,7 @@ type AudioChunk struct { SessionID string `json:"session_id"` SpeakerID string `json:"speaker_id"` // User ID or SSRC ChatID string `json:"chat_id"` // Where to respond + Channel string `json:"channel"` // Source channel type (e.g. "discord") Sequence uint64 `json:"sequence"` Timestamp uint32 `json:"timestamp"` SampleRate int `json:"sample_rate"` diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index 69c444afa..c8f1b3f87 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -162,7 +162,7 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) erro if c.cancelTTS != nil { c.cancelTTS() } - ttsCtx, ttsCancel := context.WithCancel(context.Background()) + ttsCtx, ttsCancel := context.WithCancel(c.ctx) c.cancelTTS = ttsCancel c.ttsMu.Unlock() @@ -652,7 +652,10 @@ func (c *DiscordChannel) listenVoiceControl(ctx context.Context) { select { case <-ctx.Done(): return - case ctrl := <-c.bus.VoiceControlsChan(): + case ctrl, ok := <-c.bus.VoiceControlsChan(): + if !ok { + return + } if ctrl.Type == "command" && ctrl.Action == "leave" { if strings.HasPrefix(ctrl.SessionID, "discord_vc_") { guildID := strings.TrimPrefix(ctrl.SessionID, "discord_vc_") diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index 8c4b84824..8dd1a6b55 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -1,7 +1,6 @@ package discord import ( - "bytes" "context" "fmt" "io" @@ -9,6 +8,7 @@ import ( "github.com/bwmarrin/discordgo" + "github.com/sipeed/picoclaw/pkg/audio" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/logger" ) @@ -60,58 +60,14 @@ func streamOggOpusToDiscord(ctx context.Context, vc *discordgo.VoiceConnection, vc.Speaking(true) defer vc.Speaking(false) - var packet []byte - header := make([]byte, 27) - - for { - // Check for interruption + return audio.DecodeOggOpus(r, func(frame []byte) error { select { case <-ctx.Done(): return ctx.Err() - default: + case vc.OpusSend <- frame: + return nil } - - if _, err := io.ReadFull(r, header); err != nil { - if err == io.EOF || err == io.ErrUnexpectedEOF { - return nil - } - return fmt.Errorf("failed to read ogg header: %w", err) - } - if string(header[:4]) != "OggS" { - return fmt.Errorf("invalid ogg magic string") - } - - pageSegments := int(header[26]) - segmentTable := make([]byte, pageSegments) - if _, err := io.ReadFull(r, segmentTable); err != nil { - return fmt.Errorf("failed to read segment table: %w", err) - } - - for _, lacing := range segmentTable { - segment := make([]byte, lacing) - if _, err := io.ReadFull(r, segment); err != nil { - return fmt.Errorf("failed to read segment data: %w", err) - } - - packet = append(packet, segment...) - - // If lacing is less than 255, the packet is complete - if lacing < 255 { - if len(packet) > 0 { - // Ignore Ogg Opus headers - if !bytes.HasPrefix(packet, []byte("OpusHead")) && !bytes.HasPrefix(packet, []byte("OpusTags")) { - select { - case <-ctx.Done(): - return ctx.Err() - case vc.OpusSend <- packet: - } - } - // Start new packet - packet = nil - } - } - } - } + }) } func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID string, chatID string) { @@ -193,6 +149,7 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str SessionID: sessionID, SpeakerID: fmt.Sprintf("%d", p.SSRC), ChatID: chatID, + Channel: "discord", Sequence: sequence, Timestamp: p.Timestamp, SampleRate: 48000, diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index 5ab19bc01..ceb14540e 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -26,6 +26,7 @@ type speechAccumulator struct { chatID string speakerID string sessionID string + channel string } func (a *speechAccumulator) Push(chunk bus.AudioChunk) { @@ -117,6 +118,7 @@ func (a *Agent) handleChunk(chunk bus.AudioChunk) { chatID: chunk.ChatID, speakerID: chunk.SpeakerID, sessionID: chunk.SessionID, + channel: chunk.Channel, } a.sessions[key] = acc logger.DebugCF("voice-agent", "Started accumulating voice", map[string]any{"key": key, "file": filename}) @@ -185,7 +187,10 @@ func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) { logger.InfoCF("voice-agent", "Transcription result", map[string]any{"text": res.Text, "duration": res.Duration}) - channelType := "discord" + channelType := acc.channel + if channelType == "" { + channelType = "discord" // fallback for legacy chunks + } text := strings.ToLower(strings.TrimSpace(res.Text)) if strings.Contains(text, "leave the voice channel") || strings.Contains(text, "leave voice") || From 49970ffbabd7c70432e0d5b58c0c25e12470c6a8 Mon Sep 17 00:00:00 2001 From: Hua Audio Date: Sat, 21 Mar 2026 09:33:15 +0100 Subject: [PATCH 36/51] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/channels/discord/voice.go | 30 ++++++++++++++++++++++++------ pkg/voice/agent.go | 5 ++++- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index 8dd1a6b55..d5d5d303f 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -113,16 +113,24 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str return } + if p == nil { + logger.DebugCF("discord", "Received nil Opus packet", nil) + continue + } + + if len(p.Opus) == 0 { + logger.DebugCF("discord", "Received empty Opus packet", map[string]any{ + "seq": p.Sequence, + "ssrc": p.SSRC, + }) + continue + } + logger.DebugCF("discord", "Received Opus packet", map[string]any{ "seq": p.Sequence, "len": len(p.Opus), "ssrc": p.SSRC, }) - - if p == nil || len(p.Opus) == 0 { - continue - } - // Interruption detection: if user sends voice while TTS is playing, // cancel TTS after a short debounce (3 packets in 200ms) now := time.Now() @@ -158,7 +166,17 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str Data: p.Opus, } - c.bus.PublishAudioChunk(c.ctx, chunk) + ctx, cancel := context.WithTimeout(c.ctx, 100*time.Millisecond) + err := c.bus.PublishAudioChunk(ctx, chunk) + cancel() + if err != nil { + logger.ErrorCF("discord", "Failed to publish audio chunk", map[string]any{ + "guild": guildID, + "sessionID": sessionID, + "sequence": sequence, + "error": err.Error(), + }) + } } } } diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index ceb14540e..8e978487a 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -90,7 +90,10 @@ func (a *Agent) listenChunks(ctx context.Context) { select { case <-ctx.Done(): return - case chunk := <-chunks: + case chunk, ok := <-chunks: + if !ok { + return + } a.handleChunk(chunk) } } From 4daaeb2a4860cc0febb4f5b91cc6d0449f9d917b Mon Sep 17 00:00:00 2001 From: Hua Audio Date: Sat, 21 Mar 2026 09:42:09 +0100 Subject: [PATCH 37/51] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/bus/types.go | 2 +- pkg/channels/discord/voice.go | 39 +++++++++++++++++++++++++++---- pkg/tts/tts.go | 44 +++++++++++++++++++++++++++++++---- 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/pkg/bus/types.go b/pkg/bus/types.go index 794db5b0f..c648fb0ce 100644 --- a/pkg/bus/types.go +++ b/pkg/bus/types.go @@ -70,5 +70,5 @@ type AudioChunk struct { type VoiceControl struct { SessionID string `json:"session_id"` Type string `json:"type"` // "state", "command" - Action string `json:"action"` // "idle", "listening", "start", "stop" + Action string `json:"action"` // "idle", "listening", "start", "stop", "leave" } diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index d5d5d303f..37c882b25 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -73,17 +73,46 @@ func streamOggOpusToDiscord(ctx context.Context, vc *discordgo.VoiceConnection, func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID string, chatID string) { logger.InfoCF("discord", "Started listening for voice", map[string]any{"guild": guildID}) - go func() { + go func(ctx context.Context, vc *discordgo.VoiceConnection) { + // Recover from potential panics if OpusSend is closed mid-send. + defer func() { + if rec := recover(); rec != nil { + logger.WarnCF("discord", "Recovered from panic while sending wake-up frames", map[string]any{ + "error": rec, + "guild": guildID, + }) + } + }() + + // If the voice connection or OpusSend are not available, nothing to do. + if vc == nil || vc.OpusSend == nil { + return + } + time.Sleep(250 * time.Millisecond) // Wait a bit for connection to settle + + // Abort if the context has already been cancelled. + select { + case <-ctx.Done(): + return + default: + } + vc.Speaking(true) + defer vc.Speaking(false) + + silenceFrame := []byte{0xF8, 0xFF, 0xFE} for i := 0; i < 5; i++ { - vc.OpusSend <- []byte{0xF8, 0xFF, 0xFE} + select { + case <-ctx.Done(): + return + case vc.OpusSend <- silenceFrame: + } time.Sleep(20 * time.Millisecond) } - vc.Speaking(false) - logger.DebugCF("discord", "Sent wake-up silence frames", nil) - }() + logger.DebugCF("discord", "Sent wake-up silence frames", map[string]any{"guild": guildID}) + }(c.ctx, vc) sessionID := fmt.Sprintf("discord_vc_%s", guildID) c.bus.PublishVoiceControl(c.ctx, bus.VoiceControl{ diff --git a/pkg/tts/tts.go b/pkg/tts/tts.go index 63b4ecd24..25b533326 100644 --- a/pkg/tts/tts.go +++ b/pkg/tts/tts.go @@ -29,11 +29,47 @@ type OpenAITTSProvider struct { } func NewOpenAITTSProvider(apiKey string, apiBase string, proxyURL string) *OpenAITTSProvider { - if apiBase == "" || apiBase == "https://api.openai.com/v1" { + // Normalize apiBase to avoid malformed endpoints like + // "https://api.openai.com/audio/speech" when "/v1" is required. + if apiBase == "" { apiBase = "https://api.openai.com/v1/audio/speech" - } else if !strings.HasSuffix(apiBase, "/audio/speech") { - // Just in case they provide openrouter base or standard base - apiBase = strings.TrimSuffix(apiBase, "/") + "/audio/speech" + } else { + if u, err := url.Parse(apiBase); err == nil && u.Scheme != "" && u.Host != "" { + path := u.Path + if u.Host == "api.openai.com" { + // For the official OpenAI host, ensure exactly one /v1 prefix and + // that the path ends with /audio/speech. + if path == "" || path == "/" || path == "/v1" { + path = "/v1/audio/speech" + } else { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + if !strings.HasPrefix(path, "/v1/") { + path = "/v1" + strings.TrimSuffix(path, "/") + } + if !strings.HasSuffix(path, "/audio/speech") { + path = strings.TrimSuffix(path, "/") + "/audio/speech" + } + } + } else { + // For non-OpenAI hosts (e.g., proxies), preserve the existing base + // path and only ensure it ends with /audio/speech. + if !strings.HasSuffix(path, "/audio/speech") { + path = strings.TrimSuffix(path, "/") + "/audio/speech" + } + } + u.Path = path + apiBase = u.String() + } else { + // Fallback to the previous string-based behavior if parsing fails. + if apiBase == "https://api.openai.com/v1" { + apiBase = "https://api.openai.com/v1/audio/speech" + } else if !strings.HasSuffix(apiBase, "/audio/speech") { + // Just in case they provide openrouter base or standard base + apiBase = strings.TrimSuffix(apiBase, "/") + "/audio/speech" + } + } } client := &http.Client{ From e9cd7d952664eb7212f211beb55149c1095e44e9 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 09:54:45 +0100 Subject: [PATCH 38/51] Code review --- pkg/asr/asr.go | 27 ++++++++++++++++------ pkg/channels/discord/discord.go | 25 ++++++++++++++------- pkg/channels/discord/voice.go | 2 +- pkg/voice/agent.go | 40 +++++++++++++++++++++++++++------ 4 files changed, 71 insertions(+), 23 deletions(-) diff --git a/pkg/asr/asr.go b/pkg/asr/asr.go index a73887d39..2d2a87235 100644 --- a/pkg/asr/asr.go +++ b/pkg/asr/asr.go @@ -92,25 +92,38 @@ func (t *GroqTranscriber) Transcribe(ctx context.Context, audioFilePath string) audioFile, err := os.Open(audioFilePath) if err != nil { - return nil, fmt.Errorf("failed to open audio file: %w", err) + return nil, fmt.Errorf("failed to open audio file %s: %w", audioFilePath, err) } defer audioFile.Close() fileInfo, err := audioFile.Stat() if err != nil { - return nil, err + return nil, fmt.Errorf("failed to stat audio file %s: %w", audioFilePath, err) } var requestBody bytes.Buffer writer := multipart.NewWriter(&requestBody) + part, err := writer.CreateFormFile("file", filepath.Base(audioFilePath)) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create form file: %w", err) + } + + if _, copyErr := io.Copy(part, audioFile); copyErr != nil { + return nil, fmt.Errorf("failed to copy audio data: %w", copyErr) + } + + if err = writer.WriteField("model", "whisper-large-v3-turbo"); err != nil { + return nil, fmt.Errorf("failed to write model field: %w", err) + } + + if err = writer.WriteField("response_format", "json"); err != nil { + return nil, fmt.Errorf("failed to write response_format field: %w", err) + } + + if err = writer.Close(); err != nil { + return nil, fmt.Errorf("failed to close multipart writer: %w", err) } - io.Copy(part, audioFile) - writer.WriteField("model", "whisper-large-v3") - writer.WriteField("response_format", "json") - writer.Close() return t.doRequest(ctx, &requestBody, writer.FormDataContentType(), fileInfo.Size()) } diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index c8f1b3f87..d059379aa 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -670,6 +670,13 @@ func (c *DiscordChannel) listenVoiceControl(ctx context.Context) { } func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnection, text string) { + // Clear cancelTTS when playback finishes (normal or interrupted) + defer func() { + c.ttsMu.Lock() + c.cancelTTS = nil + c.ttsMu.Unlock() + }() + sentences := audio.SplitSentences(text) if len(sentences) == 0 { return @@ -685,6 +692,16 @@ func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnect var prefetch chan ttResult + // Ensure any in-flight prefetch is drained on exit to prevent stream leaks + defer func() { + if prefetch != nil { + result := <-prefetch + if result.stream != nil { + result.stream.Close() + } + } + }() + for i, sentence := range sentences { // Check for cancellation (interruption) select { @@ -731,12 +748,4 @@ func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnect prefetch = nextPrefetch } - - // Drain any leftover prefetch - if prefetch != nil { - result := <-prefetch - if result.stream != nil { - result.stream.Close() - } - } } diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index 37c882b25..233cfcccb 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -91,7 +91,7 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str time.Sleep(250 * time.Millisecond) // Wait a bit for connection to settle - // Abort if the context has already been cancelled. + // Abort if the context has already been canceled. select { case <-ctx.Done(): return diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index 8e978487a..0885e6b2e 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -82,6 +82,19 @@ func (a *Agent) Start(ctx context.Context) { logger.InfoCF("voice-agent", "Started Voice Agent orchestrator", nil) go a.listenChunks(ctx) go a.vadTick(ctx) + + // Cleanup sessions on shutdown + go func() { + <-ctx.Done() + a.mu.Lock() + for key, acc := range a.sessions { + acc.Close() + os.Remove(acc.file) + delete(a.sessions, key) + } + a.mu.Unlock() + logger.InfoCF("voice-agent", "Cleaned up voice sessions on shutdown", nil) + }() } func (a *Agent) listenChunks(ctx context.Context) { @@ -100,6 +113,12 @@ func (a *Agent) listenChunks(ctx context.Context) { } func (a *Agent) handleChunk(chunk bus.AudioChunk) { + // Only accept Opus-encoded audio + if chunk.Format != "opus" { + logger.DebugCF("voice-agent", "Ignoring unsupported audio format", map[string]any{"format": chunk.Format}) + return + } + a.mu.Lock() defer a.mu.Unlock() @@ -197,24 +216,29 @@ func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) { text := strings.ToLower(strings.TrimSpace(res.Text)) if strings.Contains(text, "leave the voice channel") || strings.Contains(text, "leave voice") || - strings.Contains(text, "disconnect voice") { + strings.Contains(text, "disconnect voice") || strings.Contains(text, "leave the channel") || + strings.Contains(text, "leave channel") { logger.InfoCF("voice-agent", "Voice command triggered: leave", nil) - a.bus.PublishVoiceControl(ctx, bus.VoiceControl{ + if err := a.bus.PublishVoiceControl(ctx, bus.VoiceControl{ SessionID: acc.sessionID, Type: "command", Action: "leave", - }) - a.bus.PublishOutbound(ctx, bus.OutboundMessage{ + }); err != nil { + logger.ErrorCF("voice-agent", "Failed to publish leave control", map[string]any{"error": err}) + } + if err := a.bus.PublishOutbound(ctx, bus.OutboundMessage{ Channel: channelType, ChatID: acc.chatID, Content: "Goodbye! Leaving the voice channel.", - }) + }); err != nil { + logger.ErrorCF("voice-agent", "Failed to publish goodbye message", map[string]any{"error": err}) + } return } oralPrompt := "\n\n[SYSTEM]: The user just spoke this to you over voice chat. Please reply in a highly concise, conversational, oral style suitable for text-to-speech. Do not use markdown, emojis, asterisks, or code blocks. Speak naturally." - a.bus.PublishInbound(ctx, bus.InboundMessage{ + if err := a.bus.PublishInbound(ctx, bus.InboundMessage{ Channel: channelType, SenderID: acc.speakerID, ChatID: acc.chatID, @@ -223,5 +247,7 @@ func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) { Metadata: map[string]string{ "is_voice": "true", }, - }) + }); err != nil { + logger.ErrorCF("voice-agent", "Failed to publish inbound message", map[string]any{"error": err}) + } } From 324956216b53799d95b0142c4ef4eab826185490 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 18:31:15 +0100 Subject: [PATCH 39/51] fix tool_call tts --- pkg/bus/types.go | 9 +++++---- pkg/channels/discord/discord.go | 9 ++++++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/pkg/bus/types.go b/pkg/bus/types.go index c648fb0ce..15aebc345 100644 --- a/pkg/bus/types.go +++ b/pkg/bus/types.go @@ -30,10 +30,11 @@ type InboundMessage struct { } type OutboundMessage struct { - Channel string `json:"channel"` - ChatID string `json:"chat_id"` - Content string `json:"content"` - ReplyToMessageID string `json:"reply_to_message_id,omitempty"` + Channel string `json:"channel"` + ChatID string `json:"chat_id"` + Content string `json:"content"` + ReplyToMessageID string `json:"reply_to_message_id,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` } // MediaPart describes a single media attachment to send. diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index d059379aa..38d365678 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -154,7 +154,14 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) erro return nil } - if c.tts != nil { + isToolCall := false + if msg.Metadata != nil { + if val, ok := msg.Metadata["is_tool_call"]; ok && val == "true" { + isToolCall = true + } + } + + if c.tts != nil && !isToolCall { if ch, err := c.session.State.Channel(channelID); err == nil && ch.GuildID != "" { if vc, ok := c.session.VoiceConnections[ch.GuildID]; ok && vc != nil { // Cancel any previous TTS playback From 9b768b678297ac0f691f511e9b5c004445a25c77 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 03:35:03 +0100 Subject: [PATCH 40/51] fix asr --- pkg/{voice => asr}/audio_model_transcriber.go | 2 +- pkg/{voice => asr}/audio_model_transcriber_test.go | 2 +- pkg/{voice => asr}/groq_transcriber.go | 2 +- pkg/{voice => asr}/groq_transcriber_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename pkg/{voice => asr}/audio_model_transcriber.go (99%) rename pkg/{voice => asr}/audio_model_transcriber_test.go (99%) rename pkg/{voice => asr}/groq_transcriber.go (99%) rename pkg/{voice => asr}/groq_transcriber_test.go (99%) diff --git a/pkg/voice/audio_model_transcriber.go b/pkg/asr/audio_model_transcriber.go similarity index 99% rename from pkg/voice/audio_model_transcriber.go rename to pkg/asr/audio_model_transcriber.go index f3ca81961..e8ded15dd 100644 --- a/pkg/voice/audio_model_transcriber.go +++ b/pkg/asr/audio_model_transcriber.go @@ -1,4 +1,4 @@ -package voice +package asr import ( "context" diff --git a/pkg/voice/audio_model_transcriber_test.go b/pkg/asr/audio_model_transcriber_test.go similarity index 99% rename from pkg/voice/audio_model_transcriber_test.go rename to pkg/asr/audio_model_transcriber_test.go index c33e3bf97..5aaa82061 100644 --- a/pkg/voice/audio_model_transcriber_test.go +++ b/pkg/asr/audio_model_transcriber_test.go @@ -1,4 +1,4 @@ -package voice +package asr import ( "context" diff --git a/pkg/voice/groq_transcriber.go b/pkg/asr/groq_transcriber.go similarity index 99% rename from pkg/voice/groq_transcriber.go rename to pkg/asr/groq_transcriber.go index b42e598f7..ca6a5eb5b 100644 --- a/pkg/voice/groq_transcriber.go +++ b/pkg/asr/groq_transcriber.go @@ -1,4 +1,4 @@ -package voice +package asr import ( "bytes" diff --git a/pkg/voice/groq_transcriber_test.go b/pkg/asr/groq_transcriber_test.go similarity index 99% rename from pkg/voice/groq_transcriber_test.go rename to pkg/asr/groq_transcriber_test.go index fdcaa7580..b05700d80 100644 --- a/pkg/voice/groq_transcriber_test.go +++ b/pkg/asr/groq_transcriber_test.go @@ -1,4 +1,4 @@ -package voice +package asr import ( "context" From 61b5e9221066e0a314a8ff3f2271632c893060a7 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 04:02:55 +0100 Subject: [PATCH 41/51] resolve conflicts --- pkg/agent/loop.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 05ccbe449..ed81df73d 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -18,7 +18,6 @@ import ( "sync/atomic" "time" - "github.com/sipeed/picoclaw/pkg/asr" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/commands" @@ -32,6 +31,7 @@ import ( "github.com/sipeed/picoclaw/pkg/state" "github.com/sipeed/picoclaw/pkg/tools" "github.com/sipeed/picoclaw/pkg/utils" + "github.com/sipeed/picoclaw/pkg/asr" ) type AgentLoop struct { From 1c674bb6a013a8dd31f72bedb736eef77888ffc4 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 04:10:07 +0100 Subject: [PATCH 42/51] fix lint --- pkg/agent/loop.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index ed81df73d..05ccbe449 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -18,6 +18,7 @@ import ( "sync/atomic" "time" + "github.com/sipeed/picoclaw/pkg/asr" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/commands" @@ -31,7 +32,6 @@ import ( "github.com/sipeed/picoclaw/pkg/state" "github.com/sipeed/picoclaw/pkg/tools" "github.com/sipeed/picoclaw/pkg/utils" - "github.com/sipeed/picoclaw/pkg/asr" ) type AgentLoop struct { From cbdce867dcd1cf1cb59a261de007ecbaf077891e Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 04:57:31 +0100 Subject: [PATCH 43/51] update voice system prompt override --- pkg/agent/loop.go | 39 +++++++++++++++++++++++++++------------ pkg/utils/string.go | 5 +++++ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 05ccbe449..cfffce606 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -81,6 +81,7 @@ type processOptions struct { ForcedSkills []string // Skills explicitly requested for this message SystemPromptOverride string // Override the default system prompt (Used by SubTurns) Media []string // media:// refs from inbound message + IsVoice bool // True if this message comes from an audio/voice call InitialSteeringMessages []providers.Message // Steering messages from refactor/agent DefaultResponse string // Response when LLM returns empty EnableSummary bool // Whether to trigger summarization @@ -1311,17 +1312,25 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage) "route_channel": route.Channel, }) + isVoice := msg.Metadata != nil && msg.Metadata["is_voice"] == "true" + var systemPromptOverride string + if isVoice { + systemPromptOverride = "You are a helpful AI assistant. The user is speaking to you over voice chat. Reply in a concise, conversational, and natural oral style suitable for text-to-speech. Get straight to the point. Do not use Markdown, emojis, asterisks, or code blocks." + } + opts := processOptions{ - SessionKey: sessionKey, - Channel: msg.Channel, - ChatID: msg.ChatID, - SenderID: msg.SenderID, - SenderDisplayName: msg.Sender.DisplayName, - UserMessage: msg.Content, - Media: msg.Media, - DefaultResponse: defaultResponse, - EnableSummary: true, - SendResponse: false, + SessionKey: sessionKey, + Channel: msg.Channel, + ChatID: msg.ChatID, + SenderID: msg.SenderID, + SenderDisplayName: msg.Sender.DisplayName, + UserMessage: msg.Content, + SystemPromptOverride: systemPromptOverride, + Media: msg.Media, + IsVoice: isVoice, + DefaultResponse: defaultResponse, + EnableSummary: true, + SendResponse: false, } // context-dependent commands check their own Runtime fields and report @@ -1640,7 +1649,10 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState) (turnResult, er messages = resolveMediaRefs(messages, al.mediaStore, maxMediaSize) if !ts.opts.NoHistory { - toolDefs := ts.agent.Tools.ToProviderDefs() + var toolDefs []providers.ToolDefinition + if !ts.opts.IsVoice { + toolDefs = ts.agent.Tools.ToProviderDefs() + } if isOverContextBudget(ts.agent.ContextWindow, messages, toolDefs, ts.agent.MaxTokens) { logger.WarnCF("agent", "Proactive compression: context budget exceeded before LLM call", map[string]any{"session_key": ts.sessionKey}) @@ -1779,7 +1791,10 @@ turnLoop: }) gracefulTerminal, _ := ts.gracefulInterruptRequested() - providerToolDefs := ts.agent.Tools.ToProviderDefs() + var providerToolDefs []providers.ToolDefinition + if !ts.opts.IsVoice { + providerToolDefs = ts.agent.Tools.ToProviderDefs() + } // Native web search support (from HEAD) _, hasWebSearch := ts.agent.Tools.Get("web_search") diff --git a/pkg/utils/string.go b/pkg/utils/string.go index dbaafdb7f..99337b557 100644 --- a/pkg/utils/string.go +++ b/pkg/utils/string.go @@ -65,3 +65,8 @@ func DerefStr(s *string, fallback string) string { } return *s } + +// IsTruncationDisabled returns whether truncation is disabled globally +func IsTruncationDisabled() bool { +return disableTruncation.Load() +} From 06b85ec6f9cf4860c2d0d7d1f95a6b1a1066d32a Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 13:34:56 +0100 Subject: [PATCH 44/51] Revert "update voice system prompt override" This reverts commit f237aba2a30eb77d50aec0450fed35fffb7ec70e. --- pkg/agent/loop.go | 39 ++++++++++++--------------------------- pkg/utils/string.go | 5 ----- 2 files changed, 12 insertions(+), 32 deletions(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index cfffce606..05ccbe449 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -81,7 +81,6 @@ type processOptions struct { ForcedSkills []string // Skills explicitly requested for this message SystemPromptOverride string // Override the default system prompt (Used by SubTurns) Media []string // media:// refs from inbound message - IsVoice bool // True if this message comes from an audio/voice call InitialSteeringMessages []providers.Message // Steering messages from refactor/agent DefaultResponse string // Response when LLM returns empty EnableSummary bool // Whether to trigger summarization @@ -1312,25 +1311,17 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage) "route_channel": route.Channel, }) - isVoice := msg.Metadata != nil && msg.Metadata["is_voice"] == "true" - var systemPromptOverride string - if isVoice { - systemPromptOverride = "You are a helpful AI assistant. The user is speaking to you over voice chat. Reply in a concise, conversational, and natural oral style suitable for text-to-speech. Get straight to the point. Do not use Markdown, emojis, asterisks, or code blocks." - } - opts := processOptions{ - SessionKey: sessionKey, - Channel: msg.Channel, - ChatID: msg.ChatID, - SenderID: msg.SenderID, - SenderDisplayName: msg.Sender.DisplayName, - UserMessage: msg.Content, - SystemPromptOverride: systemPromptOverride, - Media: msg.Media, - IsVoice: isVoice, - DefaultResponse: defaultResponse, - EnableSummary: true, - SendResponse: false, + SessionKey: sessionKey, + Channel: msg.Channel, + ChatID: msg.ChatID, + SenderID: msg.SenderID, + SenderDisplayName: msg.Sender.DisplayName, + UserMessage: msg.Content, + Media: msg.Media, + DefaultResponse: defaultResponse, + EnableSummary: true, + SendResponse: false, } // context-dependent commands check their own Runtime fields and report @@ -1649,10 +1640,7 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState) (turnResult, er messages = resolveMediaRefs(messages, al.mediaStore, maxMediaSize) if !ts.opts.NoHistory { - var toolDefs []providers.ToolDefinition - if !ts.opts.IsVoice { - toolDefs = ts.agent.Tools.ToProviderDefs() - } + toolDefs := ts.agent.Tools.ToProviderDefs() if isOverContextBudget(ts.agent.ContextWindow, messages, toolDefs, ts.agent.MaxTokens) { logger.WarnCF("agent", "Proactive compression: context budget exceeded before LLM call", map[string]any{"session_key": ts.sessionKey}) @@ -1791,10 +1779,7 @@ turnLoop: }) gracefulTerminal, _ := ts.gracefulInterruptRequested() - var providerToolDefs []providers.ToolDefinition - if !ts.opts.IsVoice { - providerToolDefs = ts.agent.Tools.ToProviderDefs() - } + providerToolDefs := ts.agent.Tools.ToProviderDefs() // Native web search support (from HEAD) _, hasWebSearch := ts.agent.Tools.Get("web_search") diff --git a/pkg/utils/string.go b/pkg/utils/string.go index 99337b557..dbaafdb7f 100644 --- a/pkg/utils/string.go +++ b/pkg/utils/string.go @@ -65,8 +65,3 @@ func DerefStr(s *string, fallback string) string { } return *s } - -// IsTruncationDisabled returns whether truncation is disabled globally -func IsTruncationDisabled() bool { -return disableTruncation.Load() -} From 9fe634895ea67f259f63526e895e626e443b91bd Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 13:35:07 +0100 Subject: [PATCH 45/51] fix isVoice after revert --- pkg/agent/loop.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 05ccbe449..39032156a 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -87,6 +87,7 @@ type processOptions struct { SendResponse bool // Whether to send response via bus NoHistory bool // If true, don't load session history (for heartbeat) SkipInitialSteeringPoll bool // If true, skip the steering poll at loop start (used by Continue) + IsVoice bool // If true, treat the message as voice input (transcribe before steering) } type continuationTarget struct { From 4b6fd66e045620a3592df4a9e422f8ebbd7cb936 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 14:15:51 +0100 Subject: [PATCH 46/51] add tts tool --- config/config.example.json | 3 + pkg/agent/loop.go | 17 ++++++ pkg/config/config.go | 3 + pkg/config/defaults.go | 3 + pkg/tools/tts_send.go | 115 +++++++++++++++++++++++++++++++++++++ 5 files changed, 141 insertions(+) create mode 100644 pkg/tools/tts_send.go diff --git a/config/config.example.json b/config/config.example.json index 88578701a..94cd58a19 100644 --- a/config/config.example.json +++ b/config/config.example.json @@ -522,6 +522,9 @@ "read_file": { "enabled": true }, + "send_tts": { + "enabled": false + }, "spawn": { "enabled": true }, diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 39032156a..22b9bf3d7 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -31,6 +31,7 @@ import ( "github.com/sipeed/picoclaw/pkg/skills" "github.com/sipeed/picoclaw/pkg/state" "github.com/sipeed/picoclaw/pkg/tools" + "github.com/sipeed/picoclaw/pkg/tts" "github.com/sipeed/picoclaw/pkg/utils" ) @@ -155,6 +156,13 @@ func registerSharedTools( provider providers.LLMProvider, ) { allowReadPaths := buildAllowReadPatterns(cfg) + var ttsProvider tts.TTSProvider + if cfg.Tools.IsToolEnabled("send_tts") { + ttsProvider = tts.DetectTTS(cfg) + if ttsProvider == nil { + logger.WarnCF("voice-tts", "send_tts enabled but no TTS provider configured", nil) + } + } for _, agentID := range registry.ListAgentIDs() { agent, ok := registry.GetAgent(agentID) @@ -251,6 +259,10 @@ func registerSharedTools( agent.Tools.Register(sendFileTool) } + if ttsProvider != nil { + agent.Tools.Register(tools.NewSendTTSTool(ttsProvider, nil)) + } + // Skill discovery and installation tools skills_enabled := cfg.Tools.IsToolEnabled("skills") find_skills_enable := cfg.Tools.IsToolEnabled("find_skills") @@ -1038,6 +1050,11 @@ func (al *AgentLoop) SetMediaStore(s media.MediaStore) { sf.SetMediaStore(s) } }) + registry.ForEachTool("send_tts", func(t tools.Tool) { + if st, ok := t.(*tools.SendTTSTool); ok { + st.SetMediaStore(s) + } + }) } // SetTranscriber injects a voice transcriber for agent-level audio transcription. diff --git a/pkg/config/config.go b/pkg/config/config.go index 33919d9d7..9c4e842b4 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1218,6 +1218,7 @@ type ToolsConfig struct { Message ToolConfig `json:"message" envPrefix:"PICOCLAW_TOOLS_MESSAGE_"` ReadFile ReadFileToolConfig `json:"read_file" envPrefix:"PICOCLAW_TOOLS_READ_FILE_"` SendFile ToolConfig `json:"send_file" envPrefix:"PICOCLAW_TOOLS_SEND_FILE_"` + SendTTS ToolConfig `json:"send_tts" envPrefix:"PICOCLAW_TOOLS_SEND_TTS_"` Spawn ToolConfig `json:"spawn" envPrefix:"PICOCLAW_TOOLS_SPAWN_"` SpawnStatus ToolConfig `json:"spawn_status" envPrefix:"PICOCLAW_TOOLS_SPAWN_STATUS_"` SPI ToolConfig `json:"spi" envPrefix:"PICOCLAW_TOOLS_SPI_"` @@ -2156,6 +2157,8 @@ func (t *ToolsConfig) IsToolEnabled(name string) bool { return t.WebFetch.Enabled case "send_file": return t.SendFile.Enabled + case "send_tts": + return t.SendTTS.Enabled case "write_file": return t.WriteFile.Enabled case "mcp": diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index ccfd5732a..2901c3687 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -460,6 +460,9 @@ func DefaultConfig() *Config { SendFile: ToolConfig{ Enabled: true, }, + SendTTS: ToolConfig{ + Enabled: false, + }, MCP: MCPConfig{ ToolConfig: ToolConfig{ Enabled: false, diff --git a/pkg/tools/tts_send.go b/pkg/tools/tts_send.go new file mode 100644 index 000000000..4fae77a50 --- /dev/null +++ b/pkg/tools/tts_send.go @@ -0,0 +1,115 @@ +package tools + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/sipeed/picoclaw/pkg/media" + "github.com/sipeed/picoclaw/pkg/tts" +) + +type SendTTSTool struct { + provider tts.TTSProvider + mediaStore media.MediaStore +} + +func NewSendTTSTool(provider tts.TTSProvider, store media.MediaStore) *SendTTSTool { + return &SendTTSTool{ + provider: provider, + mediaStore: store, + } +} + +func (t *SendTTSTool) Name() string { return "send_tts" } + +func (t *SendTTSTool) Description() string { + return "Synthesize speech from text and send it as an audio file to the user." +} + +func (t *SendTTSTool) Parameters() map[string]any { + return map[string]any{ + "type": "object", + "properties": map[string]any{ + "text": map[string]any{ + "type": "string", + "description": "The text to synthesize into speech.", + }, + "filename": map[string]any{ + "type": "string", + "description": "Optional filename for the audio file (e.g., response.ogg).", + }, + }, + "required": []string{"text"}, + } +} + +func (t *SendTTSTool) SetMediaStore(store media.MediaStore) { + t.mediaStore = store +} + +func (t *SendTTSTool) Execute(ctx context.Context, args map[string]any) *ToolResult { + text, _ := args["text"].(string) + text = strings.TrimSpace(text) + if text == "" { + return ErrorResult("text is required") + } + + if t.provider == nil { + return ErrorResult("tts provider is not configured") + } + if t.mediaStore == nil { + return ErrorResult("media store not configured") + } + + channel := ToolChannel(ctx) + chatID := ToolChatID(ctx) + if channel == "" || chatID == "" { + return ErrorResult("no target channel/chat available") + } + + stream, err := t.provider.Synthesize(ctx, text) + if err != nil { + return ErrorResult(fmt.Sprintf("tts synthesize failed: %v", err)).WithError(err) + } + defer stream.Close() + + if err := os.MkdirAll(media.TempDir(), 0o755); err != nil { + return ErrorResult(fmt.Sprintf("failed to create media temp dir: %v", err)).WithError(err) + } + + file, err := os.CreateTemp(media.TempDir(), "tts-*.ogg") + if err != nil { + return ErrorResult(fmt.Sprintf("failed to create temp file: %v", err)).WithError(err) + } + defer file.Close() + + if _, err := io.Copy(file, stream); err != nil { + return ErrorResult(fmt.Sprintf("failed to write tts audio: %v", err)).WithError(err) + } + + filename, _ := args["filename"].(string) + filename = strings.TrimSpace(filename) + if filename == "" { + filename = fmt.Sprintf("tts-%d.ogg", time.Now().Unix()) + } + if filepath.Ext(filename) == "" { + filename += ".ogg" + } + + scope := fmt.Sprintf("tool:send_tts:%s:%s:%d", channel, chatID, time.Now().UnixNano()) + ref, err := t.mediaStore.Store(file.Name(), media.MediaMeta{ + Filename: filename, + ContentType: "audio/ogg", + Source: "tool:send_tts", + }, scope) + if err != nil { + return ErrorResult(fmt.Sprintf("failed to register audio: %v", err)).WithError(err) + } + + return MediaResult("TTS audio sent", []string{ref}) +} From 29373f121d07ea04553e037168fa9a8446ecffe3 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 15:03:10 +0100 Subject: [PATCH 47/51] remove voice history cut and fix lint --- pkg/agent/loop.go | 1 - pkg/tools/tts_send.go | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 22b9bf3d7..574322043 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -88,7 +88,6 @@ type processOptions struct { SendResponse bool // Whether to send response via bus NoHistory bool // If true, don't load session history (for heartbeat) SkipInitialSteeringPoll bool // If true, skip the steering poll at loop start (used by Continue) - IsVoice bool // If true, treat the message as voice input (transcribe before steering) } type continuationTarget struct { diff --git a/pkg/tools/tts_send.go b/pkg/tools/tts_send.go index 4fae77a50..71dd7e593 100644 --- a/pkg/tools/tts_send.go +++ b/pkg/tools/tts_send.go @@ -78,7 +78,8 @@ func (t *SendTTSTool) Execute(ctx context.Context, args map[string]any) *ToolRes } defer stream.Close() - if err := os.MkdirAll(media.TempDir(), 0o755); err != nil { + err = os.MkdirAll(media.TempDir(), 0o755) + if err != nil { return ErrorResult(fmt.Sprintf("failed to create media temp dir: %v", err)).WithError(err) } @@ -88,7 +89,8 @@ func (t *SendTTSTool) Execute(ctx context.Context, args map[string]any) *ToolRes } defer file.Close() - if _, err := io.Copy(file, stream); err != nil { + _, err = io.Copy(file, stream) + if err != nil { return ErrorResult(fmt.Sprintf("failed to write tts audio: %v", err)).WithError(err) } From 3e26c37db115aac9ceeda85364b9517fff2b563e Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 15:20:56 +0100 Subject: [PATCH 48/51] fix provider --- pkg/asr/asr.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/asr/asr.go b/pkg/asr/asr.go index 7b43f317f..2d607fec4 100644 --- a/pkg/asr/asr.go +++ b/pkg/asr/asr.go @@ -54,15 +54,11 @@ func DetectTranscriber(cfg *config.Config) Transcriber { } } - // Direct Groq provider config takes priority. - if key := cfg.Providers.Groq.APIKey; key != "" { - return NewGroqTranscriber(key) - } // Fall back to any model-list entry that uses the groq/ protocol or is explicitly named groq. for _, mc := range cfg.ModelList { if (strings.HasPrefix(mc.Model, "groq/") || mc.ModelName == "groq" || mc.Model == "whisper-large-v3-turbo") && - mc.APIKey != "" { - return NewGroqTranscriber(mc.APIKey) + mc.APIKey() != "" { + return NewGroqTranscriber(mc.APIKey()) } } return nil From a88b539402958a26dbc475bb0fb2d4a05f31af23 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 15:25:41 +0100 Subject: [PATCH 49/51] fix tts provider and lint --- pkg/logger/panic_win.go | 2 +- pkg/tts/tts.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/logger/panic_win.go b/pkg/logger/panic_win.go index 29d3f21d8..1e6eead02 100644 --- a/pkg/logger/panic_win.go +++ b/pkg/logger/panic_win.go @@ -12,7 +12,7 @@ import ( ) func initPanicFile(panicFile string) io.WriteCloser { - file, err := os.OpenFile(panicFile, os.O_WRONLY|os.O_CREATE|os.O_SYNC|os.O_APPEND, 0600) + file, err := os.OpenFile(panicFile, os.O_WRONLY|os.O_CREATE|os.O_SYNC|os.O_APPEND, 0o600) if err != nil { panic(fmt.Sprintf("error in open panic: %v", err)) } diff --git a/pkg/tts/tts.go b/pkg/tts/tts.go index 25b533326..b43d0bbb9 100644 --- a/pkg/tts/tts.go +++ b/pkg/tts/tts.go @@ -136,8 +136,8 @@ func (t *OpenAITTSProvider) Synthesize(ctx context.Context, text string) (io.Rea func DetectTTS(cfg *config.Config) TTSProvider { for _, mc := range cfg.ModelList { - if strings.Contains(strings.ToLower(mc.ModelName), "tts") && mc.APIKey != "" { - return NewOpenAITTSProvider(mc.APIKey, mc.APIBase, mc.Proxy) + if strings.Contains(strings.ToLower(mc.ModelName), "tts") && mc.APIKey() != "" { + return NewOpenAITTSProvider(mc.APIKey(), mc.APIBase, mc.Proxy) } } return nil From 8fa2d7bb0ea56ebbbc405d43fdebcac575d100d2 Mon Sep 17 00:00:00 2001 From: Hua Audio Date: Mon, 23 Mar 2026 15:44:14 +0100 Subject: [PATCH 50/51] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/audio/sentence.go | 2 +- pkg/channels/discord/discord.go | 25 +++++++++++++++----- pkg/channels/discord/voice.go | 42 ++++++++++++++++++++++++++++----- pkg/tts/tts.go | 2 ++ pkg/voice/agent.go | 5 ++-- 5 files changed, 61 insertions(+), 15 deletions(-) diff --git a/pkg/audio/sentence.go b/pkg/audio/sentence.go index c7a9b2f26..6a0c2c0b0 100644 --- a/pkg/audio/sentence.go +++ b/pkg/audio/sentence.go @@ -7,7 +7,7 @@ import ( // SplitSentences splits text into sentence-sized chunks suitable for TTS synthesis. // It splits on sentence-ending punctuation (.!?\n) while avoiding false splits -// on abbreviations and decimal numbers. Very short fragments are merged with +// on decimal numbers. Very short fragments are merged with // the next sentence to prevent choppy playback. func SplitSentences(text string) []string { if text == "" { diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index 38d365678..5df027e45 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -677,10 +677,18 @@ func (c *DiscordChannel) listenVoiceControl(ctx context.Context) { } func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnection, text string) { - // Clear cancelTTS when playback finishes (normal or interrupted) + // Capture the cancel func associated with this playback (if any). + c.ttsMu.Lock() + playbackCancel := c.cancelTTS + c.ttsMu.Unlock() + + // Clear cancelTTS when playback finishes (normal or interrupted), + // but only if it still refers to this playback's cancel func. defer func() { c.ttsMu.Lock() - c.cancelTTS = nil + if c.cancelTTS == playbackCancel { + c.cancelTTS = nil + } c.ttsMu.Unlock() }() @@ -699,12 +707,17 @@ func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnect var prefetch chan ttResult - // Ensure any in-flight prefetch is drained on exit to prevent stream leaks + // Ensure any in-flight prefetch is drained on exit to prevent stream leaks, + // but avoid blocking indefinitely if the prefetch goroutine is stuck or never sends. defer func() { if prefetch != nil { - result := <-prefetch - if result.stream != nil { - result.stream.Close() + select { + case result := <-prefetch: + if result.stream != nil { + result.stream.Close() + } + default: + // No prefetched result available to drain; avoid blocking on exit. } } }() diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index 233cfcccb..ce5f7c4b3 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -17,27 +17,57 @@ func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.M if m.Content == "!vc join" { vs, err := s.State.VoiceState(m.GuildID, m.Author.ID) if err != nil || vs == nil { - s.ChannelMessageSend(m.ChannelID, "You need to be in a voice channel first!") + if _, sendErr := s.ChannelMessageSend(m.ChannelID, "You need to be in a voice channel first!"); sendErr != nil { + logger.InfoCF("discord", "Failed to send voice channel requirement message", map[string]any{ + "channel": m.ChannelID, + "error": sendErr, + }) + } return true } logger.InfoCF("discord", "Joining voice channel", map[string]any{"channel": vs.ChannelID}) vc, err := s.ChannelVoiceJoin(c.ctx, m.GuildID, vs.ChannelID, false, false) if err != nil { - s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("Failed to join voice channel: %v", err)) + if _, sendErr := s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("Failed to join voice channel: %v", err)); sendErr != nil { + logger.InfoCF("discord", "Failed to send voice join error message", map[string]any{ + "channel": m.ChannelID, + "error": sendErr, + }) + } return true } go c.receiveVoice(vc, m.GuildID, m.ChannelID) - s.ChannelMessageSend(m.ChannelID, "Joined Voice Channel! Listening for audio...") + if _, sendErr := s.ChannelMessageSend(m.ChannelID, "Joined Voice Channel! Listening for audio..."); sendErr != nil { + logger.InfoCF("discord", "Failed to send voice join success message", map[string]any{ + "channel": m.ChannelID, + "error": sendErr, + }) + } return true } else if m.Content == "!vc leave" { vc, exists := s.VoiceConnections[m.GuildID] if exists && vc != nil { - vc.Disconnect(c.ctx) - s.ChannelMessageSend(m.ChannelID, "Left Voice Channel.") + if err := vc.Disconnect(c.ctx); err != nil { + logger.InfoCF("discord", "Failed to disconnect from voice channel", map[string]any{ + "guild": m.GuildID, + "error": err, + }) + } + if _, sendErr := s.ChannelMessageSend(m.ChannelID, "Left Voice Channel."); sendErr != nil { + logger.InfoCF("discord", "Failed to send voice leave success message", map[string]any{ + "channel": m.ChannelID, + "error": sendErr, + }) + } } else { - s.ChannelMessageSend(m.ChannelID, "Not in a voice channel.") + if _, sendErr := s.ChannelMessageSend(m.ChannelID, "Not in a voice channel."); sendErr != nil { + logger.InfoCF("discord", "Failed to send voice not-in-channel message", map[string]any{ + "channel": m.ChannelID, + "error": sendErr, + }) + } } return true } diff --git a/pkg/tts/tts.go b/pkg/tts/tts.go index b43d0bbb9..07da18bb8 100644 --- a/pkg/tts/tts.go +++ b/pkg/tts/tts.go @@ -81,6 +81,8 @@ func NewOpenAITTSProvider(apiKey string, apiBase string, proxyURL string) *OpenA client.Transport = &http.Transport{ Proxy: http.ProxyURL(pURL), } + } else { + logger.Warnf("NewOpenAITTSProvider: invalid proxy URL %q: %v; proceeding without proxy", proxyURL, err) } } diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index 0885e6b2e..f60e82be4 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -242,10 +242,11 @@ func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) { Channel: channelType, SenderID: acc.speakerID, ChatID: acc.chatID, - Content: res.Text + oralPrompt, + Content: res.Text, Peer: bus.Peer{Kind: "channel", ID: acc.chatID}, Metadata: map[string]string{ - "is_voice": "true", + "is_voice": "true", + "oral_prompt": oralPrompt, }, }); err != nil { logger.ErrorCF("voice-agent", "Failed to publish inbound message", map[string]any{"error": err}) From 5668545383f16f885a4870dd0fb6e1ad58bef807 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Mon, 23 Mar 2026 16:08:20 +0100 Subject: [PATCH 51/51] Update suggested fixes --- pkg/agent/loop.go | 3 ++ pkg/channels/discord/discord.go | 25 ++++------ pkg/channels/discord/voice.go | 84 ++++++++++++++++++++++++++++++--- pkg/gateway/gateway.go | 17 ++++--- pkg/tools/tts_send.go | 7 ++- pkg/tts/tts.go | 5 +- pkg/voice/agent.go | 11 ++--- 7 files changed, 116 insertions(+), 36 deletions(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 574322043..334d904cf 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -2435,6 +2435,9 @@ turnLoop: Channel: ts.channel, ChatID: ts.chatID, Content: toolResult.ForUser, + Metadata: map[string]string{ + "is_tool_call": "true", + }, }) logger.DebugCF("agent", "Sent tool result to user", map[string]any{ diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index 5df027e45..a6ff8a332 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -47,10 +47,13 @@ type DiscordChannel struct { botUserID string // stored for mention checking bus *bus.MessageBus tts tts.TTSProvider + voiceMu sync.RWMutex + voiceSSRC map[string]map[uint32]string // guildID -> ssrc -> userID // TTS interruption: cancel active playback when user speaks ttsMu sync.Mutex cancelTTS context.CancelFunc + ttsPlayID uint64 } func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordChannel, error) { @@ -83,6 +86,7 @@ func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordC ctx: context.Background(), typingStop: make(map[string]chan struct{}), bus: bus, + voiceSSRC: make(map[string]map[uint32]string), }, nil } @@ -154,14 +158,7 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) erro return nil } - isToolCall := false - if msg.Metadata != nil { - if val, ok := msg.Metadata["is_tool_call"]; ok && val == "true" { - isToolCall = true - } - } - - if c.tts != nil && !isToolCall { + if c.tts != nil { if ch, err := c.session.State.Channel(channelID); err == nil && ch.GuildID != "" { if vc, ok := c.session.VoiceConnections[ch.GuildID]; ok && vc != nil { // Cancel any previous TTS playback @@ -170,10 +167,12 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) erro c.cancelTTS() } ttsCtx, ttsCancel := context.WithCancel(c.ctx) + c.ttsPlayID++ + playID := c.ttsPlayID c.cancelTTS = ttsCancel c.ttsMu.Unlock() - go c.playTTS(ttsCtx, vc, msg.Content) + go c.playTTS(ttsCtx, vc, msg.Content, playID) } } } @@ -676,17 +675,13 @@ func (c *DiscordChannel) listenVoiceControl(ctx context.Context) { } } -func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnection, text string) { +func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnection, text string, playID uint64) { // Capture the cancel func associated with this playback (if any). - c.ttsMu.Lock() - playbackCancel := c.cancelTTS - c.ttsMu.Unlock() - // Clear cancelTTS when playback finishes (normal or interrupted), // but only if it still refers to this playback's cancel func. defer func() { c.ttsMu.Lock() - if c.cancelTTS == playbackCancel { + if c.ttsPlayID == playID { c.cancelTTS = nil } c.ttsMu.Unlock() diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index ce5f7c4b3..5b686b141 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -10,14 +10,45 @@ import ( "github.com/sipeed/picoclaw/pkg/audio" "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/identity" "github.com/sipeed/picoclaw/pkg/logger" ) +func (c *DiscordChannel) setVoiceUserID(guildID string, ssrc uint32, userID string) { + if userID == "" { + return + } + + c.voiceMu.Lock() + defer c.voiceMu.Unlock() + + ssrcMap, ok := c.voiceSSRC[guildID] + if !ok { + ssrcMap = make(map[uint32]string) + c.voiceSSRC[guildID] = ssrcMap + } + ssrcMap[ssrc] = userID +} + +func (c *DiscordChannel) voiceUserID(guildID string, ssrc uint32) string { + c.voiceMu.RLock() + defer c.voiceMu.RUnlock() + + ssrcMap, ok := c.voiceSSRC[guildID] + if !ok { + return "" + } + return ssrcMap[ssrc] +} + func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.MessageCreate) bool { if m.Content == "!vc join" { vs, err := s.State.VoiceState(m.GuildID, m.Author.ID) if err != nil || vs == nil { - if _, sendErr := s.ChannelMessageSend(m.ChannelID, "You need to be in a voice channel first!"); sendErr != nil { + if _, sendErr := s.ChannelMessageSend( + m.ChannelID, + "You need to be in a voice channel first!", + ); sendErr != nil { logger.InfoCF("discord", "Failed to send voice channel requirement message", map[string]any{ "channel": m.ChannelID, "error": sendErr, @@ -29,7 +60,10 @@ func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.M logger.InfoCF("discord", "Joining voice channel", map[string]any{"channel": vs.ChannelID}) vc, err := s.ChannelVoiceJoin(c.ctx, m.GuildID, vs.ChannelID, false, false) if err != nil { - if _, sendErr := s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("Failed to join voice channel: %v", err)); sendErr != nil { + if _, sendErr := s.ChannelMessageSend( + m.ChannelID, + fmt.Sprintf("Failed to join voice channel: %v", err), + ); sendErr != nil { logger.InfoCF("discord", "Failed to send voice join error message", map[string]any{ "channel": m.ChannelID, "error": sendErr, @@ -39,7 +73,10 @@ func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.M } go c.receiveVoice(vc, m.GuildID, m.ChannelID) - if _, sendErr := s.ChannelMessageSend(m.ChannelID, "Joined Voice Channel! Listening for audio..."); sendErr != nil { + if _, sendErr := s.ChannelMessageSend( + m.ChannelID, + "Joined Voice Channel! Listening for audio...", + ); sendErr != nil { logger.InfoCF("discord", "Failed to send voice join success message", map[string]any{ "channel": m.ChannelID, "error": sendErr, @@ -51,8 +88,8 @@ func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.M if exists && vc != nil { if err := vc.Disconnect(c.ctx); err != nil { logger.InfoCF("discord", "Failed to disconnect from voice channel", map[string]any{ - "guild": m.GuildID, - "error": err, + "guild": m.GuildID, + "error": err, }) } if _, sendErr := s.ChannelMessageSend(m.ChannelID, "Left Voice Channel."); sendErr != nil { @@ -103,6 +140,19 @@ func streamOggOpusToDiscord(ctx context.Context, vc *discordgo.VoiceConnection, func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID string, chatID string) { logger.InfoCF("discord", "Started listening for voice", map[string]any{"guild": guildID}) + vc.AddHandler(func(_ *discordgo.VoiceConnection, vs *discordgo.VoiceSpeakingUpdate) { + if vs == nil { + return + } + c.setVoiceUserID(guildID, uint32(vs.SSRC), vs.UserID) + }) + + defer func() { + c.voiceMu.Lock() + delete(c.voiceSSRC, guildID) + c.voiceMu.Unlock() + }() + go func(ctx context.Context, vc *discordgo.VoiceConnection) { // Recover from potential panics if OpusSend is closed mid-send. defer func() { @@ -210,11 +260,33 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str interruptCount = 0 } + userID := c.voiceUserID(guildID, p.SSRC) + if userID == "" { + logger.DebugCF("discord", "Dropping voice packet without user mapping", map[string]any{ + "ssrc": p.SSRC, + "guild": guildID, + }) + continue + } + + sender := bus.SenderInfo{ + Platform: "discord", + PlatformID: userID, + CanonicalID: identity.BuildCanonicalID("discord", userID), + } + if !c.IsAllowedSender(sender) { + logger.DebugCF("discord", "Voice packet rejected by allowlist", map[string]any{ + "user_id": userID, + "guild": guildID, + }) + continue + } + sequence++ chunk := bus.AudioChunk{ SessionID: sessionID, - SpeakerID: fmt.Sprintf("%d", p.SSRC), + SpeakerID: userID, ChatID: chatID, Channel: "discord", Sequence: sequence, diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go index ad7591e3a..d2f821f8f 100644 --- a/pkg/gateway/gateway.go +++ b/pkg/gateway/gateway.go @@ -304,15 +304,10 @@ func setupAndStartServices( agentLoop.SetChannelManager(runningServices.ChannelManager) agentLoop.SetMediaStore(runningServices.MediaStore) - if transcriber := asr.DetectTranscriber(cfg); transcriber != nil { + transcriber := asr.DetectTranscriber(cfg) + if transcriber != nil { agentLoop.SetTranscriber(transcriber) logger.InfoCF("voice", "Transcription enabled (agent-level)", map[string]any{"provider": transcriber.Name()}) - - // Start Voice Agent Orchestrator - vaCtx, vaCancel := context.WithCancel(context.Background()) - runningServices.VoiceAgentCancel = vaCancel - voiceAgent := voice.NewAgent(msgBus, transcriber) - voiceAgent.Start(vaCtx) } enabledChannels := runningServices.ChannelManager.GetEnabledChannels() @@ -330,6 +325,14 @@ func setupAndStartServices( return nil, fmt.Errorf("error starting channels: %w", err) } + if transcriber != nil { + // Start Voice Agent Orchestrator after channels are ready. + vaCtx, vaCancel := context.WithCancel(context.Background()) + runningServices.VoiceAgentCancel = vaCancel + voiceAgent := voice.NewAgent(msgBus, transcriber) + voiceAgent.Start(vaCtx) + } + fmt.Printf( "✓ Health endpoints available at http://%s:%d/health, /ready and /reload (POST)\n", cfg.Gateway.Host, diff --git a/pkg/tools/tts_send.go b/pkg/tools/tts_send.go index 71dd7e593..27219630e 100644 --- a/pkg/tools/tts_send.go +++ b/pkg/tools/tts_send.go @@ -87,13 +87,18 @@ func (t *SendTTSTool) Execute(ctx context.Context, args map[string]any) *ToolRes if err != nil { return ErrorResult(fmt.Sprintf("failed to create temp file: %v", err)).WithError(err) } - defer file.Close() _, err = io.Copy(file, stream) if err != nil { + file.Close() return ErrorResult(fmt.Sprintf("failed to write tts audio: %v", err)).WithError(err) } + err = file.Close() + if err != nil { + return ErrorResult(fmt.Sprintf("failed to close tts audio file: %v", err)).WithError(err) + } + filename, _ := args["filename"].(string) filename = strings.TrimSpace(filename) if filename == "" { diff --git a/pkg/tts/tts.go b/pkg/tts/tts.go index 07da18bb8..eca4d28ee 100644 --- a/pkg/tts/tts.go +++ b/pkg/tts/tts.go @@ -82,7 +82,10 @@ func NewOpenAITTSProvider(apiKey string, apiBase string, proxyURL string) *OpenA Proxy: http.ProxyURL(pURL), } } else { - logger.Warnf("NewOpenAITTSProvider: invalid proxy URL %q: %v; proceeding without proxy", proxyURL, err) + logger.WarnF( + "NewOpenAITTSProvider: invalid proxy URL; proceeding without proxy", + map[string]any{"proxyURL": proxyURL, "error": err}, + ) } } diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index f60e82be4..cd0a62ecd 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -119,16 +119,15 @@ func (a *Agent) handleChunk(chunk bus.AudioChunk) { return } - a.mu.Lock() - defer a.mu.Unlock() - key := fmt.Sprintf("%s_%s", chunk.SessionID, chunk.SpeakerID) + a.mu.Lock() acc, exists := a.sessions[key] if !exists { filename := filepath.Join(os.TempDir(), fmt.Sprintf("voice_%s_%d.ogg", key, time.Now().UnixNano())) writer, err := oggwriter.New(filename, uint32(chunk.SampleRate), uint16(chunk.Channels)) if err != nil { + a.mu.Unlock() logger.ErrorCF("voice-agent", "Failed to create OggWriter", map[string]any{"error": err}) return } @@ -145,6 +144,7 @@ func (a *Agent) handleChunk(chunk bus.AudioChunk) { a.sessions[key] = acc logger.DebugCF("voice-agent", "Started accumulating voice", map[string]any{"key": key, "file": filename}) } + a.mu.Unlock() acc.Push(chunk) } @@ -242,11 +242,10 @@ func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) { Channel: channelType, SenderID: acc.speakerID, ChatID: acc.chatID, - Content: res.Text, + Content: res.Text + oralPrompt, Peer: bus.Peer{Kind: "channel", ID: acc.chatID}, Metadata: map[string]string{ - "is_voice": "true", - "oral_prompt": oralPrompt, + "is_voice": "true", }, }); err != nil { logger.ErrorCF("voice-agent", "Failed to publish inbound message", map[string]any{"error": err})