From d632a0e3f3895c0d8934157161c1bf50aa1b659e Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Sat, 21 Mar 2026 07:18:14 +0100 Subject: [PATCH] update with corrent transcriber and leave_tool --- go.mod | 3 ++ go.sum | 7 +++-- pkg/agent/loop.go | 3 ++ pkg/bus/types.go | 3 +- pkg/channels/discord/discord.go | 30 +++++++++++++++++++ pkg/channels/discord/voice.go | 24 +++++++++++++-- pkg/tools/voice_leave.go | 52 +++++++++++++++++++++++++++++++++ pkg/voice/agent.go | 7 ++++- pkg/voice/transcriber.go | 4 +-- 9 files changed, 123 insertions(+), 10 deletions(-) create mode 100644 pkg/tools/voice_leave.go diff --git a/go.mod b/go.mod index b209e446f..20e413b82 100644 --- a/go.mod +++ b/go.mod @@ -59,6 +59,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect github.com/aws/smithy-go v1.24.2 // indirect github.com/beeper/argo-go v1.1.2 // indirect + github.com/cloudflare/circl v1.6.3 // indirect github.com/coder/websocket v1.8.14 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -117,3 +118,5 @@ require ( golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 ) + +replace github.com/bwmarrin/discordgo => github.com/yeongaori/discordgo-fork v0.0.0-20260319072544-e8e546f5d532 diff --git a/go.sum b/go.sum index 6e4ac41f5..1c8309c02 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,6 @@ github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beeper/argo-go v1.1.2 h1:UQI2G8F+NLfGTOmTUI0254pGKx/HUU/etbUGTJv91Fs= github.com/beeper/argo-go v1.1.2/go.mod h1:M+LJAnyowKVQ6Rdj6XYGEn+qcVFkb3R/MUpqkGR0hM4= -github.com/bwmarrin/discordgo v0.29.0 h1:FmWeXFaKUwrcL3Cx65c20bTRW+vOb6k8AnaP+EgjDno= -github.com/bwmarrin/discordgo v0.29.0/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -63,6 +61,8 @@ github.com/caarlos0/env/v11 v11.4.0 h1:Kcb6t5kIIr4XkoQC9AF2j+8E1Jsrl3Wz/hhm1LtoG github.com/caarlos0/env/v11 v11.4.0/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g= @@ -268,6 +268,8 @@ github.com/vektah/gqlparser/v2 v2.5.27 h1:RHPD3JOplpk5mP5JGX8RKZkt2/Vwj/PZv0HxTd github.com/vektah/gqlparser/v2 v2.5.27/go.mod h1:D1/VCZtV3LPnQrcPBeR/q5jkSQIPti0uYCP/RI0gIeo= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yeongaori/discordgo-fork v0.0.0-20260319072544-e8e546f5d532 h1:gxFHYeUDGziRb0zXYEqBFohC+NJbIW9L0tddaXMWr2o= +github.com/yeongaori/discordgo-fork v0.0.0-20260319072544-e8e546f5d532/go.mod h1:A0FcMFJKJ9fRjgSuZ2o+pIQ6mPS81SVuiLN2vYTa7Ao= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -287,7 +289,6 @@ golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 725d42614..aa15a23d5 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -238,6 +238,9 @@ func registerSharedTools( agent.Tools.Register(messageTool) } + // Always register Voice Leave Tool (it inherently checks if channel == "discord") + agent.Tools.Register(tools.NewVoiceLeaveTool(msgBus)) + // Send file tool (outbound media via MediaStore — store injected later by SetMediaStore) if cfg.Tools.IsToolEnabled("send_file") { sendFileTool := tools.NewSendFileTool( diff --git a/pkg/bus/types.go b/pkg/bus/types.go index 9c637f3e7..b4e99d955 100644 --- a/pkg/bus/types.go +++ b/pkg/bus/types.go @@ -68,6 +68,7 @@ type AudioChunk struct { // VoiceControl represents state or commands for voice sessions. type VoiceControl struct { SessionID string `json:"session_id"` - Type string `json:"type"` // "state", "command" + ChatID string `json:"chat_id"` + Type string `json:"type"` // "state", "command" Action string `json:"action"` // "idle", "listening", "start", "stop" } diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index 49121d3ad..6194685e8 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -91,6 +91,8 @@ func (c *DiscordChannel) Start(ctx context.Context) error { c.botUserID = botUser.ID c.session.AddHandler(c.handleMessage) + + go c.listenVoiceControl(c.ctx) if err := c.session.Open(); err != nil { return fmt.Errorf("failed to open discord session: %w", err) @@ -619,3 +621,31 @@ func (c *DiscordChannel) stripBotMention(text string) string { text = strings.ReplaceAll(text, fmt.Sprintf("<@!%s>", c.botUserID), "") return strings.TrimSpace(text) } + +func (c *DiscordChannel) listenVoiceControl(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case ctrl := <-c.bus.VoiceControlsChan(): + if ctrl.Type == "command" && ctrl.Action == "leave" { + var guildID string + if strings.HasPrefix(ctrl.SessionID, "discord_vc_") { + guildID = strings.TrimPrefix(ctrl.SessionID, "discord_vc_") + } else if ctrl.ChatID != "" { + ch, err := c.session.State.Channel(ctrl.ChatID) + if err == nil { + guildID = ch.GuildID + } + } + + if guildID != "" { + vc, exists := c.session.VoiceConnections[guildID] + if exists && vc != nil { + vc.Disconnect(ctx) + } + } + } + } + } +} diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index e2aae52ed..23386e642 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -2,6 +2,7 @@ package discord import ( "fmt" + "time" "github.com/bwmarrin/discordgo" "github.com/sipeed/picoclaw/pkg/bus" @@ -17,7 +18,7 @@ func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.M } logger.InfoCF("discord", "Joining voice channel", map[string]any{"channel": vs.ChannelID}) - vc, err := s.ChannelVoiceJoin(m.GuildID, vs.ChannelID, false, false) + vc, err := s.ChannelVoiceJoin(c.ctx, m.GuildID, vs.ChannelID, false, false) if err != nil { s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("Failed to join voice channel: %v", err)) return true @@ -29,7 +30,7 @@ func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.M } else if m.Content == "!vc leave" { vc, exists := s.VoiceConnections[m.GuildID] if exists && vc != nil { - vc.Disconnect() + vc.Disconnect(c.ctx) s.ChannelMessageSend(m.ChannelID, "Left Voice Channel.") } else { s.ChannelMessageSend(m.ChannelID, "Not in a voice channel.") @@ -42,6 +43,17 @@ func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.M func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID string, chatID string) { logger.InfoCF("discord", "Started listening for voice", map[string]any{"guild": guildID}) + go func() { + time.Sleep(250 * time.Millisecond) // Wait a bit for connection to settle + vc.Speaking(true) + for i := 0; i < 5; i++ { + vc.OpusSend <- []byte{0xF8, 0xFF, 0xFE} + time.Sleep(20 * time.Millisecond) + } + vc.Speaking(false) + logger.DebugCF("discord", "Sent wake-up silence frames", nil) + }() + sessionID := fmt.Sprintf("discord_vc_%s", guildID) c.bus.PublishVoiceControl(c.ctx, bus.VoiceControl{ @@ -62,7 +74,13 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str return } - if p == nil { + logger.DebugCF("discord", "Received Opus packet", map[string]any{ + "seq": p.Sequence, + "len": len(p.Opus), + "ssrc": p.SSRC, + }) + + if p == nil || len(p.Opus) == 0 { continue } diff --git a/pkg/tools/voice_leave.go b/pkg/tools/voice_leave.go new file mode 100644 index 000000000..882e7cb9d --- /dev/null +++ b/pkg/tools/voice_leave.go @@ -0,0 +1,52 @@ +package tools + +import ( + "context" + + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/logger" +) + +type VoiceLeaveTool struct { + bus *bus.MessageBus +} + +func NewVoiceLeaveTool(mb *bus.MessageBus) *VoiceLeaveTool { + return &VoiceLeaveTool{bus: mb} +} + +func (t *VoiceLeaveTool) Name() string { + return "voice_leave" +} + +func (t *VoiceLeaveTool) Description() string { + return "Disconnects the bot from the current voice channel. Use this tool when the user says goodbye or explicitly asks you to leave the voice chat." +} + +func (t *VoiceLeaveTool) Parameters() map[string]any { + return map[string]any{ + "type": "object", + "properties": map[string]any{}, + } +} + +func (t *VoiceLeaveTool) Execute(ctx context.Context, args map[string]any) *ToolResult { + channel := ToolChannel(ctx) + chatID := ToolChatID(ctx) + + if channel != "discord" { + return &ToolResult{ForLLM: "Can only leave voice channels on Discord", IsError: true} + } + + t.bus.PublishVoiceControl(ctx, bus.VoiceControl{ + ChatID: chatID, + Type: "command", + Action: "leave", + }) + + logger.InfoCF("agent", "Voice command triggered via tool: leave", map[string]any{"chat_id": chatID}) + + return &ToolResult{ + ForLLM: "Successfully sent disconnect command to voice adapter.", + } +} diff --git a/pkg/voice/agent.go b/pkg/voice/agent.go index 4362eb863..40bc02c57 100644 --- a/pkg/voice/agent.go +++ b/pkg/voice/agent.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "sync" "time" @@ -22,6 +23,7 @@ type speechAccumulator struct { closed bool chatID string speakerID string + sessionID string } func (a *speechAccumulator) Push(chunk bus.AudioChunk) { @@ -112,6 +114,7 @@ func (a *Agent) handleChunk(chunk bus.AudioChunk) { lastAudioAt: time.Now(), chatID: chunk.ChatID, speakerID: chunk.SpeakerID, + sessionID: chunk.SessionID, } a.sessions[key] = acc logger.DebugCF("voice-agent", "Started accumulating voice", map[string]any{"key": key, "file": filename}) @@ -182,11 +185,13 @@ func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) { channelType := "discord" + oralPrompt := "\n\n[SYSTEM]: The user just spoke this to you over voice chat. Please reply in a highly concise, conversational, oral style suitable for text-to-speech. Do not use markdown, emojis, asterisks, or code blocks. Speak naturally. If the user expresses that they want to end the call, say goodbye and use the voice_leave tool." + a.bus.PublishInbound(ctx, bus.InboundMessage{ Channel: channelType, SenderID: acc.speakerID, ChatID: acc.chatID, - Content: res.Text, + Content: res.Text + oralPrompt, Peer: bus.Peer{Kind: "channel", ID: acc.chatID}, Metadata: map[string]string{ "is_voice": "true", diff --git a/pkg/voice/transcriber.go b/pkg/voice/transcriber.go index a5b46fe28..3b612f26c 100644 --- a/pkg/voice/transcriber.go +++ b/pkg/voice/transcriber.go @@ -193,8 +193,8 @@ func DetectTranscriber(cfg *config.Config) Transcriber { } // Fall back to any model-list entry that uses the groq/ protocol. for _, mc := range cfg.ModelList { - if strings.HasPrefix(mc.Model, "groq/") && mc.APIKey() != "" { - return NewGroqTranscriber(mc.APIKey()) + if (strings.HasPrefix(mc.Model, "groq/") || mc.ModelName == "groq" || mc.Model == "whisper-large-v3-turbo") && mc.APIKey != "" { + return NewGroqTranscriber(mc.APIKey) } } return nil