Update suggested fixes
This commit is contained in:
parent
f57a8ba261
commit
eb5114a65d
7 changed files with 113 additions and 34 deletions
|
|
@ -2443,6 +2443,9 @@ turnLoop:
|
||||||
Channel: ts.channel,
|
Channel: ts.channel,
|
||||||
ChatID: ts.chatID,
|
ChatID: ts.chatID,
|
||||||
Content: toolResult.ForUser,
|
Content: toolResult.ForUser,
|
||||||
|
Metadata: map[string]string{
|
||||||
|
"is_tool_call": "true",
|
||||||
|
},
|
||||||
})
|
})
|
||||||
logger.DebugCF("agent", "Sent tool result to user",
|
logger.DebugCF("agent", "Sent tool result to user",
|
||||||
map[string]any{
|
map[string]any{
|
||||||
|
|
|
||||||
|
|
@ -47,10 +47,13 @@ type DiscordChannel struct {
|
||||||
botUserID string // stored for mention checking
|
botUserID string // stored for mention checking
|
||||||
bus *bus.MessageBus
|
bus *bus.MessageBus
|
||||||
tts tts.TTSProvider
|
tts tts.TTSProvider
|
||||||
|
voiceMu sync.RWMutex
|
||||||
|
voiceSSRC map[string]map[uint32]string // guildID -> ssrc -> userID
|
||||||
|
|
||||||
// TTS interruption: cancel active playback when user speaks
|
// TTS interruption: cancel active playback when user speaks
|
||||||
ttsMu sync.Mutex
|
ttsMu sync.Mutex
|
||||||
cancelTTS context.CancelFunc
|
cancelTTS context.CancelFunc
|
||||||
|
ttsPlayID uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordChannel, error) {
|
func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordChannel, error) {
|
||||||
|
|
@ -83,7 +86,7 @@ func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordC
|
||||||
ctx: context.Background(),
|
ctx: context.Background(),
|
||||||
typingStop: make(map[string]chan struct{}),
|
typingStop: make(map[string]chan struct{}),
|
||||||
bus: bus,
|
bus: bus,
|
||||||
bus: bus,
|
voiceSSRC: make(map[string]map[uint32]string),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -155,14 +158,7 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) erro
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
isToolCall := false
|
if c.tts != nil {
|
||||||
if msg.Metadata != nil {
|
|
||||||
if val, ok := msg.Metadata["is_tool_call"]; ok && val == "true" {
|
|
||||||
isToolCall = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if c.tts != nil && !isToolCall {
|
|
||||||
if ch, err := c.session.State.Channel(channelID); err == nil && ch.GuildID != "" {
|
if ch, err := c.session.State.Channel(channelID); err == nil && ch.GuildID != "" {
|
||||||
if vc, ok := c.session.VoiceConnections[ch.GuildID]; ok && vc != nil {
|
if vc, ok := c.session.VoiceConnections[ch.GuildID]; ok && vc != nil {
|
||||||
// Cancel any previous TTS playback
|
// Cancel any previous TTS playback
|
||||||
|
|
@ -171,10 +167,12 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) erro
|
||||||
c.cancelTTS()
|
c.cancelTTS()
|
||||||
}
|
}
|
||||||
ttsCtx, ttsCancel := context.WithCancel(c.ctx)
|
ttsCtx, ttsCancel := context.WithCancel(c.ctx)
|
||||||
|
c.ttsPlayID++
|
||||||
|
playID := c.ttsPlayID
|
||||||
c.cancelTTS = ttsCancel
|
c.cancelTTS = ttsCancel
|
||||||
c.ttsMu.Unlock()
|
c.ttsMu.Unlock()
|
||||||
|
|
||||||
go c.playTTS(ttsCtx, vc, msg.Content)
|
go c.playTTS(ttsCtx, vc, msg.Content, playID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -677,17 +675,13 @@ func (c *DiscordChannel) listenVoiceControl(ctx context.Context) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnection, text string) {
|
func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnection, text string, playID uint64) {
|
||||||
// Capture the cancel func associated with this playback (if any).
|
// Capture the cancel func associated with this playback (if any).
|
||||||
c.ttsMu.Lock()
|
|
||||||
playbackCancel := c.cancelTTS
|
|
||||||
c.ttsMu.Unlock()
|
|
||||||
|
|
||||||
// Clear cancelTTS when playback finishes (normal or interrupted),
|
// Clear cancelTTS when playback finishes (normal or interrupted),
|
||||||
// but only if it still refers to this playback's cancel func.
|
// but only if it still refers to this playback's cancel func.
|
||||||
defer func() {
|
defer func() {
|
||||||
c.ttsMu.Lock()
|
c.ttsMu.Lock()
|
||||||
if c.cancelTTS == playbackCancel {
|
if c.ttsPlayID == playID {
|
||||||
c.cancelTTS = nil
|
c.cancelTTS = nil
|
||||||
}
|
}
|
||||||
c.ttsMu.Unlock()
|
c.ttsMu.Unlock()
|
||||||
|
|
|
||||||
|
|
@ -10,14 +10,45 @@ import (
|
||||||
|
|
||||||
"github.com/sipeed/picoclaw/pkg/audio"
|
"github.com/sipeed/picoclaw/pkg/audio"
|
||||||
"github.com/sipeed/picoclaw/pkg/bus"
|
"github.com/sipeed/picoclaw/pkg/bus"
|
||||||
|
"github.com/sipeed/picoclaw/pkg/identity"
|
||||||
"github.com/sipeed/picoclaw/pkg/logger"
|
"github.com/sipeed/picoclaw/pkg/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func (c *DiscordChannel) setVoiceUserID(guildID string, ssrc uint32, userID string) {
|
||||||
|
if userID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
c.voiceMu.Lock()
|
||||||
|
defer c.voiceMu.Unlock()
|
||||||
|
|
||||||
|
ssrcMap, ok := c.voiceSSRC[guildID]
|
||||||
|
if !ok {
|
||||||
|
ssrcMap = make(map[uint32]string)
|
||||||
|
c.voiceSSRC[guildID] = ssrcMap
|
||||||
|
}
|
||||||
|
ssrcMap[ssrc] = userID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DiscordChannel) voiceUserID(guildID string, ssrc uint32) string {
|
||||||
|
c.voiceMu.RLock()
|
||||||
|
defer c.voiceMu.RUnlock()
|
||||||
|
|
||||||
|
ssrcMap, ok := c.voiceSSRC[guildID]
|
||||||
|
if !ok {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return ssrcMap[ssrc]
|
||||||
|
}
|
||||||
|
|
||||||
func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.MessageCreate) bool {
|
func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.MessageCreate) bool {
|
||||||
if m.Content == "!vc join" {
|
if m.Content == "!vc join" {
|
||||||
vs, err := s.State.VoiceState(m.GuildID, m.Author.ID)
|
vs, err := s.State.VoiceState(m.GuildID, m.Author.ID)
|
||||||
if err != nil || vs == nil {
|
if err != nil || vs == nil {
|
||||||
if _, sendErr := s.ChannelMessageSend(m.ChannelID, "You need to be in a voice channel first!"); sendErr != nil {
|
if _, sendErr := s.ChannelMessageSend(
|
||||||
|
m.ChannelID,
|
||||||
|
"You need to be in a voice channel first!",
|
||||||
|
); sendErr != nil {
|
||||||
logger.InfoCF("discord", "Failed to send voice channel requirement message", map[string]any{
|
logger.InfoCF("discord", "Failed to send voice channel requirement message", map[string]any{
|
||||||
"channel": m.ChannelID,
|
"channel": m.ChannelID,
|
||||||
"error": sendErr,
|
"error": sendErr,
|
||||||
|
|
@ -30,7 +61,10 @@ func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.M
|
||||||
vc, err := s.ChannelVoiceJoin(c.ctx, m.GuildID, vs.ChannelID, false, false)
|
vc, err := s.ChannelVoiceJoin(c.ctx, m.GuildID, vs.ChannelID, false, false)
|
||||||
vc, err := s.ChannelVoiceJoin(c.ctx, m.GuildID, vs.ChannelID, false, false)
|
vc, err := s.ChannelVoiceJoin(c.ctx, m.GuildID, vs.ChannelID, false, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if _, sendErr := s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("Failed to join voice channel: %v", err)); sendErr != nil {
|
if _, sendErr := s.ChannelMessageSend(
|
||||||
|
m.ChannelID,
|
||||||
|
fmt.Sprintf("Failed to join voice channel: %v", err),
|
||||||
|
); sendErr != nil {
|
||||||
logger.InfoCF("discord", "Failed to send voice join error message", map[string]any{
|
logger.InfoCF("discord", "Failed to send voice join error message", map[string]any{
|
||||||
"channel": m.ChannelID,
|
"channel": m.ChannelID,
|
||||||
"error": sendErr,
|
"error": sendErr,
|
||||||
|
|
@ -40,7 +74,10 @@ func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.M
|
||||||
}
|
}
|
||||||
|
|
||||||
go c.receiveVoice(vc, m.GuildID, m.ChannelID)
|
go c.receiveVoice(vc, m.GuildID, m.ChannelID)
|
||||||
if _, sendErr := s.ChannelMessageSend(m.ChannelID, "Joined Voice Channel! Listening for audio..."); sendErr != nil {
|
if _, sendErr := s.ChannelMessageSend(
|
||||||
|
m.ChannelID,
|
||||||
|
"Joined Voice Channel! Listening for audio...",
|
||||||
|
); sendErr != nil {
|
||||||
logger.InfoCF("discord", "Failed to send voice join success message", map[string]any{
|
logger.InfoCF("discord", "Failed to send voice join success message", map[string]any{
|
||||||
"channel": m.ChannelID,
|
"channel": m.ChannelID,
|
||||||
"error": sendErr,
|
"error": sendErr,
|
||||||
|
|
@ -104,6 +141,19 @@ func streamOggOpusToDiscord(ctx context.Context, vc *discordgo.VoiceConnection,
|
||||||
func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID string, chatID string) {
|
func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID string, chatID string) {
|
||||||
logger.InfoCF("discord", "Started listening for voice", map[string]any{"guild": guildID})
|
logger.InfoCF("discord", "Started listening for voice", map[string]any{"guild": guildID})
|
||||||
|
|
||||||
|
vc.AddHandler(func(_ *discordgo.VoiceConnection, vs *discordgo.VoiceSpeakingUpdate) {
|
||||||
|
if vs == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.setVoiceUserID(guildID, uint32(vs.SSRC), vs.UserID)
|
||||||
|
})
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
c.voiceMu.Lock()
|
||||||
|
delete(c.voiceSSRC, guildID)
|
||||||
|
c.voiceMu.Unlock()
|
||||||
|
}()
|
||||||
|
|
||||||
go func(ctx context.Context, vc *discordgo.VoiceConnection) {
|
go func(ctx context.Context, vc *discordgo.VoiceConnection) {
|
||||||
// Recover from potential panics if OpusSend is closed mid-send.
|
// Recover from potential panics if OpusSend is closed mid-send.
|
||||||
defer func() {
|
defer func() {
|
||||||
|
|
@ -211,11 +261,33 @@ func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID str
|
||||||
interruptCount = 0
|
interruptCount = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
userID := c.voiceUserID(guildID, p.SSRC)
|
||||||
|
if userID == "" {
|
||||||
|
logger.DebugCF("discord", "Dropping voice packet without user mapping", map[string]any{
|
||||||
|
"ssrc": p.SSRC,
|
||||||
|
"guild": guildID,
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
sender := bus.SenderInfo{
|
||||||
|
Platform: "discord",
|
||||||
|
PlatformID: userID,
|
||||||
|
CanonicalID: identity.BuildCanonicalID("discord", userID),
|
||||||
|
}
|
||||||
|
if !c.IsAllowedSender(sender) {
|
||||||
|
logger.DebugCF("discord", "Voice packet rejected by allowlist", map[string]any{
|
||||||
|
"user_id": userID,
|
||||||
|
"guild": guildID,
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
sequence++
|
sequence++
|
||||||
|
|
||||||
chunk := bus.AudioChunk{
|
chunk := bus.AudioChunk{
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
SpeakerID: fmt.Sprintf("%d", p.SSRC),
|
SpeakerID: userID,
|
||||||
ChatID: chatID,
|
ChatID: chatID,
|
||||||
Channel: "discord",
|
Channel: "discord",
|
||||||
Sequence: sequence,
|
Sequence: sequence,
|
||||||
|
|
|
||||||
|
|
@ -304,15 +304,10 @@ func setupAndStartServices(
|
||||||
agentLoop.SetChannelManager(runningServices.ChannelManager)
|
agentLoop.SetChannelManager(runningServices.ChannelManager)
|
||||||
agentLoop.SetMediaStore(runningServices.MediaStore)
|
agentLoop.SetMediaStore(runningServices.MediaStore)
|
||||||
|
|
||||||
if transcriber := asr.DetectTranscriber(cfg); transcriber != nil {
|
transcriber := asr.DetectTranscriber(cfg)
|
||||||
|
if transcriber != nil {
|
||||||
agentLoop.SetTranscriber(transcriber)
|
agentLoop.SetTranscriber(transcriber)
|
||||||
logger.InfoCF("voice", "Transcription enabled (agent-level)", map[string]any{"provider": transcriber.Name()})
|
logger.InfoCF("voice", "Transcription enabled (agent-level)", map[string]any{"provider": transcriber.Name()})
|
||||||
|
|
||||||
// Start Voice Agent Orchestrator
|
|
||||||
vaCtx, vaCancel := context.WithCancel(context.Background())
|
|
||||||
runningServices.VoiceAgentCancel = vaCancel
|
|
||||||
voiceAgent := voice.NewAgent(msgBus, transcriber)
|
|
||||||
voiceAgent.Start(vaCtx)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
enabledChannels := runningServices.ChannelManager.GetEnabledChannels()
|
enabledChannels := runningServices.ChannelManager.GetEnabledChannels()
|
||||||
|
|
@ -330,6 +325,14 @@ func setupAndStartServices(
|
||||||
return nil, fmt.Errorf("error starting channels: %w", err)
|
return nil, fmt.Errorf("error starting channels: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if transcriber != nil {
|
||||||
|
// Start Voice Agent Orchestrator after channels are ready.
|
||||||
|
vaCtx, vaCancel := context.WithCancel(context.Background())
|
||||||
|
runningServices.VoiceAgentCancel = vaCancel
|
||||||
|
voiceAgent := voice.NewAgent(msgBus, transcriber)
|
||||||
|
voiceAgent.Start(vaCtx)
|
||||||
|
}
|
||||||
|
|
||||||
fmt.Printf(
|
fmt.Printf(
|
||||||
"✓ Health endpoints available at http://%s:%d/health, /ready and /reload (POST)\n",
|
"✓ Health endpoints available at http://%s:%d/health, /ready and /reload (POST)\n",
|
||||||
cfg.Gateway.Host,
|
cfg.Gateway.Host,
|
||||||
|
|
|
||||||
|
|
@ -87,13 +87,18 @@ func (t *SendTTSTool) Execute(ctx context.Context, args map[string]any) *ToolRes
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ErrorResult(fmt.Sprintf("failed to create temp file: %v", err)).WithError(err)
|
return ErrorResult(fmt.Sprintf("failed to create temp file: %v", err)).WithError(err)
|
||||||
}
|
}
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
_, err = io.Copy(file, stream)
|
_, err = io.Copy(file, stream)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
file.Close()
|
||||||
return ErrorResult(fmt.Sprintf("failed to write tts audio: %v", err)).WithError(err)
|
return ErrorResult(fmt.Sprintf("failed to write tts audio: %v", err)).WithError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
err = file.Close()
|
||||||
|
if err != nil {
|
||||||
|
return ErrorResult(fmt.Sprintf("failed to close tts audio file: %v", err)).WithError(err)
|
||||||
|
}
|
||||||
|
|
||||||
filename, _ := args["filename"].(string)
|
filename, _ := args["filename"].(string)
|
||||||
filename = strings.TrimSpace(filename)
|
filename = strings.TrimSpace(filename)
|
||||||
if filename == "" {
|
if filename == "" {
|
||||||
|
|
|
||||||
|
|
@ -82,7 +82,10 @@ func NewOpenAITTSProvider(apiKey string, apiBase string, proxyURL string) *OpenA
|
||||||
Proxy: http.ProxyURL(pURL),
|
Proxy: http.ProxyURL(pURL),
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
logger.Warnf("NewOpenAITTSProvider: invalid proxy URL %q: %v; proceeding without proxy", proxyURL, err)
|
logger.WarnF(
|
||||||
|
"NewOpenAITTSProvider: invalid proxy URL; proceeding without proxy",
|
||||||
|
map[string]any{"proxyURL": proxyURL, "error": err},
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -120,16 +120,15 @@ func (a *Agent) handleChunk(chunk bus.AudioChunk) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
a.mu.Lock()
|
|
||||||
defer a.mu.Unlock()
|
|
||||||
|
|
||||||
key := fmt.Sprintf("%s_%s", chunk.SessionID, chunk.SpeakerID)
|
key := fmt.Sprintf("%s_%s", chunk.SessionID, chunk.SpeakerID)
|
||||||
|
|
||||||
|
a.mu.Lock()
|
||||||
acc, exists := a.sessions[key]
|
acc, exists := a.sessions[key]
|
||||||
if !exists {
|
if !exists {
|
||||||
filename := filepath.Join(os.TempDir(), fmt.Sprintf("voice_%s_%d.ogg", key, time.Now().UnixNano()))
|
filename := filepath.Join(os.TempDir(), fmt.Sprintf("voice_%s_%d.ogg", key, time.Now().UnixNano()))
|
||||||
writer, err := oggwriter.New(filename, uint32(chunk.SampleRate), uint16(chunk.Channels))
|
writer, err := oggwriter.New(filename, uint32(chunk.SampleRate), uint16(chunk.Channels))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
a.mu.Unlock()
|
||||||
logger.ErrorCF("voice-agent", "Failed to create OggWriter", map[string]any{"error": err})
|
logger.ErrorCF("voice-agent", "Failed to create OggWriter", map[string]any{"error": err})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
@ -146,6 +145,7 @@ func (a *Agent) handleChunk(chunk bus.AudioChunk) {
|
||||||
a.sessions[key] = acc
|
a.sessions[key] = acc
|
||||||
logger.DebugCF("voice-agent", "Started accumulating voice", map[string]any{"key": key, "file": filename})
|
logger.DebugCF("voice-agent", "Started accumulating voice", map[string]any{"key": key, "file": filename})
|
||||||
}
|
}
|
||||||
|
a.mu.Unlock()
|
||||||
|
|
||||||
acc.Push(chunk)
|
acc.Push(chunk)
|
||||||
}
|
}
|
||||||
|
|
@ -247,7 +247,6 @@ func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) {
|
||||||
Peer: bus.Peer{Kind: "channel", ID: acc.chatID},
|
Peer: bus.Peer{Kind: "channel", ID: acc.chatID},
|
||||||
Metadata: map[string]string{
|
Metadata: map[string]string{
|
||||||
"is_voice": "true",
|
"is_voice": "true",
|
||||||
"oral_prompt": oralPrompt,
|
|
||||||
},
|
},
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
logger.ErrorCF("voice-agent", "Failed to publish inbound message", map[string]any{"error": err})
|
logger.ErrorCF("voice-agent", "Failed to publish inbound message", map[string]any{"error": err})
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue