refactor for #1648
This commit is contained in:
parent
d572f8124f
commit
57a468507b
10 changed files with 73 additions and 16 deletions
|
|
@ -19,6 +19,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"github.com/sipeed/picoclaw/pkg/asr"
|
||||||
"github.com/sipeed/picoclaw/pkg/bus"
|
"github.com/sipeed/picoclaw/pkg/bus"
|
||||||
"github.com/sipeed/picoclaw/pkg/channels"
|
"github.com/sipeed/picoclaw/pkg/channels"
|
||||||
"github.com/sipeed/picoclaw/pkg/commands"
|
"github.com/sipeed/picoclaw/pkg/commands"
|
||||||
|
|
@ -32,7 +33,6 @@ import (
|
||||||
"github.com/sipeed/picoclaw/pkg/state"
|
"github.com/sipeed/picoclaw/pkg/state"
|
||||||
"github.com/sipeed/picoclaw/pkg/tools"
|
"github.com/sipeed/picoclaw/pkg/tools"
|
||||||
"github.com/sipeed/picoclaw/pkg/utils"
|
"github.com/sipeed/picoclaw/pkg/utils"
|
||||||
"github.com/sipeed/picoclaw/pkg/voice"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type AgentLoop struct {
|
type AgentLoop struct {
|
||||||
|
|
@ -45,7 +45,7 @@ type AgentLoop struct {
|
||||||
fallback *providers.FallbackChain
|
fallback *providers.FallbackChain
|
||||||
channelManager *channels.Manager
|
channelManager *channels.Manager
|
||||||
mediaStore media.MediaStore
|
mediaStore media.MediaStore
|
||||||
transcriber voice.Transcriber
|
transcriber asr.Transcriber
|
||||||
cmdRegistry *commands.Registry
|
cmdRegistry *commands.Registry
|
||||||
mcp mcpRuntime
|
mcp mcpRuntime
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
|
|
@ -502,7 +502,7 @@ func (al *AgentLoop) SetMediaStore(s media.MediaStore) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetTranscriber injects a voice transcriber for agent-level audio transcription.
|
// SetTranscriber injects a voice transcriber for agent-level audio transcription.
|
||||||
func (al *AgentLoop) SetTranscriber(t voice.Transcriber) {
|
func (al *AgentLoop) SetTranscriber(t asr.Transcriber) {
|
||||||
al.transcriber = t
|
al.transcriber = t
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
package voice
|
package asr
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
package voice
|
package asr
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
55
pkg/audio/ogg.go
Normal file
55
pkg/audio/ogg.go
Normal file
|
|
@ -0,0 +1,55 @@
|
||||||
|
package audio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DecodeOggOpus reads an Ogg format stream and extracts individual Opus payloads.
|
||||||
|
// It calls onFrame for every complete Opus frame found in the stream.
|
||||||
|
func DecodeOggOpus(r io.Reader, onFrame func([]byte) error) error {
|
||||||
|
var packet []byte
|
||||||
|
header := make([]byte, 27)
|
||||||
|
|
||||||
|
for {
|
||||||
|
if _, err := io.ReadFull(r, header); err != nil {
|
||||||
|
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("failed to read ogg header: %w", err)
|
||||||
|
}
|
||||||
|
if string(header[:4]) != "OggS" {
|
||||||
|
return fmt.Errorf("invalid ogg magic string")
|
||||||
|
}
|
||||||
|
|
||||||
|
pageSegments := int(header[26])
|
||||||
|
segmentTable := make([]byte, pageSegments)
|
||||||
|
if _, err := io.ReadFull(r, segmentTable); err != nil {
|
||||||
|
return fmt.Errorf("failed to read segment table: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, lacing := range segmentTable {
|
||||||
|
segment := make([]byte, lacing)
|
||||||
|
if _, err := io.ReadFull(r, segment); err != nil {
|
||||||
|
return fmt.Errorf("failed to read segment data: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
packet = append(packet, segment...)
|
||||||
|
|
||||||
|
// If lacing is less than 255, the packet is complete
|
||||||
|
if lacing < 255 {
|
||||||
|
if len(packet) > 0 {
|
||||||
|
// Ignore Ogg Opus headers
|
||||||
|
if !bytes.HasPrefix(packet, []byte("OpusHead")) && !bytes.HasPrefix(packet, []byte("OpusTags")) {
|
||||||
|
if err := onFrame(packet); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Start new packet
|
||||||
|
packet = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -20,8 +20,8 @@ import (
|
||||||
"github.com/sipeed/picoclaw/pkg/identity"
|
"github.com/sipeed/picoclaw/pkg/identity"
|
||||||
"github.com/sipeed/picoclaw/pkg/logger"
|
"github.com/sipeed/picoclaw/pkg/logger"
|
||||||
"github.com/sipeed/picoclaw/pkg/media"
|
"github.com/sipeed/picoclaw/pkg/media"
|
||||||
|
"github.com/sipeed/picoclaw/pkg/tts"
|
||||||
"github.com/sipeed/picoclaw/pkg/utils"
|
"github.com/sipeed/picoclaw/pkg/utils"
|
||||||
"github.com/sipeed/picoclaw/pkg/voice"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|
@ -44,7 +44,7 @@ type DiscordChannel struct {
|
||||||
typingStop map[string]chan struct{} // chatID → stop signal
|
typingStop map[string]chan struct{} // chatID → stop signal
|
||||||
botUserID string // stored for mention checking
|
botUserID string // stored for mention checking
|
||||||
bus *bus.MessageBus
|
bus *bus.MessageBus
|
||||||
tts voice.TTSProvider
|
tts tts.TTSProvider
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordChannel, error) {
|
func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordChannel, error) {
|
||||||
|
|
|
||||||
|
|
@ -4,14 +4,14 @@ import (
|
||||||
"github.com/sipeed/picoclaw/pkg/bus"
|
"github.com/sipeed/picoclaw/pkg/bus"
|
||||||
"github.com/sipeed/picoclaw/pkg/channels"
|
"github.com/sipeed/picoclaw/pkg/channels"
|
||||||
"github.com/sipeed/picoclaw/pkg/config"
|
"github.com/sipeed/picoclaw/pkg/config"
|
||||||
"github.com/sipeed/picoclaw/pkg/voice"
|
"github.com/sipeed/picoclaw/pkg/tts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
channels.RegisterFactory("discord", func(cfg *config.Config, b *bus.MessageBus) (channels.Channel, error) {
|
channels.RegisterFactory("discord", func(cfg *config.Config, b *bus.MessageBus) (channels.Channel, error) {
|
||||||
ch, err := NewDiscordChannel(cfg.Channels.Discord, b)
|
ch, err := NewDiscordChannel(cfg.Channels.Discord, b)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
ch.tts = voice.DetectTTS(cfg)
|
ch.tts = tts.DetectTTS(cfg)
|
||||||
}
|
}
|
||||||
return ch, err
|
return ch, err
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/sipeed/picoclaw/pkg/agent"
|
"github.com/sipeed/picoclaw/pkg/agent"
|
||||||
|
"github.com/sipeed/picoclaw/pkg/asr"
|
||||||
"github.com/sipeed/picoclaw/pkg/bus"
|
"github.com/sipeed/picoclaw/pkg/bus"
|
||||||
"github.com/sipeed/picoclaw/pkg/channels"
|
"github.com/sipeed/picoclaw/pkg/channels"
|
||||||
_ "github.com/sipeed/picoclaw/pkg/channels/dingtalk"
|
_ "github.com/sipeed/picoclaw/pkg/channels/dingtalk"
|
||||||
|
|
@ -284,7 +285,7 @@ func setupAndStartServices(
|
||||||
agentLoop.SetChannelManager(runningServices.ChannelManager)
|
agentLoop.SetChannelManager(runningServices.ChannelManager)
|
||||||
agentLoop.SetMediaStore(runningServices.MediaStore)
|
agentLoop.SetMediaStore(runningServices.MediaStore)
|
||||||
|
|
||||||
if transcriber := voice.DetectTranscriber(cfg); transcriber != nil {
|
if transcriber := asr.DetectTranscriber(cfg); transcriber != nil {
|
||||||
agentLoop.SetTranscriber(transcriber)
|
agentLoop.SetTranscriber(transcriber)
|
||||||
logger.InfoCF("voice", "Transcription enabled (agent-level)", map[string]any{"provider": transcriber.Name()})
|
logger.InfoCF("voice", "Transcription enabled (agent-level)", map[string]any{"provider": transcriber.Name()})
|
||||||
|
|
||||||
|
|
@ -522,7 +523,7 @@ func restartServices(
|
||||||
fmt.Println(" ✓ Device event service restarted")
|
fmt.Println(" ✓ Device event service restarted")
|
||||||
}
|
}
|
||||||
|
|
||||||
transcriber := voice.DetectTranscriber(cfg)
|
transcriber := asr.DetectTranscriber(cfg)
|
||||||
al.SetTranscriber(transcriber)
|
al.SetTranscriber(transcriber)
|
||||||
if transcriber != nil {
|
if transcriber != nil {
|
||||||
logger.InfoCF("voice", "Transcription re-enabled (agent-level)", map[string]any{"provider": transcriber.Name()})
|
logger.InfoCF("voice", "Transcription re-enabled (agent-level)", map[string]any{"provider": transcriber.Name()})
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
package voice
|
package tts
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
|
@ -11,6 +11,7 @@ import (
|
||||||
|
|
||||||
"github.com/pion/rtp"
|
"github.com/pion/rtp"
|
||||||
"github.com/pion/webrtc/v3/pkg/media/oggwriter"
|
"github.com/pion/webrtc/v3/pkg/media/oggwriter"
|
||||||
|
"github.com/sipeed/picoclaw/pkg/asr"
|
||||||
"github.com/sipeed/picoclaw/pkg/bus"
|
"github.com/sipeed/picoclaw/pkg/bus"
|
||||||
"github.com/sipeed/picoclaw/pkg/logger"
|
"github.com/sipeed/picoclaw/pkg/logger"
|
||||||
)
|
)
|
||||||
|
|
@ -61,13 +62,13 @@ func (a *speechAccumulator) Close() {
|
||||||
|
|
||||||
type Agent struct {
|
type Agent struct {
|
||||||
bus *bus.MessageBus
|
bus *bus.MessageBus
|
||||||
transcriber Transcriber
|
transcriber asr.Transcriber
|
||||||
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
sessions map[string]*speechAccumulator // keyed by sessionID_speakerID
|
sessions map[string]*speechAccumulator // keyed by sessionID_speakerID
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewAgent(mb *bus.MessageBus, t Transcriber) *Agent {
|
func NewAgent(mb *bus.MessageBus, t asr.Transcriber) *Agent {
|
||||||
return &Agent{
|
return &Agent{
|
||||||
bus: mb,
|
bus: mb,
|
||||||
transcriber: t,
|
transcriber: t,
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue