init commit
This commit is contained in:
parent
96e312680d
commit
2d14da50a3
9 changed files with 512 additions and 19 deletions
3
go.mod
3
go.mod
|
|
@ -22,6 +22,8 @@ require (
|
|||
github.com/mymmrac/telego v1.7.0
|
||||
github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1
|
||||
github.com/openai/openai-go/v3 v3.22.0
|
||||
github.com/pion/rtp v1.8.7
|
||||
github.com/pion/webrtc/v3 v3.3.6
|
||||
github.com/rivo/tview v0.42.0
|
||||
github.com/rs/zerolog v1.34.0
|
||||
github.com/slack-go/slack v0.17.3
|
||||
|
|
@ -53,6 +55,7 @@ require (
|
|||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 // indirect
|
||||
github.com/pion/randutil v0.1.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
|
|
|
|||
6
go.sum
6
go.sum
|
|
@ -162,6 +162,12 @@ github.com/openai/openai-go/v3 v3.22.0 h1:6MEoNoV8sbjOVmXdvhmuX3BjVbVdcExbVyGixi
|
|||
github.com/openai/openai-go/v3 v3.22.0/go.mod h1:cdufnVK14cWcT9qA1rRtrXx4FTRsgbDPW7Ia7SS5cZo=
|
||||
github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 h1:rh2lKw/P/EqHa724vYH2+VVQ1YnW4u6EOXl0PMAovZE=
|
||||
github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
|
||||
github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
|
||||
github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
|
||||
github.com/pion/rtp v1.8.7 h1:qslKkG8qxvQ7hqaxkmL7Pl0XcUm+/Er7nMnu6Vq+ZxM=
|
||||
github.com/pion/rtp v1.8.7/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU=
|
||||
github.com/pion/webrtc/v3 v3.3.6 h1:7XAh4RPtlY1Vul6/GmZrv7z+NnxKA6If0KStXBI2ZLE=
|
||||
github.com/pion/webrtc/v3 v3.3.6/go.mod h1:zyN7th4mZpV27eXybfR/cnUf3J2DRy8zw/mdjD9JTNM=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
|
|
|
|||
|
|
@ -34,6 +34,8 @@ type MessageBus struct {
|
|||
inbound chan InboundMessage
|
||||
outbound chan OutboundMessage
|
||||
outboundMedia chan OutboundMediaMessage
|
||||
audioChunks chan AudioChunk
|
||||
voiceControls chan VoiceControl
|
||||
|
||||
closeOnce sync.Once
|
||||
done chan struct{}
|
||||
|
|
@ -47,6 +49,8 @@ func NewMessageBus() *MessageBus {
|
|||
inbound: make(chan InboundMessage, defaultBusBufferSize),
|
||||
outbound: make(chan OutboundMessage, defaultBusBufferSize),
|
||||
outboundMedia: make(chan OutboundMediaMessage, defaultBusBufferSize),
|
||||
audioChunks: make(chan AudioChunk, defaultBusBufferSize*4), // Audio chunks need more buffer
|
||||
voiceControls: make(chan VoiceControl, defaultBusBufferSize),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
|
@ -103,6 +107,22 @@ func (mb *MessageBus) OutboundMediaChan() <-chan OutboundMediaMessage {
|
|||
return mb.outboundMedia
|
||||
}
|
||||
|
||||
func (mb *MessageBus) PublishAudioChunk(ctx context.Context, chunk AudioChunk) error {
|
||||
return publish(ctx, mb, mb.audioChunks, chunk)
|
||||
}
|
||||
|
||||
func (mb *MessageBus) AudioChunksChan() <-chan AudioChunk {
|
||||
return mb.audioChunks
|
||||
}
|
||||
|
||||
func (mb *MessageBus) PublishVoiceControl(ctx context.Context, ctrl VoiceControl) error {
|
||||
return publish(ctx, mb, mb.voiceControls, ctrl)
|
||||
}
|
||||
|
||||
func (mb *MessageBus) VoiceControlsChan() <-chan VoiceControl {
|
||||
return mb.voiceControls
|
||||
}
|
||||
|
||||
// SetStreamDelegate registers a StreamDelegate (typically the channel Manager).
|
||||
func (mb *MessageBus) SetStreamDelegate(d StreamDelegate) {
|
||||
mb.streamDelegate.Store(d)
|
||||
|
|
@ -132,6 +152,8 @@ func (mb *MessageBus) Close() {
|
|||
close(mb.inbound)
|
||||
close(mb.outbound)
|
||||
close(mb.outboundMedia)
|
||||
close(mb.audioChunks)
|
||||
close(mb.voiceControls)
|
||||
|
||||
// clean up any remaining messages in channels
|
||||
drained := 0
|
||||
|
|
@ -144,6 +166,12 @@ func (mb *MessageBus) Close() {
|
|||
for range mb.outboundMedia {
|
||||
drained++
|
||||
}
|
||||
for range mb.audioChunks {
|
||||
drained++
|
||||
}
|
||||
for range mb.voiceControls {
|
||||
drained++
|
||||
}
|
||||
|
||||
if drained > 0 {
|
||||
logger.DebugCF("bus", "Drained buffered messages during close", map[string]any{
|
||||
|
|
|
|||
|
|
@ -51,3 +51,23 @@ type OutboundMediaMessage struct {
|
|||
ChatID string `json:"chat_id"`
|
||||
Parts []MediaPart `json:"parts"`
|
||||
}
|
||||
|
||||
// AudioChunk represents a chunk of streaming voice data.
|
||||
type AudioChunk struct {
|
||||
SessionID string `json:"session_id"`
|
||||
SpeakerID string `json:"speaker_id"` // User ID or SSRC
|
||||
ChatID string `json:"chat_id"` // Where to respond
|
||||
Sequence uint64 `json:"sequence"`
|
||||
Timestamp uint32 `json:"timestamp"`
|
||||
SampleRate int `json:"sample_rate"`
|
||||
Channels int `json:"channels"`
|
||||
Format string `json:"format"` // "opus", "pcm", etc
|
||||
Data []byte `json:"data"`
|
||||
}
|
||||
|
||||
// VoiceControl represents state or commands for voice sessions.
|
||||
type VoiceControl struct {
|
||||
SessionID string `json:"session_id"`
|
||||
Type string `json:"type"` // "state", "command"
|
||||
Action string `json:"action"` // "idle", "listening", "start", "stop"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ type DiscordChannel struct {
|
|||
typingMu sync.Mutex
|
||||
typingStop map[string]chan struct{} // chatID → stop signal
|
||||
botUserID string // stored for mention checking
|
||||
bus *bus.MessageBus
|
||||
}
|
||||
|
||||
func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordChannel, error) {
|
||||
|
|
@ -73,6 +74,7 @@ func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordC
|
|||
config: cfg,
|
||||
ctx: context.Background(),
|
||||
typingStop: make(map[string]chan struct{}),
|
||||
bus: bus,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
@ -321,6 +323,10 @@ func (c *DiscordChannel) handleMessage(s *discordgo.Session, m *discordgo.Messag
|
|||
return
|
||||
}
|
||||
|
||||
if c.handleVoiceCommand(s, m) {
|
||||
return
|
||||
}
|
||||
|
||||
// Check allowlist first to avoid downloading attachments for rejected users
|
||||
sender := bus.SenderInfo{
|
||||
Platform: "discord",
|
||||
|
|
|
|||
86
pkg/channels/discord/voice.go
Normal file
86
pkg/channels/discord/voice.go
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
package discord
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/bwmarrin/discordgo"
|
||||
"github.com/sipeed/picoclaw/pkg/bus"
|
||||
"github.com/sipeed/picoclaw/pkg/logger"
|
||||
)
|
||||
|
||||
func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.MessageCreate) bool {
|
||||
if m.Content == "!vc join" {
|
||||
vs, err := s.State.VoiceState(m.GuildID, m.Author.ID)
|
||||
if err != nil || vs == nil {
|
||||
s.ChannelMessageSend(m.ChannelID, "You need to be in a voice channel first!")
|
||||
return true
|
||||
}
|
||||
|
||||
logger.InfoCF("discord", "Joining voice channel", map[string]any{"channel": vs.ChannelID})
|
||||
vc, err := s.ChannelVoiceJoin(m.GuildID, vs.ChannelID, false, false)
|
||||
if err != nil {
|
||||
s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("Failed to join voice channel: %v", err))
|
||||
return true
|
||||
}
|
||||
|
||||
go c.receiveVoice(vc, m.GuildID, m.ChannelID)
|
||||
s.ChannelMessageSend(m.ChannelID, "Joined Voice Channel! Listening for audio...")
|
||||
return true
|
||||
} else if m.Content == "!vc leave" {
|
||||
vc, exists := s.VoiceConnections[m.GuildID]
|
||||
if exists && vc != nil {
|
||||
vc.Disconnect()
|
||||
s.ChannelMessageSend(m.ChannelID, "Left Voice Channel.")
|
||||
} else {
|
||||
s.ChannelMessageSend(m.ChannelID, "Not in a voice channel.")
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID string, chatID string) {
|
||||
logger.InfoCF("discord", "Started listening for voice", map[string]any{"guild": guildID})
|
||||
|
||||
sessionID := fmt.Sprintf("discord_vc_%s", guildID)
|
||||
|
||||
c.bus.PublishVoiceControl(c.ctx, bus.VoiceControl{
|
||||
SessionID: sessionID,
|
||||
Type: "state",
|
||||
Action: "listening",
|
||||
})
|
||||
|
||||
var sequence uint64 = 0
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-c.ctx.Done():
|
||||
return
|
||||
case p, ok := <-vc.OpusRecv:
|
||||
if !ok {
|
||||
logger.InfoCF("discord", "Voice channel closed", map[string]any{"guild": guildID})
|
||||
return
|
||||
}
|
||||
|
||||
if p == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
sequence++
|
||||
|
||||
chunk := bus.AudioChunk{
|
||||
SessionID: sessionID,
|
||||
SpeakerID: fmt.Sprintf("%d", p.SSRC),
|
||||
ChatID: chatID,
|
||||
Sequence: sequence,
|
||||
Timestamp: p.Timestamp,
|
||||
SampleRate: 48000,
|
||||
Channels: 2,
|
||||
Format: "opus",
|
||||
Data: p.Opus,
|
||||
}
|
||||
|
||||
c.bus.PublishAudioChunk(c.ctx, chunk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -60,6 +60,7 @@ type services struct {
|
|||
ChannelManager *channels.Manager
|
||||
DeviceService *devices.Service
|
||||
HealthServer *health.Server
|
||||
VoiceAgentCancel context.CancelFunc
|
||||
manualReloadChan chan struct{}
|
||||
reloading atomic.Bool
|
||||
}
|
||||
|
|
@ -305,6 +306,12 @@ func setupAndStartServices(
|
|||
if transcriber := voice.DetectTranscriber(cfg); transcriber != nil {
|
||||
agentLoop.SetTranscriber(transcriber)
|
||||
logger.InfoCF("voice", "Transcription enabled (agent-level)", map[string]any{"provider": transcriber.Name()})
|
||||
|
||||
// Start Voice Agent Orchestrator
|
||||
vaCtx, vaCancel := context.WithCancel(context.Background())
|
||||
runningServices.VoiceAgentCancel = vaCancel
|
||||
voiceAgent := voice.NewAgent(msgBus, transcriber)
|
||||
voiceAgent.Start(vaCtx)
|
||||
}
|
||||
|
||||
enabledChannels := runningServices.ChannelManager.GetEnabledChannels()
|
||||
|
|
@ -351,6 +358,9 @@ func stopAndCleanupServices(runningServices *services, shutdownTimeout time.Dura
|
|||
if !isReload && runningServices.ChannelManager != nil {
|
||||
runningServices.ChannelManager.StopAll(shutdownCtx)
|
||||
}
|
||||
if runningServices.VoiceAgentCancel != nil {
|
||||
runningServices.VoiceAgentCancel()
|
||||
}
|
||||
if runningServices.DeviceService != nil {
|
||||
runningServices.DeviceService.Stop()
|
||||
}
|
||||
|
|
@ -532,6 +542,12 @@ func restartServices(
|
|||
al.SetTranscriber(transcriber)
|
||||
if transcriber != nil {
|
||||
logger.InfoCF("voice", "Transcription re-enabled (agent-level)", map[string]any{"provider": transcriber.Name()})
|
||||
|
||||
// Start Voice Agent Orchestrator on reload
|
||||
vaCtx, vaCancel := context.WithCancel(context.Background())
|
||||
runningServices.VoiceAgentCancel = vaCancel
|
||||
voiceAgent := voice.NewAgent(msgBus, transcriber)
|
||||
voiceAgent.Start(vaCtx)
|
||||
} else {
|
||||
logger.InfoCF("voice", "Transcription disabled", nil)
|
||||
}
|
||||
|
|
|
|||
195
pkg/voice/agent.go
Normal file
195
pkg/voice/agent.go
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
package voice
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/pion/rtp"
|
||||
"github.com/pion/webrtc/v3/pkg/media/oggwriter"
|
||||
"github.com/sipeed/picoclaw/pkg/bus"
|
||||
"github.com/sipeed/picoclaw/pkg/logger"
|
||||
)
|
||||
|
||||
type speechAccumulator struct {
|
||||
writer *oggwriter.OggWriter
|
||||
file string
|
||||
lastAudioAt time.Time
|
||||
mu sync.Mutex
|
||||
closed bool
|
||||
chatID string
|
||||
speakerID string
|
||||
}
|
||||
|
||||
func (a *speechAccumulator) Push(chunk bus.AudioChunk) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
|
||||
if a.closed {
|
||||
return
|
||||
}
|
||||
|
||||
a.lastAudioAt = time.Now()
|
||||
|
||||
pkt := &rtp.Packet{
|
||||
Header: rtp.Header{
|
||||
SequenceNumber: uint16(chunk.Sequence),
|
||||
Timestamp: chunk.Timestamp,
|
||||
SSRC: uint32(chunk.Sequence), // Arbitrary dummy
|
||||
},
|
||||
Payload: chunk.Data,
|
||||
}
|
||||
|
||||
if err := a.writer.WriteRTP(pkt); err != nil {
|
||||
logger.ErrorCF("voice-agent", "Failed to write RTP", map[string]any{"error": err})
|
||||
}
|
||||
}
|
||||
|
||||
func (a *speechAccumulator) Close() {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if !a.closed {
|
||||
a.writer.Close()
|
||||
a.closed = true
|
||||
}
|
||||
}
|
||||
|
||||
type Agent struct {
|
||||
bus *bus.MessageBus
|
||||
transcriber Transcriber
|
||||
|
||||
mu sync.Mutex
|
||||
sessions map[string]*speechAccumulator // keyed by sessionID_speakerID
|
||||
}
|
||||
|
||||
func NewAgent(mb *bus.MessageBus, t Transcriber) *Agent {
|
||||
return &Agent{
|
||||
bus: mb,
|
||||
transcriber: t,
|
||||
sessions: make(map[string]*speechAccumulator),
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Agent) Start(ctx context.Context) {
|
||||
logger.InfoCF("voice-agent", "Started Voice Agent orchestrator", nil)
|
||||
go a.listenChunks(ctx)
|
||||
go a.vadTick(ctx)
|
||||
}
|
||||
|
||||
func (a *Agent) listenChunks(ctx context.Context) {
|
||||
chunks := a.bus.AudioChunksChan()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case chunk := <-chunks:
|
||||
a.handleChunk(chunk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Agent) handleChunk(chunk bus.AudioChunk) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
|
||||
key := fmt.Sprintf("%s_%s", chunk.SessionID, chunk.SpeakerID)
|
||||
|
||||
acc, exists := a.sessions[key]
|
||||
if !exists {
|
||||
filename := filepath.Join(os.TempDir(), fmt.Sprintf("voice_%s_%d.ogg", key, time.Now().UnixNano()))
|
||||
writer, err := oggwriter.New(filename, uint32(chunk.SampleRate), uint16(chunk.Channels))
|
||||
if err != nil {
|
||||
logger.ErrorCF("voice-agent", "Failed to create OggWriter", map[string]any{"error": err})
|
||||
return
|
||||
}
|
||||
|
||||
acc = &speechAccumulator{
|
||||
writer: writer,
|
||||
file: filename,
|
||||
lastAudioAt: time.Now(),
|
||||
chatID: chunk.ChatID,
|
||||
speakerID: chunk.SpeakerID,
|
||||
}
|
||||
a.sessions[key] = acc
|
||||
logger.DebugCF("voice-agent", "Started accumulating voice", map[string]any{"key": key, "file": filename})
|
||||
}
|
||||
|
||||
acc.Push(chunk)
|
||||
}
|
||||
|
||||
func (a *Agent) vadTick(ctx context.Context) {
|
||||
ticker := time.NewTicker(500 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
a.checkSilence(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Agent) checkSilence(ctx context.Context) {
|
||||
a.mu.Lock()
|
||||
now := time.Now()
|
||||
var finished []*speechAccumulator
|
||||
|
||||
for key, acc := range a.sessions {
|
||||
acc.mu.Lock()
|
||||
last := acc.lastAudioAt
|
||||
acc.mu.Unlock()
|
||||
|
||||
if now.Sub(last) > 1500*time.Millisecond {
|
||||
acc.Close()
|
||||
delete(a.sessions, key)
|
||||
finished = append(finished, acc)
|
||||
}
|
||||
}
|
||||
a.mu.Unlock()
|
||||
|
||||
for _, acc := range finished {
|
||||
go a.processUtterance(ctx, acc)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) {
|
||||
defer os.Remove(acc.file)
|
||||
|
||||
logger.InfoCF("voice-agent", "User finished speaking, transcribing...", map[string]any{"file": acc.file})
|
||||
|
||||
if a.transcriber == nil {
|
||||
logger.ErrorCF("voice-agent", "No STT configured!", nil)
|
||||
return
|
||||
}
|
||||
|
||||
res, err := a.transcriber.Transcribe(ctx, acc.file)
|
||||
if err != nil {
|
||||
logger.ErrorCF("voice-agent", "Transcription failed", map[string]any{"error": err})
|
||||
return
|
||||
}
|
||||
|
||||
if res.Text == "" {
|
||||
logger.DebugCF("voice-agent", "Ignored empty transcription", map[string]any{"file": acc.file})
|
||||
return
|
||||
}
|
||||
|
||||
logger.InfoCF("voice-agent", "Transcription result", map[string]any{"text": res.Text, "duration": res.Duration})
|
||||
|
||||
channelType := "discord"
|
||||
|
||||
a.bus.PublishInbound(ctx, bus.InboundMessage{
|
||||
Channel: channelType,
|
||||
SenderID: acc.speakerID,
|
||||
ChatID: acc.chatID,
|
||||
Content: res.Text,
|
||||
Peer: bus.Peer{Kind: "channel", ID: acc.chatID},
|
||||
Metadata: map[string]string{
|
||||
"is_voice": "true",
|
||||
},
|
||||
})
|
||||
}
|
||||
|
|
@ -1,11 +1,21 @@
|
|||
package voice
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/sipeed/picoclaw/pkg/config"
|
||||
"github.com/sipeed/picoclaw/pkg/providers"
|
||||
"github.com/sipeed/picoclaw/pkg/logger"
|
||||
"github.com/sipeed/picoclaw/pkg/utils"
|
||||
)
|
||||
|
||||
type Transcriber interface {
|
||||
|
|
@ -13,34 +23,157 @@ type Transcriber interface {
|
|||
Transcribe(ctx context.Context, audioFilePath string) (*TranscriptionResponse, error)
|
||||
}
|
||||
|
||||
type GroqTranscriber struct {
|
||||
apiKey string
|
||||
apiBase string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
type TranscriptionResponse struct {
|
||||
Text string `json:"text"`
|
||||
Language string `json:"language,omitempty"`
|
||||
Duration float64 `json:"duration,omitempty"`
|
||||
}
|
||||
|
||||
func supportsAudioTranscription(model string) bool {
|
||||
protocol, _ := providers.ExtractProtocol(model)
|
||||
func NewGroqTranscriber(apiKey string) *GroqTranscriber {
|
||||
logger.DebugCF("voice", "Creating Groq transcriber", map[string]any{"has_api_key": apiKey != ""})
|
||||
|
||||
switch protocol {
|
||||
case "openai", "azure", "azure-openai",
|
||||
"litellm", "openrouter", "groq", "zhipu", "gemini", "nvidia",
|
||||
"ollama", "moonshot", "shengsuanyun", "deepseek", "cerebras",
|
||||
"vivgrid", "volcengine", "vllm", "qwen", "qwen-intl", "qwen-international", "dashscope-intl",
|
||||
"qwen-us", "dashscope-us", "mistral", "avian", "minimax", "longcat", "modelscope", "novita",
|
||||
"coding-plan", "alibaba-coding", "qwen-coding":
|
||||
// These protocols all go through the OpenAI-compatible or Azure provider path in
|
||||
// providers.CreateProviderFromConfig, so they are the only ones that can supply
|
||||
// the audio media payload shape expected by NewAudioModelTranscriber.
|
||||
|
||||
// TODO: Further restrict this by modelID, since not every model under these
|
||||
// protocols supports audio transcription.
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
apiBase := "https://api.groq.com/openai/v1"
|
||||
return &GroqTranscriber{
|
||||
apiKey: apiKey,
|
||||
apiBase: apiBase,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 60 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (t *GroqTranscriber) TranscribeData(ctx context.Context, data []byte, filename string) (*TranscriptionResponse, error) {
|
||||
logger.InfoCF("voice", "Starting memory transcription", map[string]any{"filename": filename, "bytes": len(data)})
|
||||
|
||||
var requestBody bytes.Buffer
|
||||
writer := multipart.NewWriter(&requestBody)
|
||||
|
||||
part, err := writer.CreateFormFile("file", filename)
|
||||
if err != nil {
|
||||
logger.ErrorCF("voice", "Failed to create form file", map[string]any{"error": err})
|
||||
return nil, fmt.Errorf("failed to create form file: %w", err)
|
||||
}
|
||||
|
||||
if _, err := io.Copy(part, bytes.NewReader(data)); err != nil {
|
||||
logger.ErrorCF("voice", "Failed to copy file content", map[string]any{"error": err})
|
||||
return nil, fmt.Errorf("failed to copy file content: %w", err)
|
||||
}
|
||||
|
||||
if err = writer.WriteField("model", "whisper-large-v3-turbo"); err != nil {
|
||||
logger.ErrorCF("voice", "Failed to write model field", map[string]any{"error": err})
|
||||
return nil, fmt.Errorf("failed to write model field: %w", err)
|
||||
}
|
||||
|
||||
if err = writer.WriteField("response_format", "json"); err != nil {
|
||||
logger.ErrorCF("voice", "Failed to write response_format field", map[string]any{"error": err})
|
||||
return nil, fmt.Errorf("failed to write response_format field: %w", err)
|
||||
}
|
||||
|
||||
if err = writer.Close(); err != nil {
|
||||
logger.ErrorCF("voice", "Failed to close multipart writer", map[string]any{"error": err})
|
||||
return nil, fmt.Errorf("failed to close multipart writer: %w", err)
|
||||
}
|
||||
|
||||
return t.doRequest(ctx, &requestBody, writer.FormDataContentType(), int64(len(data)))
|
||||
}
|
||||
|
||||
func (t *GroqTranscriber) Transcribe(ctx context.Context, audioFilePath string) (*TranscriptionResponse, error) {
|
||||
logger.InfoCF("voice", "Starting transcription", map[string]any{"audio_file": audioFilePath})
|
||||
|
||||
audioFile, err := os.Open(audioFilePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open audio file: %w", err)
|
||||
}
|
||||
defer audioFile.Close()
|
||||
|
||||
fileInfo, err := audioFile.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var requestBody bytes.Buffer
|
||||
writer := multipart.NewWriter(&requestBody)
|
||||
part, err := writer.CreateFormFile("file", filepath.Base(audioFilePath))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
io.Copy(part, audioFile)
|
||||
writer.WriteField("model", "whisper-large-v3")
|
||||
writer.WriteField("response_format", "json")
|
||||
writer.Close()
|
||||
|
||||
return t.doRequest(ctx, &requestBody, writer.FormDataContentType(), fileInfo.Size())
|
||||
}
|
||||
|
||||
func (t *GroqTranscriber) doRequest(ctx context.Context, requestBody *bytes.Buffer, contentType string, fileSize int64) (*TranscriptionResponse, error) {
|
||||
url := t.apiBase + "/audio/transcriptions"
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", url, requestBody)
|
||||
if err != nil {
|
||||
logger.ErrorCF("voice", "Failed to create request", map[string]any{"error": err})
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
req.Header.Set("Authorization", "Bearer "+t.apiKey)
|
||||
|
||||
logger.DebugCF("voice", "Sending transcription request to Groq API", map[string]any{
|
||||
"url": url,
|
||||
"request_size_bytes": requestBody.Len(),
|
||||
"file_size_bytes": fileSize,
|
||||
})
|
||||
|
||||
resp, err := t.httpClient.Do(req)
|
||||
if err != nil {
|
||||
logger.ErrorCF("voice", "Failed to send request", map[string]any{"error": err})
|
||||
return nil, fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
logger.ErrorCF("voice", "Failed to read response", map[string]any{"error": err})
|
||||
return nil, fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
logger.ErrorCF("voice", "API error", map[string]any{
|
||||
"status_code": resp.StatusCode,
|
||||
"response": string(body),
|
||||
})
|
||||
return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
logger.DebugCF("voice", "Received response from Groq API", map[string]any{
|
||||
"status_code": resp.StatusCode,
|
||||
"response_size_bytes": len(body),
|
||||
})
|
||||
|
||||
var result TranscriptionResponse
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
logger.ErrorCF("voice", "Failed to unmarshal response", map[string]any{"error": err})
|
||||
return nil, fmt.Errorf("failed to unmarshal response: %w", err)
|
||||
}
|
||||
|
||||
logger.InfoCF("voice", "Transcription completed successfully", map[string]any{
|
||||
"text_length": len(result.Text),
|
||||
"language": result.Language,
|
||||
"duration_seconds": result.Duration,
|
||||
"transcription_preview": utils.Truncate(result.Text, 50),
|
||||
})
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (t *GroqTranscriber) Name() string {
|
||||
return "groq"
|
||||
}
|
||||
|
||||
// DetectTranscriber inspects cfg and returns the appropriate Transcriber, or
|
||||
// nil if no supported transcription provider is configured.
|
||||
func DetectTranscriber(cfg *config.Config) Transcriber {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue