feat: add audio transcription via Groq Whisper API
Transcribe inbound voice/audio messages before passing to the LLM. Reuses existing pkg/voice.GroqTranscriber (no duplicate implementation). - Add TranscribeConfig to ToolsConfig (enabled, api_key, model) - Wire transcriber into AgentLoop.processMessage - Use inferMediaType() to detect audio files (reuses existing helper) - Supports env vars: PICOCLAW_TRANSCRIBE_ENABLED, PICOCLAW_TRANSCRIBE_API_KEY - Zero overhead when disabled (transcriber is nil)
This commit is contained in:
parent
d5370c9605
commit
54887caf8e
2 changed files with 66 additions and 1 deletions
|
|
@ -30,6 +30,7 @@ import (
|
||||||
"github.com/sipeed/picoclaw/pkg/state"
|
"github.com/sipeed/picoclaw/pkg/state"
|
||||||
"github.com/sipeed/picoclaw/pkg/tools"
|
"github.com/sipeed/picoclaw/pkg/tools"
|
||||||
"github.com/sipeed/picoclaw/pkg/utils"
|
"github.com/sipeed/picoclaw/pkg/utils"
|
||||||
|
"github.com/sipeed/picoclaw/pkg/voice"
|
||||||
)
|
)
|
||||||
|
|
||||||
type AgentLoop struct {
|
type AgentLoop struct {
|
||||||
|
|
@ -42,6 +43,7 @@ type AgentLoop struct {
|
||||||
fallback *providers.FallbackChain
|
fallback *providers.FallbackChain
|
||||||
channelManager *channels.Manager
|
channelManager *channels.Manager
|
||||||
mediaStore media.MediaStore
|
mediaStore media.MediaStore
|
||||||
|
transcriber *voice.GroqTranscriber
|
||||||
}
|
}
|
||||||
|
|
||||||
// processOptions configures how a message is processed
|
// processOptions configures how a message is processed
|
||||||
|
|
@ -75,6 +77,13 @@ func NewAgentLoop(cfg *config.Config, msgBus *bus.MessageBus, provider providers
|
||||||
stateManager = state.NewManager(defaultAgent.Workspace)
|
stateManager = state.NewManager(defaultAgent.Workspace)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Set up transcriber if configured
|
||||||
|
var t *voice.GroqTranscriber
|
||||||
|
if cfg.Tools.Transcribe.Enabled && cfg.Tools.Transcribe.APIKey != "" {
|
||||||
|
t = voice.NewGroqTranscriber(cfg.Tools.Transcribe.APIKey)
|
||||||
|
logger.InfoC("agent", "Audio transcription enabled (Groq Whisper)")
|
||||||
|
}
|
||||||
|
|
||||||
return &AgentLoop{
|
return &AgentLoop{
|
||||||
bus: msgBus,
|
bus: msgBus,
|
||||||
cfg: cfg,
|
cfg: cfg,
|
||||||
|
|
@ -82,6 +91,7 @@ func NewAgentLoop(cfg *config.Config, msgBus *bus.MessageBus, provider providers
|
||||||
state: stateManager,
|
state: stateManager,
|
||||||
summarizing: sync.Map{},
|
summarizing: sync.Map{},
|
||||||
fallback: fallbackChain,
|
fallback: fallbackChain,
|
||||||
|
transcriber: t,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -412,11 +422,59 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage)
|
||||||
"matched_by": route.MatchedBy,
|
"matched_by": route.MatchedBy,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Transcribe any audio media refs before passing to the agent.
|
||||||
|
userMessage := msg.Content
|
||||||
|
if al.transcriber != nil && len(msg.Media) > 0 && al.mediaStore != nil {
|
||||||
|
logger.DebugCF("agent", "Checking media for transcription", map[string]any{
|
||||||
|
"media_count": len(msg.Media),
|
||||||
|
})
|
||||||
|
for _, ref := range msg.Media {
|
||||||
|
localPath, meta, err := al.mediaStore.ResolveWithMeta(ref)
|
||||||
|
if err != nil {
|
||||||
|
logger.WarnCF("agent", "Failed to resolve media ref for transcription", map[string]any{
|
||||||
|
"ref": ref, "error": err.Error(),
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if inferMediaType(meta.Filename, meta.ContentType) != "audio" {
|
||||||
|
logger.DebugCF("agent", "Skipping non-audio media", map[string]any{
|
||||||
|
"ref": ref, "filename": meta.Filename,
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
logger.InfoCF("agent", "Transcribing audio", map[string]any{
|
||||||
|
"ref": ref, "path": localPath, "filename": meta.Filename,
|
||||||
|
})
|
||||||
|
result, err := al.transcriber.Transcribe(ctx, localPath)
|
||||||
|
if err != nil {
|
||||||
|
logger.WarnCF("agent", "Audio transcription failed", map[string]any{
|
||||||
|
"ref": ref, "error": err.Error(),
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
logger.InfoCF("agent", "Transcribed audio", map[string]any{
|
||||||
|
"ref": ref, "length": len(result.Text),
|
||||||
|
})
|
||||||
|
// Replace the [voice]/[audio] placeholder with the actual transcript
|
||||||
|
userMessage = strings.NewReplacer("[voice]", "", "[audio]", "").Replace(userMessage)
|
||||||
|
userMessage = strings.TrimSpace(userMessage)
|
||||||
|
if userMessage != "" {
|
||||||
|
userMessage = userMessage + "\n\n[Voice transcript]: " + result.Text
|
||||||
|
} else {
|
||||||
|
userMessage = result.Text
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if al.transcriber == nil && len(msg.Media) > 0 {
|
||||||
|
logger.WarnCF("agent", "Transcriber not configured, skipping media", map[string]any{
|
||||||
|
"media_count": len(msg.Media),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
return al.runAgentLoop(ctx, agent, processOptions{
|
return al.runAgentLoop(ctx, agent, processOptions{
|
||||||
SessionKey: sessionKey,
|
SessionKey: sessionKey,
|
||||||
Channel: msg.Channel,
|
Channel: msg.Channel,
|
||||||
ChatID: msg.ChatID,
|
ChatID: msg.ChatID,
|
||||||
UserMessage: msg.Content,
|
UserMessage: userMessage,
|
||||||
DefaultResponse: defaultResponse,
|
DefaultResponse: defaultResponse,
|
||||||
EnableSummary: true,
|
EnableSummary: true,
|
||||||
SendResponse: false,
|
SendResponse: false,
|
||||||
|
|
|
||||||
|
|
@ -544,6 +544,12 @@ type MediaCleanupConfig struct {
|
||||||
Interval int `json:"interval_minutes" env:"PICOCLAW_MEDIA_CLEANUP_INTERVAL"`
|
Interval int `json:"interval_minutes" env:"PICOCLAW_MEDIA_CLEANUP_INTERVAL"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type TranscribeConfig struct {
|
||||||
|
Enabled bool `json:"enabled" env:"PICOCLAW_TRANSCRIBE_ENABLED"`
|
||||||
|
APIKey string `json:"api_key" env:"PICOCLAW_TRANSCRIBE_API_KEY"`
|
||||||
|
Model string `json:"model" env:"PICOCLAW_TRANSCRIBE_MODEL"`
|
||||||
|
}
|
||||||
|
|
||||||
type ToolsConfig struct {
|
type ToolsConfig struct {
|
||||||
AllowReadPaths []string `json:"allow_read_paths" env:"PICOCLAW_TOOLS_ALLOW_READ_PATHS"`
|
AllowReadPaths []string `json:"allow_read_paths" env:"PICOCLAW_TOOLS_ALLOW_READ_PATHS"`
|
||||||
AllowWritePaths []string `json:"allow_write_paths" env:"PICOCLAW_TOOLS_ALLOW_WRITE_PATHS"`
|
AllowWritePaths []string `json:"allow_write_paths" env:"PICOCLAW_TOOLS_ALLOW_WRITE_PATHS"`
|
||||||
|
|
@ -552,6 +558,7 @@ type ToolsConfig struct {
|
||||||
Exec ExecConfig `json:"exec"`
|
Exec ExecConfig `json:"exec"`
|
||||||
Skills SkillsToolsConfig `json:"skills"`
|
Skills SkillsToolsConfig `json:"skills"`
|
||||||
MediaCleanup MediaCleanupConfig `json:"media_cleanup"`
|
MediaCleanup MediaCleanupConfig `json:"media_cleanup"`
|
||||||
|
Transcribe TranscribeConfig `json:"transcribe"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type SkillsToolsConfig struct {
|
type SkillsToolsConfig struct {
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue