feat: add audio transcription via Groq Whisper API

Transcribe inbound voice/audio messages before passing to the LLM.
Reuses existing pkg/voice.GroqTranscriber (no duplicate implementation).

- Add TranscribeConfig to ToolsConfig (enabled, api_key, model)
- Wire transcriber into AgentLoop.processMessage
- Use inferMediaType() to detect audio files (reuses existing helper)
- Supports env vars: PICOCLAW_TRANSCRIBE_ENABLED, PICOCLAW_TRANSCRIBE_API_KEY
- Zero overhead when disabled (transcriber is nil)
This commit is contained in:
Virat Mankali 2026-03-02 10:25:56 +05:30
parent d5370c9605
commit 54887caf8e
2 changed files with 66 additions and 1 deletions

View file

@ -30,6 +30,7 @@ import (
"github.com/sipeed/picoclaw/pkg/state"
"github.com/sipeed/picoclaw/pkg/tools"
"github.com/sipeed/picoclaw/pkg/utils"
"github.com/sipeed/picoclaw/pkg/voice"
)
type AgentLoop struct {
@ -42,6 +43,7 @@ type AgentLoop struct {
fallback *providers.FallbackChain
channelManager *channels.Manager
mediaStore media.MediaStore
transcriber *voice.GroqTranscriber
}
// processOptions configures how a message is processed
@ -75,6 +77,13 @@ func NewAgentLoop(cfg *config.Config, msgBus *bus.MessageBus, provider providers
stateManager = state.NewManager(defaultAgent.Workspace)
}
// Set up transcriber if configured
var t *voice.GroqTranscriber
if cfg.Tools.Transcribe.Enabled && cfg.Tools.Transcribe.APIKey != "" {
t = voice.NewGroqTranscriber(cfg.Tools.Transcribe.APIKey)
logger.InfoC("agent", "Audio transcription enabled (Groq Whisper)")
}
return &AgentLoop{
bus: msgBus,
cfg: cfg,
@ -82,6 +91,7 @@ func NewAgentLoop(cfg *config.Config, msgBus *bus.MessageBus, provider providers
state: stateManager,
summarizing: sync.Map{},
fallback: fallbackChain,
transcriber: t,
}
}
@ -412,11 +422,59 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage)
"matched_by": route.MatchedBy,
})
// Transcribe any audio media refs before passing to the agent.
userMessage := msg.Content
if al.transcriber != nil && len(msg.Media) > 0 && al.mediaStore != nil {
logger.DebugCF("agent", "Checking media for transcription", map[string]any{
"media_count": len(msg.Media),
})
for _, ref := range msg.Media {
localPath, meta, err := al.mediaStore.ResolveWithMeta(ref)
if err != nil {
logger.WarnCF("agent", "Failed to resolve media ref for transcription", map[string]any{
"ref": ref, "error": err.Error(),
})
continue
}
if inferMediaType(meta.Filename, meta.ContentType) != "audio" {
logger.DebugCF("agent", "Skipping non-audio media", map[string]any{
"ref": ref, "filename": meta.Filename,
})
continue
}
logger.InfoCF("agent", "Transcribing audio", map[string]any{
"ref": ref, "path": localPath, "filename": meta.Filename,
})
result, err := al.transcriber.Transcribe(ctx, localPath)
if err != nil {
logger.WarnCF("agent", "Audio transcription failed", map[string]any{
"ref": ref, "error": err.Error(),
})
continue
}
logger.InfoCF("agent", "Transcribed audio", map[string]any{
"ref": ref, "length": len(result.Text),
})
// Replace the [voice]/[audio] placeholder with the actual transcript
userMessage = strings.NewReplacer("[voice]", "", "[audio]", "").Replace(userMessage)
userMessage = strings.TrimSpace(userMessage)
if userMessage != "" {
userMessage = userMessage + "\n\n[Voice transcript]: " + result.Text
} else {
userMessage = result.Text
}
}
} else if al.transcriber == nil && len(msg.Media) > 0 {
logger.WarnCF("agent", "Transcriber not configured, skipping media", map[string]any{
"media_count": len(msg.Media),
})
}
return al.runAgentLoop(ctx, agent, processOptions{
SessionKey: sessionKey,
Channel: msg.Channel,
ChatID: msg.ChatID,
UserMessage: msg.Content,
UserMessage: userMessage,
DefaultResponse: defaultResponse,
EnableSummary: true,
SendResponse: false,

View file

@ -544,6 +544,12 @@ type MediaCleanupConfig struct {
Interval int `json:"interval_minutes" env:"PICOCLAW_MEDIA_CLEANUP_INTERVAL"`
}
type TranscribeConfig struct {
Enabled bool `json:"enabled" env:"PICOCLAW_TRANSCRIBE_ENABLED"`
APIKey string `json:"api_key" env:"PICOCLAW_TRANSCRIBE_API_KEY"`
Model string `json:"model" env:"PICOCLAW_TRANSCRIBE_MODEL"`
}
type ToolsConfig struct {
AllowReadPaths []string `json:"allow_read_paths" env:"PICOCLAW_TOOLS_ALLOW_READ_PATHS"`
AllowWritePaths []string `json:"allow_write_paths" env:"PICOCLAW_TOOLS_ALLOW_WRITE_PATHS"`
@ -552,6 +558,7 @@ type ToolsConfig struct {
Exec ExecConfig `json:"exec"`
Skills SkillsToolsConfig `json:"skills"`
MediaCleanup MediaCleanupConfig `json:"media_cleanup"`
Transcribe TranscribeConfig `json:"transcribe"`
}
type SkillsToolsConfig struct {