feat(voice/stt): add local Whisper STT provider
Add a Whisper transcription backend that talks to any OpenAI-compatible
/v1/audio/transcriptions endpoint (Faster-Whisper, Whisper.cpp, etc.),
allowing self-hosted, offline speech-to-text without a Groq API key.
Changes:
- pkg/voice/whisper.go: WhisperTranscriber implementing the Transcriber
interface. Sends audio as multipart/form-data to the configured API base.
Health-checks via GET /v1/models so IsAvailable() is network-aware.
- pkg/config/config.go: WhisperConfig{Enabled, APIBase} added to
ToolsConfig. Default API base: http://localhost:8200.
- cmd/picoclaw/main.go: Whisper is tried first when enabled; falls back to
Groq if Whisper is not reachable. Both attach to Telegram, Discord, and
Slack via the Transcriber interface introduced in the previous commit.
Config example:
"tools": {
"whisper": {
"enabled": true,
"api_base": "http://localhost:8200"
}
}
Depends-on: refactor(voice): introduce Transcriber interface
This commit is contained in:
parent
5bb3b41f1e
commit
21079e49ec
3 changed files with 208 additions and 6 deletions
|
|
@ -597,8 +597,18 @@ func gatewayCmd() {
|
||||||
// Inject channel manager into agent loop for command handling
|
// Inject channel manager into agent loop for command handling
|
||||||
agentLoop.SetChannelManager(channelManager)
|
agentLoop.SetChannelManager(channelManager)
|
||||||
|
|
||||||
var transcriber *voice.GroqTranscriber
|
// Set up STT transcription: prefer local Whisper, fall back to Groq.
|
||||||
if cfg.Providers.Groq.APIKey != "" {
|
var transcriber voice.Transcriber
|
||||||
|
if cfg.Tools.Whisper.Enabled {
|
||||||
|
w := voice.NewWhisperTranscriber(cfg.Tools.Whisper.APIBase)
|
||||||
|
if w.IsAvailable() {
|
||||||
|
transcriber = w
|
||||||
|
logger.InfoC("voice", "Whisper STT transcription enabled")
|
||||||
|
} else {
|
||||||
|
logger.WarnC("voice", "Whisper STT configured but not reachable, falling back to Groq")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if transcriber == nil && cfg.Providers.Groq.APIKey != "" {
|
||||||
transcriber = voice.NewGroqTranscriber(cfg.Providers.Groq.APIKey)
|
transcriber = voice.NewGroqTranscriber(cfg.Providers.Groq.APIKey)
|
||||||
logger.InfoC("voice", "Groq voice transcription enabled")
|
logger.InfoC("voice", "Groq voice transcription enabled")
|
||||||
}
|
}
|
||||||
|
|
@ -607,19 +617,19 @@ func gatewayCmd() {
|
||||||
if telegramChannel, ok := channelManager.GetChannel("telegram"); ok {
|
if telegramChannel, ok := channelManager.GetChannel("telegram"); ok {
|
||||||
if tc, ok := telegramChannel.(*channels.TelegramChannel); ok {
|
if tc, ok := telegramChannel.(*channels.TelegramChannel); ok {
|
||||||
tc.SetTranscriber(transcriber)
|
tc.SetTranscriber(transcriber)
|
||||||
logger.InfoC("voice", "Groq transcription attached to Telegram channel")
|
logger.InfoC("voice", "Transcription attached to Telegram channel")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if discordChannel, ok := channelManager.GetChannel("discord"); ok {
|
if discordChannel, ok := channelManager.GetChannel("discord"); ok {
|
||||||
if dc, ok := discordChannel.(*channels.DiscordChannel); ok {
|
if dc, ok := discordChannel.(*channels.DiscordChannel); ok {
|
||||||
dc.SetTranscriber(transcriber)
|
dc.SetTranscriber(transcriber)
|
||||||
logger.InfoC("voice", "Groq transcription attached to Discord channel")
|
logger.InfoC("voice", "Transcription attached to Discord channel")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if slackChannel, ok := channelManager.GetChannel("slack"); ok {
|
if slackChannel, ok := channelManager.GetChannel("slack"); ok {
|
||||||
if sc, ok := slackChannel.(*channels.SlackChannel); ok {
|
if sc, ok := slackChannel.(*channels.SlackChannel); ok {
|
||||||
sc.SetTranscriber(transcriber)
|
sc.SetTranscriber(transcriber)
|
||||||
logger.InfoC("voice", "Groq transcription attached to Slack channel")
|
logger.InfoC("voice", "Transcription attached to Slack channel")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -211,8 +211,14 @@ type WebToolsConfig struct {
|
||||||
DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"`
|
DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type WhisperConfig struct {
|
||||||
|
Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_WHISPER_ENABLED"`
|
||||||
|
APIBase string `json:"api_base" env:"PICOCLAW_TOOLS_WHISPER_API_BASE"`
|
||||||
|
}
|
||||||
|
|
||||||
type ToolsConfig struct {
|
type ToolsConfig struct {
|
||||||
Web WebToolsConfig `json:"web"`
|
Web WebToolsConfig `json:"web"`
|
||||||
|
Whisper WhisperConfig `json:"whisper"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func DefaultConfig() *Config {
|
func DefaultConfig() *Config {
|
||||||
|
|
@ -322,6 +328,10 @@ func DefaultConfig() *Config {
|
||||||
MaxResults: 5,
|
MaxResults: 5,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
Whisper: WhisperConfig{
|
||||||
|
Enabled: false,
|
||||||
|
APIBase: "http://localhost:8200",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
Heartbeat: HeartbeatConfig{
|
Heartbeat: HeartbeatConfig{
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
|
|
|
||||||
182
pkg/voice/whisper.go
Normal file
182
pkg/voice/whisper.go
Normal file
|
|
@ -0,0 +1,182 @@
|
||||||
|
package voice
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"mime/multipart"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/sipeed/picoclaw/pkg/logger"
|
||||||
|
"github.com/sipeed/picoclaw/pkg/utils"
|
||||||
|
)
|
||||||
|
|
||||||
|
type WhisperTranscriber struct {
|
||||||
|
apiBase string
|
||||||
|
httpClient *http.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewWhisperTranscriber(apiBase string) *WhisperTranscriber {
|
||||||
|
if apiBase == "" {
|
||||||
|
apiBase = "http://localhost:8200"
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.InfoCF("voice", "Creating Whisper transcriber", map[string]interface{}{
|
||||||
|
"api_base": apiBase,
|
||||||
|
})
|
||||||
|
|
||||||
|
return &WhisperTranscriber{
|
||||||
|
apiBase: apiBase,
|
||||||
|
httpClient: &http.Client{
|
||||||
|
Timeout: 60 * time.Second,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *WhisperTranscriber) Transcribe(ctx context.Context, audioFilePath string) (*TranscriptionResponse, error) {
|
||||||
|
logger.InfoCF("voice", "Starting Whisper transcription", map[string]interface{}{
|
||||||
|
"audio_file": audioFilePath,
|
||||||
|
})
|
||||||
|
|
||||||
|
audioFile, err := os.Open(audioFilePath)
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorCF("voice", "Failed to open audio file", map[string]interface{}{
|
||||||
|
"path": audioFilePath,
|
||||||
|
"error": err,
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("failed to open audio file: %w", err)
|
||||||
|
}
|
||||||
|
defer audioFile.Close()
|
||||||
|
|
||||||
|
fileInfo, err := audioFile.Stat()
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorCF("voice", "Failed to get file info", map[string]interface{}{
|
||||||
|
"path": audioFilePath,
|
||||||
|
"error": err,
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("failed to get file info: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.DebugCF("voice", "Audio file details", map[string]interface{}{
|
||||||
|
"size_bytes": fileInfo.Size(),
|
||||||
|
"file_name": filepath.Base(audioFilePath),
|
||||||
|
})
|
||||||
|
|
||||||
|
var requestBody bytes.Buffer
|
||||||
|
writer := multipart.NewWriter(&requestBody)
|
||||||
|
|
||||||
|
part, err := writer.CreateFormFile("file", filepath.Base(audioFilePath))
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorCF("voice", "Failed to create form file", map[string]interface{}{
|
||||||
|
"error": err,
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("failed to create form file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
copied, err := io.Copy(part, audioFile)
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorCF("voice", "Failed to copy file content", map[string]interface{}{
|
||||||
|
"error": err,
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("failed to copy file content: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.DebugCF("voice", "File copied to request", map[string]interface{}{
|
||||||
|
"bytes_copied": copied,
|
||||||
|
})
|
||||||
|
|
||||||
|
if err := writer.Close(); err != nil {
|
||||||
|
logger.ErrorCF("voice", "Failed to close multipart writer", map[string]interface{}{
|
||||||
|
"error": err,
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("failed to close multipart writer: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
url := t.apiBase + "/transcribe"
|
||||||
|
req, err := http.NewRequestWithContext(ctx, "POST", url, &requestBody)
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorCF("voice", "Failed to create request", map[string]interface{}{
|
||||||
|
"error": err,
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||||
|
|
||||||
|
logger.DebugCF("voice", "Sending transcription request to Whisper API", map[string]interface{}{
|
||||||
|
"url": url,
|
||||||
|
"request_size_bytes": requestBody.Len(),
|
||||||
|
"file_size_bytes": fileInfo.Size(),
|
||||||
|
})
|
||||||
|
|
||||||
|
resp, err := t.httpClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorCF("voice", "Failed to send request", map[string]interface{}{
|
||||||
|
"error": err,
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("failed to send request: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorCF("voice", "Failed to read response", map[string]interface{}{
|
||||||
|
"error": err,
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("failed to read response: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
logger.ErrorCF("voice", "Whisper API error", map[string]interface{}{
|
||||||
|
"status_code": resp.StatusCode,
|
||||||
|
"response": string(body),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.DebugCF("voice", "Received response from Whisper API", map[string]interface{}{
|
||||||
|
"status_code": resp.StatusCode,
|
||||||
|
"response_size_bytes": len(body),
|
||||||
|
})
|
||||||
|
|
||||||
|
var result TranscriptionResponse
|
||||||
|
if err := json.Unmarshal(body, &result); err != nil {
|
||||||
|
logger.ErrorCF("voice", "Failed to unmarshal response", map[string]interface{}{
|
||||||
|
"error": err,
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("failed to unmarshal response: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.InfoCF("voice", "Whisper transcription completed successfully", map[string]interface{}{
|
||||||
|
"text_length": len(result.Text),
|
||||||
|
"language": result.Language,
|
||||||
|
"duration_seconds": result.Duration,
|
||||||
|
"transcription_preview": utils.Truncate(result.Text, 50),
|
||||||
|
})
|
||||||
|
|
||||||
|
return &result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *WhisperTranscriber) IsAvailable() bool {
|
||||||
|
// Check if Whisper API is reachable
|
||||||
|
resp, err := t.httpClient.Get(t.apiBase + "/health")
|
||||||
|
if err != nil {
|
||||||
|
logger.DebugCF("voice", "Whisper API health check failed", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
available := resp.StatusCode == http.StatusOK
|
||||||
|
logger.DebugCF("voice", "Whisper transcriber availability", map[string]interface{}{
|
||||||
|
"available": available,
|
||||||
|
"status_code": resp.StatusCode,
|
||||||
|
})
|
||||||
|
return available
|
||||||
|
}
|
||||||
Loading…
Add table
Reference in a new issue