feat(voice/tts): add Chatterbox support with emotion controls

Chatterbox exposes a /synthesize endpoint alongside the standard
/v1/audio/speech one. The native endpoint adds two parameters unavailable
in the OpenAI-compatible API:
  - exaggeration (0.0–1.0): emotional expressiveness of the voice
  - cfg_weight  (0.0–1.0): how closely the voice follows the prompt

Routing: when model starts with 'chatterbox' (case-insensitive), Synthesize()
posts to /synthesize with the Chatterbox body; otherwise it uses the standard
/v1/audio/speech path. All other backends are unaffected.

Changes:
- kokoro.go: chatterboxRequest struct, isChatterbox() helper, Synthesize()
  branching logic, exaggeration/cfgWeight fields on KokoroSynthesizer
- TTSProfile: Exaggeration + CFGWeight fields (defaults: 0.5 / 0.5)
- config.go: TTSConfig gains Exaggeration + CFGWeight (env-overridable)
- main.go: wire new fields through TTSProfile
- config.example.json: document exaggeration + cfg_weight

Chatterbox config example:
  "tts": {
    "enabled": true,
    "api_base": "http://localhost:8100",
    "model":    "chatterbox-1",
    "voice":    "default",
    "format":   "mp3",
    "exaggeration": 0.5,
    "cfg_weight":   0.5
  }
This commit is contained in:
Myka 2026-02-17 11:30:05 +03:00
parent 1e68aa12b0
commit ef5c2de460
4 changed files with 114 additions and 54 deletions

View file

@ -637,11 +637,13 @@ func gatewayCmd() {
// Attach TTS synthesis callbacks to the message tool (enables voice=true). // Attach TTS synthesis callbacks to the message tool (enables voice=true).
if cfg.Tools.TTS.Enabled { if cfg.Tools.TTS.Enabled {
synthesizer := voice.NewKokoroSynthesizerFromProfile(voice.TTSProfile{ synthesizer := voice.NewKokoroSynthesizerFromProfile(voice.TTSProfile{
APIBase: cfg.Tools.TTS.APIBase, APIBase: cfg.Tools.TTS.APIBase,
Voice: cfg.Tools.TTS.Voice, Voice: cfg.Tools.TTS.Voice,
Model: cfg.Tools.TTS.Model, Model: cfg.Tools.TTS.Model,
Format: cfg.Tools.TTS.Format, Format: cfg.Tools.TTS.Format,
Speed: cfg.Tools.TTS.Speed, Speed: cfg.Tools.TTS.Speed,
Exaggeration: cfg.Tools.TTS.Exaggeration,
CFGWeight: cfg.Tools.TTS.CFGWeight,
}) })
if synthesizer.IsAvailable() { if synthesizer.IsAvailable() {
logger.InfoCF("voice", "TTS enabled — voice=true supported in message tool", map[string]interface{}{ logger.InfoCF("voice", "TTS enabled — voice=true supported in message tool", map[string]interface{}{

View file

@ -128,7 +128,9 @@
"voice": "en_us-lessac-medium", "voice": "en_us-lessac-medium",
"model": "tts-1", "model": "tts-1",
"format": "mp3", "format": "mp3",
"speed": 1.0 "speed": 1.0,
"exaggeration": 0.5,
"cfg_weight": 0.5
} }
}, },
"heartbeat": { "heartbeat": {

View file

@ -217,12 +217,14 @@ type WhisperConfig struct {
} }
type TTSConfig struct { type TTSConfig struct {
Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_TTS_ENABLED"` Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_TTS_ENABLED"`
APIBase string `json:"api_base" env:"PICOCLAW_TOOLS_TTS_API_BASE"` APIBase string `json:"api_base" env:"PICOCLAW_TOOLS_TTS_API_BASE"`
Voice string `json:"voice" env:"PICOCLAW_TOOLS_TTS_VOICE"` Voice string `json:"voice" env:"PICOCLAW_TOOLS_TTS_VOICE"`
Model string `json:"model" env:"PICOCLAW_TOOLS_TTS_MODEL"` Model string `json:"model" env:"PICOCLAW_TOOLS_TTS_MODEL"`
Format string `json:"format" env:"PICOCLAW_TOOLS_TTS_FORMAT"` Format string `json:"format" env:"PICOCLAW_TOOLS_TTS_FORMAT"`
Speed float64 `json:"speed" env:"PICOCLAW_TOOLS_TTS_SPEED"` Speed float64 `json:"speed" env:"PICOCLAW_TOOLS_TTS_SPEED"`
Exaggeration float64 `json:"exaggeration" env:"PICOCLAW_TOOLS_TTS_EXAGGERATION"` // Chatterbox: emotion expressiveness 0.01.0
CFGWeight float64 `json:"cfg_weight" env:"PICOCLAW_TOOLS_TTS_CFG_WEIGHT"` // Chatterbox: voice guidance weight 0.01.0
} }
type ToolsConfig struct { type ToolsConfig struct {
@ -343,12 +345,14 @@ func DefaultConfig() *Config {
APIBase: "http://localhost:8200", APIBase: "http://localhost:8200",
}, },
TTS: TTSConfig{ TTS: TTSConfig{
Enabled: false, Enabled: false,
APIBase: "http://localhost:8100", APIBase: "http://localhost:8100",
Voice: "en_us-lessac-medium", Voice: "en_us-lessac-medium",
Model: "tts-1", Model: "tts-1",
Format: "mp3", Format: "mp3",
Speed: 1.0, Speed: 1.0,
Exaggeration: 0.5,
CFGWeight: 0.5,
}, },
}, },
Heartbeat: HeartbeatConfig{ Heartbeat: HeartbeatConfig{

View file

@ -8,23 +8,28 @@ import (
"io" "io"
"net/http" "net/http"
"os" "os"
"strings"
"time" "time"
"github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/logger"
) )
// KokoroSynthesizer uses any OpenAI-compatible /v1/audio/speech endpoint // KokoroSynthesizer uses any OpenAI-compatible /v1/audio/speech endpoint
// (Kokoro, Piper, Chatterbox, OpenAI, etc.). // (Kokoro, Piper, OpenAI, etc.) and also supports Chatterbox's native
// /synthesize endpoint for exaggeration and cfg_weight control.
type KokoroSynthesizer struct { type KokoroSynthesizer struct {
apiBase string apiBase string
voice string voice string
model string model string
format string format string
speed float64 speed float64
httpClient *http.Client exaggeration float64
cfgWeight float64
httpClient *http.Client
} }
type kokoroRequest struct { // openaiRequest is the body for the standard /v1/audio/speech endpoint.
type openaiRequest struct {
Model string `json:"model"` Model string `json:"model"`
Input string `json:"input"` Input string `json:"input"`
Voice string `json:"voice"` Voice string `json:"voice"`
@ -32,13 +37,25 @@ type kokoroRequest struct {
Speed float64 `json:"speed,omitempty"` Speed float64 `json:"speed,omitempty"`
} }
// chatterboxRequest is the body for Chatterbox's native /synthesize endpoint.
// Used when model starts with "chatterbox" — gives access to emotion controls.
type chatterboxRequest struct {
Text string `json:"text"`
Voice string `json:"voice,omitempty"`
Exaggeration float64 `json:"exaggeration"`
CFGWeight float64 `json:"cfg_weight"`
Format string `json:"format,omitempty"`
}
// TTSProfile holds the full voice profile for the synthesizer. // TTSProfile holds the full voice profile for the synthesizer.
type TTSProfile struct { type TTSProfile struct {
APIBase string APIBase string
Voice string Voice string
Model string Model string
Format string Format string
Speed float64 Speed float64
Exaggeration float64 // Chatterbox only: emotion expressiveness 0.01.0
CFGWeight float64 // Chatterbox only: voice guidance weight 0.01.0
} }
// NewKokoroSynthesizer creates a TTS client from a voice profile. // NewKokoroSynthesizer creates a TTS client from a voice profile.
@ -67,21 +84,31 @@ func NewKokoroSynthesizerFromProfile(p TTSProfile) *KokoroSynthesizer {
if p.Speed == 0 { if p.Speed == 0 {
p.Speed = 1.0 p.Speed = 1.0
} }
if p.Exaggeration == 0 {
p.Exaggeration = 0.5
}
if p.CFGWeight == 0 {
p.CFGWeight = 0.5
}
logger.InfoCF("voice", "Creating TTS synthesizer", map[string]interface{}{ logger.InfoCF("voice", "Creating TTS synthesizer", map[string]interface{}{
"api_base": p.APIBase, "api_base": p.APIBase,
"voice": p.Voice, "voice": p.Voice,
"model": p.Model, "model": p.Model,
"format": p.Format, "format": p.Format,
"speed": p.Speed, "speed": p.Speed,
"exaggeration": p.Exaggeration,
"cfg_weight": p.CFGWeight,
}) })
return &KokoroSynthesizer{ return &KokoroSynthesizer{
apiBase: p.APIBase, apiBase: p.APIBase,
voice: p.Voice, voice: p.Voice,
model: p.Model, model: p.Model,
format: p.Format, format: p.Format,
speed: p.Speed, speed: p.Speed,
exaggeration: p.Exaggeration,
cfgWeight: p.CFGWeight,
httpClient: &http.Client{ httpClient: &http.Client{
Timeout: 60 * time.Second, Timeout: 60 * time.Second,
}, },
@ -90,26 +117,52 @@ func NewKokoroSynthesizerFromProfile(p TTSProfile) *KokoroSynthesizer {
// Synthesize converts text to audio, writes it to a temp file, and returns the path. // Synthesize converts text to audio, writes it to a temp file, and returns the path.
// The caller must delete the file when done. // The caller must delete the file when done.
// isChatterbox returns true when the configured model targets the Chatterbox
// server, which exposes a richer /synthesize endpoint alongside the standard
// /v1/audio/speech one.
func (s *KokoroSynthesizer) isChatterbox() bool {
return strings.HasPrefix(strings.ToLower(s.model), "chatterbox")
}
func (s *KokoroSynthesizer) Synthesize(ctx context.Context, text string) (string, error) { func (s *KokoroSynthesizer) Synthesize(ctx context.Context, text string) (string, error) {
logger.InfoCF("voice", "Synthesizing speech", map[string]interface{}{ logger.InfoCF("voice", "Synthesizing speech", map[string]interface{}{
"text_length": len(text), "text_length": len(text),
"voice": s.voice, "voice": s.voice,
"model": s.model,
"chatterbox": s.isChatterbox(),
}) })
reqBody := kokoroRequest{ var (
Model: s.model, bodyBytes []byte
Input: text, url string
Voice: s.voice, err error
Format: s.format, )
Speed: s.speed,
}
bodyBytes, err := json.Marshal(reqBody) if s.isChatterbox() {
// Chatterbox native endpoint — supports exaggeration and cfg_weight.
url = s.apiBase + "/synthesize"
bodyBytes, err = json.Marshal(chatterboxRequest{
Text: text,
Voice: s.voice,
Exaggeration: s.exaggeration,
CFGWeight: s.cfgWeight,
Format: s.format,
})
} else {
// Standard OpenAI-compatible endpoint.
url = s.apiBase + "/v1/audio/speech"
bodyBytes, err = json.Marshal(openaiRequest{
Model: s.model,
Input: text,
Voice: s.voice,
Format: s.format,
Speed: s.speed,
})
}
if err != nil { if err != nil {
return "", fmt.Errorf("failed to marshal TTS request: %w", err) return "", fmt.Errorf("failed to marshal TTS request: %w", err)
} }
url := s.apiBase + "/v1/audio/speech"
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes)) req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes))
if err != nil { if err != nil {
return "", fmt.Errorf("failed to create TTS request: %w", err) return "", fmt.Errorf("failed to create TTS request: %w", err)
@ -124,10 +177,9 @@ func (s *KokoroSynthesizer) Synthesize(ctx context.Context, text string) (string
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body) body, _ := io.ReadAll(resp.Body)
return "", fmt.Errorf("Kokoro TTS error (status %d): %s", resp.StatusCode, string(body)) return "", fmt.Errorf("TTS error (status %d): %s", resp.StatusCode, string(body))
} }
// Write audio to temp file
tmpFile, err := os.CreateTemp("", "picoclaw-tts-*."+s.format) tmpFile, err := os.CreateTemp("", "picoclaw-tts-*."+s.format)
if err != nil { if err != nil {
return "", fmt.Errorf("failed to create temp audio file: %w", err) return "", fmt.Errorf("failed to create temp audio file: %w", err)