From ef5c2de460f199adcb4b0125d606639bedd8b099 Mon Sep 17 00:00:00 2001 From: Myka Date: Tue, 17 Feb 2026 11:30:05 +0300 Subject: [PATCH] feat(voice/tts): add Chatterbox support with emotion controls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chatterbox exposes a /synthesize endpoint alongside the standard /v1/audio/speech one. The native endpoint adds two parameters unavailable in the OpenAI-compatible API: - exaggeration (0.0–1.0): emotional expressiveness of the voice - cfg_weight (0.0–1.0): how closely the voice follows the prompt Routing: when model starts with 'chatterbox' (case-insensitive), Synthesize() posts to /synthesize with the Chatterbox body; otherwise it uses the standard /v1/audio/speech path. All other backends are unaffected. Changes: - kokoro.go: chatterboxRequest struct, isChatterbox() helper, Synthesize() branching logic, exaggeration/cfgWeight fields on KokoroSynthesizer - TTSProfile: Exaggeration + CFGWeight fields (defaults: 0.5 / 0.5) - config.go: TTSConfig gains Exaggeration + CFGWeight (env-overridable) - main.go: wire new fields through TTSProfile - config.example.json: document exaggeration + cfg_weight Chatterbox config example: "tts": { "enabled": true, "api_base": "http://localhost:8100", "model": "chatterbox-1", "voice": "default", "format": "mp3", "exaggeration": 0.5, "cfg_weight": 0.5 } --- cmd/picoclaw/main.go | 12 ++-- config/config.example.json | 4 +- pkg/config/config.go | 28 +++++---- pkg/voice/kokoro.go | 124 ++++++++++++++++++++++++++----------- 4 files changed, 114 insertions(+), 54 deletions(-) diff --git a/cmd/picoclaw/main.go b/cmd/picoclaw/main.go index 52f82d84f..60e7ad98e 100644 --- a/cmd/picoclaw/main.go +++ b/cmd/picoclaw/main.go @@ -637,11 +637,13 @@ func gatewayCmd() { // Attach TTS synthesis callbacks to the message tool (enables voice=true). if cfg.Tools.TTS.Enabled { synthesizer := voice.NewKokoroSynthesizerFromProfile(voice.TTSProfile{ - APIBase: cfg.Tools.TTS.APIBase, - Voice: cfg.Tools.TTS.Voice, - Model: cfg.Tools.TTS.Model, - Format: cfg.Tools.TTS.Format, - Speed: cfg.Tools.TTS.Speed, + APIBase: cfg.Tools.TTS.APIBase, + Voice: cfg.Tools.TTS.Voice, + Model: cfg.Tools.TTS.Model, + Format: cfg.Tools.TTS.Format, + Speed: cfg.Tools.TTS.Speed, + Exaggeration: cfg.Tools.TTS.Exaggeration, + CFGWeight: cfg.Tools.TTS.CFGWeight, }) if synthesizer.IsAvailable() { logger.InfoCF("voice", "TTS enabled — voice=true supported in message tool", map[string]interface{}{ diff --git a/config/config.example.json b/config/config.example.json index d7883532a..2057008b8 100644 --- a/config/config.example.json +++ b/config/config.example.json @@ -128,7 +128,9 @@ "voice": "en_us-lessac-medium", "model": "tts-1", "format": "mp3", - "speed": 1.0 + "speed": 1.0, + "exaggeration": 0.5, + "cfg_weight": 0.5 } }, "heartbeat": { diff --git a/pkg/config/config.go b/pkg/config/config.go index 00cd00a2e..f06c059ef 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -217,12 +217,14 @@ type WhisperConfig struct { } type TTSConfig struct { - Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_TTS_ENABLED"` - APIBase string `json:"api_base" env:"PICOCLAW_TOOLS_TTS_API_BASE"` - Voice string `json:"voice" env:"PICOCLAW_TOOLS_TTS_VOICE"` - Model string `json:"model" env:"PICOCLAW_TOOLS_TTS_MODEL"` - Format string `json:"format" env:"PICOCLAW_TOOLS_TTS_FORMAT"` - Speed float64 `json:"speed" env:"PICOCLAW_TOOLS_TTS_SPEED"` + Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_TTS_ENABLED"` + APIBase string `json:"api_base" env:"PICOCLAW_TOOLS_TTS_API_BASE"` + Voice string `json:"voice" env:"PICOCLAW_TOOLS_TTS_VOICE"` + Model string `json:"model" env:"PICOCLAW_TOOLS_TTS_MODEL"` + Format string `json:"format" env:"PICOCLAW_TOOLS_TTS_FORMAT"` + Speed float64 `json:"speed" env:"PICOCLAW_TOOLS_TTS_SPEED"` + Exaggeration float64 `json:"exaggeration" env:"PICOCLAW_TOOLS_TTS_EXAGGERATION"` // Chatterbox: emotion expressiveness 0.0–1.0 + CFGWeight float64 `json:"cfg_weight" env:"PICOCLAW_TOOLS_TTS_CFG_WEIGHT"` // Chatterbox: voice guidance weight 0.0–1.0 } type ToolsConfig struct { @@ -343,12 +345,14 @@ func DefaultConfig() *Config { APIBase: "http://localhost:8200", }, TTS: TTSConfig{ - Enabled: false, - APIBase: "http://localhost:8100", - Voice: "en_us-lessac-medium", - Model: "tts-1", - Format: "mp3", - Speed: 1.0, + Enabled: false, + APIBase: "http://localhost:8100", + Voice: "en_us-lessac-medium", + Model: "tts-1", + Format: "mp3", + Speed: 1.0, + Exaggeration: 0.5, + CFGWeight: 0.5, }, }, Heartbeat: HeartbeatConfig{ diff --git a/pkg/voice/kokoro.go b/pkg/voice/kokoro.go index a73d80f31..e42269567 100644 --- a/pkg/voice/kokoro.go +++ b/pkg/voice/kokoro.go @@ -8,23 +8,28 @@ import ( "io" "net/http" "os" + "strings" "time" "github.com/sipeed/picoclaw/pkg/logger" ) // KokoroSynthesizer uses any OpenAI-compatible /v1/audio/speech endpoint -// (Kokoro, Piper, Chatterbox, OpenAI, etc.). +// (Kokoro, Piper, OpenAI, etc.) and also supports Chatterbox's native +// /synthesize endpoint for exaggeration and cfg_weight control. type KokoroSynthesizer struct { - apiBase string - voice string - model string - format string - speed float64 - httpClient *http.Client + apiBase string + voice string + model string + format string + speed float64 + exaggeration float64 + cfgWeight float64 + httpClient *http.Client } -type kokoroRequest struct { +// openaiRequest is the body for the standard /v1/audio/speech endpoint. +type openaiRequest struct { Model string `json:"model"` Input string `json:"input"` Voice string `json:"voice"` @@ -32,13 +37,25 @@ type kokoroRequest struct { Speed float64 `json:"speed,omitempty"` } +// chatterboxRequest is the body for Chatterbox's native /synthesize endpoint. +// Used when model starts with "chatterbox" — gives access to emotion controls. +type chatterboxRequest struct { + Text string `json:"text"` + Voice string `json:"voice,omitempty"` + Exaggeration float64 `json:"exaggeration"` + CFGWeight float64 `json:"cfg_weight"` + Format string `json:"format,omitempty"` +} + // TTSProfile holds the full voice profile for the synthesizer. type TTSProfile struct { - APIBase string - Voice string - Model string - Format string - Speed float64 + APIBase string + Voice string + Model string + Format string + Speed float64 + Exaggeration float64 // Chatterbox only: emotion expressiveness 0.0–1.0 + CFGWeight float64 // Chatterbox only: voice guidance weight 0.0–1.0 } // NewKokoroSynthesizer creates a TTS client from a voice profile. @@ -67,21 +84,31 @@ func NewKokoroSynthesizerFromProfile(p TTSProfile) *KokoroSynthesizer { if p.Speed == 0 { p.Speed = 1.0 } + if p.Exaggeration == 0 { + p.Exaggeration = 0.5 + } + if p.CFGWeight == 0 { + p.CFGWeight = 0.5 + } logger.InfoCF("voice", "Creating TTS synthesizer", map[string]interface{}{ - "api_base": p.APIBase, - "voice": p.Voice, - "model": p.Model, - "format": p.Format, - "speed": p.Speed, + "api_base": p.APIBase, + "voice": p.Voice, + "model": p.Model, + "format": p.Format, + "speed": p.Speed, + "exaggeration": p.Exaggeration, + "cfg_weight": p.CFGWeight, }) return &KokoroSynthesizer{ - apiBase: p.APIBase, - voice: p.Voice, - model: p.Model, - format: p.Format, - speed: p.Speed, + apiBase: p.APIBase, + voice: p.Voice, + model: p.Model, + format: p.Format, + speed: p.Speed, + exaggeration: p.Exaggeration, + cfgWeight: p.CFGWeight, httpClient: &http.Client{ Timeout: 60 * time.Second, }, @@ -90,26 +117,52 @@ func NewKokoroSynthesizerFromProfile(p TTSProfile) *KokoroSynthesizer { // Synthesize converts text to audio, writes it to a temp file, and returns the path. // The caller must delete the file when done. +// isChatterbox returns true when the configured model targets the Chatterbox +// server, which exposes a richer /synthesize endpoint alongside the standard +// /v1/audio/speech one. +func (s *KokoroSynthesizer) isChatterbox() bool { + return strings.HasPrefix(strings.ToLower(s.model), "chatterbox") +} + func (s *KokoroSynthesizer) Synthesize(ctx context.Context, text string) (string, error) { logger.InfoCF("voice", "Synthesizing speech", map[string]interface{}{ - "text_length": len(text), - "voice": s.voice, + "text_length": len(text), + "voice": s.voice, + "model": s.model, + "chatterbox": s.isChatterbox(), }) - reqBody := kokoroRequest{ - Model: s.model, - Input: text, - Voice: s.voice, - Format: s.format, - Speed: s.speed, - } + var ( + bodyBytes []byte + url string + err error + ) - bodyBytes, err := json.Marshal(reqBody) + if s.isChatterbox() { + // Chatterbox native endpoint — supports exaggeration and cfg_weight. + url = s.apiBase + "/synthesize" + bodyBytes, err = json.Marshal(chatterboxRequest{ + Text: text, + Voice: s.voice, + Exaggeration: s.exaggeration, + CFGWeight: s.cfgWeight, + Format: s.format, + }) + } else { + // Standard OpenAI-compatible endpoint. + url = s.apiBase + "/v1/audio/speech" + bodyBytes, err = json.Marshal(openaiRequest{ + Model: s.model, + Input: text, + Voice: s.voice, + Format: s.format, + Speed: s.speed, + }) + } if err != nil { return "", fmt.Errorf("failed to marshal TTS request: %w", err) } - url := s.apiBase + "/v1/audio/speech" req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes)) if err != nil { return "", fmt.Errorf("failed to create TTS request: %w", err) @@ -124,10 +177,9 @@ func (s *KokoroSynthesizer) Synthesize(ctx context.Context, text string) (string if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) - return "", fmt.Errorf("Kokoro TTS error (status %d): %s", resp.StatusCode, string(body)) + return "", fmt.Errorf("TTS error (status %d): %s", resp.StatusCode, string(body)) } - // Write audio to temp file tmpFile, err := os.CreateTemp("", "picoclaw-tts-*."+s.format) if err != nil { return "", fmt.Errorf("failed to create temp audio file: %w", err)