feat(voice/tts): add Chatterbox support with emotion controls
Chatterbox exposes a /synthesize endpoint alongside the standard
/v1/audio/speech one. The native endpoint adds two parameters unavailable
in the OpenAI-compatible API:
- exaggeration (0.0–1.0): emotional expressiveness of the voice
- cfg_weight (0.0–1.0): how closely the voice follows the prompt
Routing: when model starts with 'chatterbox' (case-insensitive), Synthesize()
posts to /synthesize with the Chatterbox body; otherwise it uses the standard
/v1/audio/speech path. All other backends are unaffected.
Changes:
- kokoro.go: chatterboxRequest struct, isChatterbox() helper, Synthesize()
branching logic, exaggeration/cfgWeight fields on KokoroSynthesizer
- TTSProfile: Exaggeration + CFGWeight fields (defaults: 0.5 / 0.5)
- config.go: TTSConfig gains Exaggeration + CFGWeight (env-overridable)
- main.go: wire new fields through TTSProfile
- config.example.json: document exaggeration + cfg_weight
Chatterbox config example:
"tts": {
"enabled": true,
"api_base": "http://localhost:8100",
"model": "chatterbox-1",
"voice": "default",
"format": "mp3",
"exaggeration": 0.5,
"cfg_weight": 0.5
}
This commit is contained in:
parent
1e68aa12b0
commit
ef5c2de460
4 changed files with 114 additions and 54 deletions
|
|
@ -637,11 +637,13 @@ func gatewayCmd() {
|
|||
// Attach TTS synthesis callbacks to the message tool (enables voice=true).
|
||||
if cfg.Tools.TTS.Enabled {
|
||||
synthesizer := voice.NewKokoroSynthesizerFromProfile(voice.TTSProfile{
|
||||
APIBase: cfg.Tools.TTS.APIBase,
|
||||
Voice: cfg.Tools.TTS.Voice,
|
||||
Model: cfg.Tools.TTS.Model,
|
||||
Format: cfg.Tools.TTS.Format,
|
||||
Speed: cfg.Tools.TTS.Speed,
|
||||
APIBase: cfg.Tools.TTS.APIBase,
|
||||
Voice: cfg.Tools.TTS.Voice,
|
||||
Model: cfg.Tools.TTS.Model,
|
||||
Format: cfg.Tools.TTS.Format,
|
||||
Speed: cfg.Tools.TTS.Speed,
|
||||
Exaggeration: cfg.Tools.TTS.Exaggeration,
|
||||
CFGWeight: cfg.Tools.TTS.CFGWeight,
|
||||
})
|
||||
if synthesizer.IsAvailable() {
|
||||
logger.InfoCF("voice", "TTS enabled — voice=true supported in message tool", map[string]interface{}{
|
||||
|
|
|
|||
|
|
@ -128,7 +128,9 @@
|
|||
"voice": "en_us-lessac-medium",
|
||||
"model": "tts-1",
|
||||
"format": "mp3",
|
||||
"speed": 1.0
|
||||
"speed": 1.0,
|
||||
"exaggeration": 0.5,
|
||||
"cfg_weight": 0.5
|
||||
}
|
||||
},
|
||||
"heartbeat": {
|
||||
|
|
|
|||
|
|
@ -217,12 +217,14 @@ type WhisperConfig struct {
|
|||
}
|
||||
|
||||
type TTSConfig struct {
|
||||
Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_TTS_ENABLED"`
|
||||
APIBase string `json:"api_base" env:"PICOCLAW_TOOLS_TTS_API_BASE"`
|
||||
Voice string `json:"voice" env:"PICOCLAW_TOOLS_TTS_VOICE"`
|
||||
Model string `json:"model" env:"PICOCLAW_TOOLS_TTS_MODEL"`
|
||||
Format string `json:"format" env:"PICOCLAW_TOOLS_TTS_FORMAT"`
|
||||
Speed float64 `json:"speed" env:"PICOCLAW_TOOLS_TTS_SPEED"`
|
||||
Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_TTS_ENABLED"`
|
||||
APIBase string `json:"api_base" env:"PICOCLAW_TOOLS_TTS_API_BASE"`
|
||||
Voice string `json:"voice" env:"PICOCLAW_TOOLS_TTS_VOICE"`
|
||||
Model string `json:"model" env:"PICOCLAW_TOOLS_TTS_MODEL"`
|
||||
Format string `json:"format" env:"PICOCLAW_TOOLS_TTS_FORMAT"`
|
||||
Speed float64 `json:"speed" env:"PICOCLAW_TOOLS_TTS_SPEED"`
|
||||
Exaggeration float64 `json:"exaggeration" env:"PICOCLAW_TOOLS_TTS_EXAGGERATION"` // Chatterbox: emotion expressiveness 0.0–1.0
|
||||
CFGWeight float64 `json:"cfg_weight" env:"PICOCLAW_TOOLS_TTS_CFG_WEIGHT"` // Chatterbox: voice guidance weight 0.0–1.0
|
||||
}
|
||||
|
||||
type ToolsConfig struct {
|
||||
|
|
@ -343,12 +345,14 @@ func DefaultConfig() *Config {
|
|||
APIBase: "http://localhost:8200",
|
||||
},
|
||||
TTS: TTSConfig{
|
||||
Enabled: false,
|
||||
APIBase: "http://localhost:8100",
|
||||
Voice: "en_us-lessac-medium",
|
||||
Model: "tts-1",
|
||||
Format: "mp3",
|
||||
Speed: 1.0,
|
||||
Enabled: false,
|
||||
APIBase: "http://localhost:8100",
|
||||
Voice: "en_us-lessac-medium",
|
||||
Model: "tts-1",
|
||||
Format: "mp3",
|
||||
Speed: 1.0,
|
||||
Exaggeration: 0.5,
|
||||
CFGWeight: 0.5,
|
||||
},
|
||||
},
|
||||
Heartbeat: HeartbeatConfig{
|
||||
|
|
|
|||
|
|
@ -8,23 +8,28 @@ import (
|
|||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/sipeed/picoclaw/pkg/logger"
|
||||
)
|
||||
|
||||
// KokoroSynthesizer uses any OpenAI-compatible /v1/audio/speech endpoint
|
||||
// (Kokoro, Piper, Chatterbox, OpenAI, etc.).
|
||||
// (Kokoro, Piper, OpenAI, etc.) and also supports Chatterbox's native
|
||||
// /synthesize endpoint for exaggeration and cfg_weight control.
|
||||
type KokoroSynthesizer struct {
|
||||
apiBase string
|
||||
voice string
|
||||
model string
|
||||
format string
|
||||
speed float64
|
||||
httpClient *http.Client
|
||||
apiBase string
|
||||
voice string
|
||||
model string
|
||||
format string
|
||||
speed float64
|
||||
exaggeration float64
|
||||
cfgWeight float64
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
type kokoroRequest struct {
|
||||
// openaiRequest is the body for the standard /v1/audio/speech endpoint.
|
||||
type openaiRequest struct {
|
||||
Model string `json:"model"`
|
||||
Input string `json:"input"`
|
||||
Voice string `json:"voice"`
|
||||
|
|
@ -32,13 +37,25 @@ type kokoroRequest struct {
|
|||
Speed float64 `json:"speed,omitempty"`
|
||||
}
|
||||
|
||||
// chatterboxRequest is the body for Chatterbox's native /synthesize endpoint.
|
||||
// Used when model starts with "chatterbox" — gives access to emotion controls.
|
||||
type chatterboxRequest struct {
|
||||
Text string `json:"text"`
|
||||
Voice string `json:"voice,omitempty"`
|
||||
Exaggeration float64 `json:"exaggeration"`
|
||||
CFGWeight float64 `json:"cfg_weight"`
|
||||
Format string `json:"format,omitempty"`
|
||||
}
|
||||
|
||||
// TTSProfile holds the full voice profile for the synthesizer.
|
||||
type TTSProfile struct {
|
||||
APIBase string
|
||||
Voice string
|
||||
Model string
|
||||
Format string
|
||||
Speed float64
|
||||
APIBase string
|
||||
Voice string
|
||||
Model string
|
||||
Format string
|
||||
Speed float64
|
||||
Exaggeration float64 // Chatterbox only: emotion expressiveness 0.0–1.0
|
||||
CFGWeight float64 // Chatterbox only: voice guidance weight 0.0–1.0
|
||||
}
|
||||
|
||||
// NewKokoroSynthesizer creates a TTS client from a voice profile.
|
||||
|
|
@ -67,21 +84,31 @@ func NewKokoroSynthesizerFromProfile(p TTSProfile) *KokoroSynthesizer {
|
|||
if p.Speed == 0 {
|
||||
p.Speed = 1.0
|
||||
}
|
||||
if p.Exaggeration == 0 {
|
||||
p.Exaggeration = 0.5
|
||||
}
|
||||
if p.CFGWeight == 0 {
|
||||
p.CFGWeight = 0.5
|
||||
}
|
||||
|
||||
logger.InfoCF("voice", "Creating TTS synthesizer", map[string]interface{}{
|
||||
"api_base": p.APIBase,
|
||||
"voice": p.Voice,
|
||||
"model": p.Model,
|
||||
"format": p.Format,
|
||||
"speed": p.Speed,
|
||||
"api_base": p.APIBase,
|
||||
"voice": p.Voice,
|
||||
"model": p.Model,
|
||||
"format": p.Format,
|
||||
"speed": p.Speed,
|
||||
"exaggeration": p.Exaggeration,
|
||||
"cfg_weight": p.CFGWeight,
|
||||
})
|
||||
|
||||
return &KokoroSynthesizer{
|
||||
apiBase: p.APIBase,
|
||||
voice: p.Voice,
|
||||
model: p.Model,
|
||||
format: p.Format,
|
||||
speed: p.Speed,
|
||||
apiBase: p.APIBase,
|
||||
voice: p.Voice,
|
||||
model: p.Model,
|
||||
format: p.Format,
|
||||
speed: p.Speed,
|
||||
exaggeration: p.Exaggeration,
|
||||
cfgWeight: p.CFGWeight,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 60 * time.Second,
|
||||
},
|
||||
|
|
@ -90,26 +117,52 @@ func NewKokoroSynthesizerFromProfile(p TTSProfile) *KokoroSynthesizer {
|
|||
|
||||
// Synthesize converts text to audio, writes it to a temp file, and returns the path.
|
||||
// The caller must delete the file when done.
|
||||
// isChatterbox returns true when the configured model targets the Chatterbox
|
||||
// server, which exposes a richer /synthesize endpoint alongside the standard
|
||||
// /v1/audio/speech one.
|
||||
func (s *KokoroSynthesizer) isChatterbox() bool {
|
||||
return strings.HasPrefix(strings.ToLower(s.model), "chatterbox")
|
||||
}
|
||||
|
||||
func (s *KokoroSynthesizer) Synthesize(ctx context.Context, text string) (string, error) {
|
||||
logger.InfoCF("voice", "Synthesizing speech", map[string]interface{}{
|
||||
"text_length": len(text),
|
||||
"voice": s.voice,
|
||||
"text_length": len(text),
|
||||
"voice": s.voice,
|
||||
"model": s.model,
|
||||
"chatterbox": s.isChatterbox(),
|
||||
})
|
||||
|
||||
reqBody := kokoroRequest{
|
||||
Model: s.model,
|
||||
Input: text,
|
||||
Voice: s.voice,
|
||||
Format: s.format,
|
||||
Speed: s.speed,
|
||||
}
|
||||
var (
|
||||
bodyBytes []byte
|
||||
url string
|
||||
err error
|
||||
)
|
||||
|
||||
bodyBytes, err := json.Marshal(reqBody)
|
||||
if s.isChatterbox() {
|
||||
// Chatterbox native endpoint — supports exaggeration and cfg_weight.
|
||||
url = s.apiBase + "/synthesize"
|
||||
bodyBytes, err = json.Marshal(chatterboxRequest{
|
||||
Text: text,
|
||||
Voice: s.voice,
|
||||
Exaggeration: s.exaggeration,
|
||||
CFGWeight: s.cfgWeight,
|
||||
Format: s.format,
|
||||
})
|
||||
} else {
|
||||
// Standard OpenAI-compatible endpoint.
|
||||
url = s.apiBase + "/v1/audio/speech"
|
||||
bodyBytes, err = json.Marshal(openaiRequest{
|
||||
Model: s.model,
|
||||
Input: text,
|
||||
Voice: s.voice,
|
||||
Format: s.format,
|
||||
Speed: s.speed,
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to marshal TTS request: %w", err)
|
||||
}
|
||||
|
||||
url := s.apiBase + "/v1/audio/speech"
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create TTS request: %w", err)
|
||||
|
|
@ -124,10 +177,9 @@ func (s *KokoroSynthesizer) Synthesize(ctx context.Context, text string) (string
|
|||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return "", fmt.Errorf("Kokoro TTS error (status %d): %s", resp.StatusCode, string(body))
|
||||
return "", fmt.Errorf("TTS error (status %d): %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
// Write audio to temp file
|
||||
tmpFile, err := os.CreateTemp("", "picoclaw-tts-*."+s.format)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create temp audio file: %w", err)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue