From 1712650e61f456c02a0b47dc7894c2763691b2c4 Mon Sep 17 00:00:00 2001 From: Huaaudio Date: Tue, 31 Mar 2026 16:02:54 +0200 Subject: [PATCH] add mimo tts --- pkg/audio/tts/mimo_tts.go | 162 ++++++++++++++++++++++++++++++++++++++ pkg/audio/tts/tts.go | 28 +++++-- pkg/audio/tts/tts_test.go | 86 +++++++++++++++++++- 3 files changed, 264 insertions(+), 12 deletions(-) create mode 100644 pkg/audio/tts/mimo_tts.go diff --git a/pkg/audio/tts/mimo_tts.go b/pkg/audio/tts/mimo_tts.go new file mode 100644 index 000000000..a8aee6b8c --- /dev/null +++ b/pkg/audio/tts/mimo_tts.go @@ -0,0 +1,162 @@ +package tts + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/sipeed/picoclaw/pkg/logger" +) + +type MimoTTSProvider struct { + apiKey string + apiBase string + voice string + format string + model string + httpClient *http.Client +} + +func NewMimoTTSProvider(apiKey string, apiBase string, model string, proxyURL string) *MimoTTSProvider { + if apiBase == "" { + apiBase = "https://api.xiaomimimo.com/v1/chat/completions" + } else { + if u, err := url.Parse(apiBase); err == nil && u.Scheme != "" && u.Host != "" { + path := u.Path + if u.Host == "api.xiaomimimo.com" { + if path == "" || path == "/" || path == "/v1" || path == "/v1/" { + path = "/v1/chat/completions" + } else { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + if !strings.HasPrefix(path, "/v1/") { + path = "/v1" + strings.TrimSuffix(path, "/") + } + if !strings.HasSuffix(path, "/chat/completions") { + path = strings.TrimSuffix(path, "/") + "/chat/completions" + } + } + } else { + if !strings.HasSuffix(path, "/chat/completions") { + path = strings.TrimSuffix(path, "/") + "/chat/completions" + } + } + u.Path = path + apiBase = u.String() + } else { + if apiBase == "https://api.xiaomimimo.com/v1" { + apiBase = "https://api.xiaomimimo.com/v1/chat/completions" + } else if !strings.HasSuffix(apiBase, "/chat/completions") { + apiBase = strings.TrimSuffix(apiBase, "/") + "/chat/completions" + } + } + } + + model = strings.TrimSpace(model) + if model == "" { + model = "mimo-v2-tts" + } + + client := &http.Client{Timeout: 60 * time.Second} + if proxyURL != "" { + if pURL, err := url.Parse(proxyURL); err == nil { + client.Transport = &http.Transport{Proxy: http.ProxyURL(pURL)} + } else { + logger.WarnF( + "NewMimoTTSProvider: invalid proxy URL; proceeding without proxy", + map[string]any{"proxyURL": proxyURL, "error": err}, + ) + } + } + + return &MimoTTSProvider{ + apiKey: apiKey, + apiBase: apiBase, + voice: "default_zh", // mimo_default now seems to be an alias for default_en, which is not working for Chinese TTS. default_zh seems to work fine with both English and Chinese, and is likely the intended default for TTS. + format: "mp3", + model: model, + httpClient: client, + } +} + +func (t *MimoTTSProvider) Name() string { + return "mimo-tts" +} + +func (t *MimoTTSProvider) Synthesize(ctx context.Context, text string) (io.ReadCloser, error) { + logger.DebugCF("voice-tts", "Starting TTS synthesis", map[string]any{"text_len": len(text), "provider": t.Name()}) + + reqBody := map[string]any{ + "model": t.model, + "messages": []map[string]string{ + {"role": "assistant", "content": text}, + }, + "audio": map[string]string{ + "format": t.format, + "voice": t.voice, + }, + "stream": false, + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", t.apiBase, bytes.NewReader(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Api-Key", t.apiKey) + + resp, err := t.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body)) + } + + var payload struct { + Choices []struct { + Message struct { + Audio struct { + Data string `json:"data"` + } `json:"audio"` + } `json:"message"` + } `json:"choices"` + } + + err = json.Unmarshal(body, &payload) + if err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + if len(payload.Choices) == 0 || payload.Choices[0].Message.Audio.Data == "" { + return nil, fmt.Errorf("invalid TTS response: missing audio data") + } + + audioBytes, err := base64.StdEncoding.DecodeString(payload.Choices[0].Message.Audio.Data) + if err != nil { + return nil, fmt.Errorf("failed to decode audio data: %w", err) + } + + return io.NopCloser(bytes.NewReader(audioBytes)), nil +} diff --git a/pkg/audio/tts/tts.go b/pkg/audio/tts/tts.go index 9a012e026..99a9ef203 100644 --- a/pkg/audio/tts/tts.go +++ b/pkg/audio/tts/tts.go @@ -24,12 +24,17 @@ func providerFromModelConfig(mc *config.ModelConfig) TTSProvider { return nil } - _, modelID := providers.ExtractProtocol(mc.Model) + protocol, modelID := providers.ExtractProtocol(mc.Model) if modelID == "" { modelID = strings.TrimSpace(mc.Model) } - return NewOpenAITTSProvider(mc.APIKey(), providers.ResolveAPIBase(mc), mc.Proxy, modelID) + switch protocol { + case "mimo": + return NewMimoTTSProvider(mc.APIKey(), providers.ResolveAPIBase(mc), modelID, mc.Proxy) + default: + return NewOpenAITTSProvider(mc.APIKey(), providers.ResolveAPIBase(mc), mc.Proxy, modelID) + } } func DetectTTS(cfg *config.Config) TTSProvider { @@ -89,7 +94,14 @@ func SynthesizeAndStore( return "", fmt.Errorf("failed to create media temp dir: %w", err) } - file, err := os.CreateTemp(media.TempDir(), "tts-*.ogg") + fileExt := ".ogg" + contentType := "audio/ogg" + if provider.Name() == "mimo-tts" { + fileExt = ".mp3" + contentType = "audio/mpeg" + } + + file, err := os.CreateTemp(media.TempDir(), "tts-*"+fileExt) if err != nil { return "", fmt.Errorf("failed to create temp file: %w", err) } @@ -114,20 +126,20 @@ func SynthesizeAndStore( filename = strings.TrimSpace(filename) if filename == "" { - filename = fmt.Sprintf("tts-%d.ogg", time.Now().Unix()) + filename = fmt.Sprintf("tts-%d%s", time.Now().Unix(), fileExt) } ext := strings.ToLower(filepath.Ext(filename)) if ext == "" { - filename += ".ogg" - } else if ext != ".ogg" { - filename = strings.TrimSuffix(filename, filepath.Ext(filename)) + ".ogg" + filename += fileExt + } else if ext != fileExt { + filename = strings.TrimSuffix(filename, filepath.Ext(filename)) + fileExt } scope := fmt.Sprintf("tool:send_tts:%s:%s:%d", channel, chatID, time.Now().UnixNano()) ref, err := store.Store(file.Name(), media.MediaMeta{ Filename: filename, - ContentType: "audio/ogg", + ContentType: contentType, Source: "tool:send_tts", }, scope) if err != nil { diff --git a/pkg/audio/tts/tts_test.go b/pkg/audio/tts/tts_test.go index 158314e3b..053aa7220 100644 --- a/pkg/audio/tts/tts_test.go +++ b/pkg/audio/tts/tts_test.go @@ -6,10 +6,12 @@ import ( "io" "net/http" "net/http/httptest" + "path/filepath" "strings" "testing" "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/media" ) func TestNewOpenAITTSProvider_APIBaseNormalization(t *testing.T) { @@ -142,7 +144,7 @@ func TestNewOpenAITTSProvider_UsesConfiguredModel(t *testing.T) { } } -func TestDetectTTS_UsesConfiguredModelAndProviderBase(t *testing.T) { +func TestDetectTTS_UsesMimoProviderForMimoModels(t *testing.T) { t.Parallel() provider := DetectTTS(&config.Config{ @@ -156,14 +158,90 @@ func TestDetectTTS_UsesConfiguredModelAndProviderBase(t *testing.T) { }, }) - ttsProvider, ok := provider.(*OpenAITTSProvider) + ttsProvider, ok := provider.(*MimoTTSProvider) if !ok { - t.Fatalf("DetectTTS() type = %T, want *OpenAITTSProvider", provider) + t.Fatalf("DetectTTS() type = %T, want *MimoTTSProvider", provider) } if ttsProvider.model != "mimo-v2-tts" { t.Fatalf("model mismatch: got %q, want %q", ttsProvider.model, "mimo-v2-tts") } - if ttsProvider.apiBase != "https://api.xiaomimimo.com/v1/audio/speech" { + if ttsProvider.apiBase != "https://api.xiaomimimo.com/v1/chat/completions" { t.Fatalf("apiBase mismatch: got %q", ttsProvider.apiBase) } } + +type stubTTSProvider struct { + name string +} + +func (s stubTTSProvider) Name() string { + return s.name +} + +func (s stubTTSProvider) Synthesize(ctx context.Context, text string) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("audio")), nil +} + +func TestSynthesizeAndStore_UsesOggMetadataByDefault(t *testing.T) { + t.Parallel() + + store := media.NewFileMediaStore() + ref, err := SynthesizeAndStore( + context.Background(), + stubTTSProvider{name: "openai-tts"}, + store, + "hello", + "", + "discord", + "chat123", + ) + if err != nil { + t.Fatalf("SynthesizeAndStore failed: %v", err) + } + + path, meta, err := store.ResolveWithMeta(ref) + if err != nil { + t.Fatalf("ResolveWithMeta failed: %v", err) + } + if meta.ContentType != "audio/ogg" { + t.Fatalf("ContentType = %q, want %q", meta.ContentType, "audio/ogg") + } + if filepath.Ext(path) != ".ogg" { + t.Fatalf("stored file extension = %q, want %q", filepath.Ext(path), ".ogg") + } + if filepath.Ext(meta.Filename) != ".ogg" { + t.Fatalf("filename extension = %q, want %q", filepath.Ext(meta.Filename), ".ogg") + } +} + +func TestSynthesizeAndStore_UsesMp3MetadataForMimo(t *testing.T) { + t.Parallel() + + store := media.NewFileMediaStore() + ref, err := SynthesizeAndStore( + context.Background(), + stubTTSProvider{name: "mimo-tts"}, + store, + "hello", + "", + "discord", + "chat123", + ) + if err != nil { + t.Fatalf("SynthesizeAndStore failed: %v", err) + } + + path, meta, err := store.ResolveWithMeta(ref) + if err != nil { + t.Fatalf("ResolveWithMeta failed: %v", err) + } + if meta.ContentType != "audio/mpeg" { + t.Fatalf("ContentType = %q, want %q", meta.ContentType, "audio/mpeg") + } + if filepath.Ext(path) != ".mp3" { + t.Fatalf("stored file extension = %q, want %q", filepath.Ext(path), ".mp3") + } + if filepath.Ext(meta.Filename) != ".mp3" { + t.Fatalf("filename extension = %q, want %q", filepath.Ext(meta.Filename), ".mp3") + } +}