diff --git a/pkg/audio/asr/asr.go b/pkg/audio/asr/asr.go index 1482f40bb..7c9ee9f8c 100644 --- a/pkg/audio/asr/asr.go +++ b/pkg/audio/asr/asr.go @@ -57,7 +57,7 @@ func supportsWhisperTranscription(modelCfg *config.ModelConfig) bool { } func whisperModelID(modelCfg *config.ModelConfig) string { - if modelCfg == nil || modelCfg.APIKey() == "" { + if modelCfg == nil { return "" } @@ -66,7 +66,11 @@ func whisperModelID(modelCfg *config.ModelConfig) string { } _, modelID := providers.ExtractProtocol(modelCfg) - if strings.Contains(strings.ToLower(modelID), "whisper") { + normalized := strings.ToLower(modelID) + if strings.Contains(normalized, "whisper") || strings.Contains(normalized, "transcribe") { + if modelCfg.APIKey() == "" && modelCfg.AuthMethod != "oauth" { + return "" + } return modelID } return "" diff --git a/pkg/audio/asr/whisper_transcriber.go b/pkg/audio/asr/whisper_transcriber.go index fc1101e1c..5367c31bd 100644 --- a/pkg/audio/asr/whisper_transcriber.go +++ b/pkg/audio/asr/whisper_transcriber.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/sipeed/picoclaw/pkg/auth" "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/providers" @@ -24,6 +25,7 @@ type WhisperTranscriber struct { apiBase string modelID string providerName string + tokenSource func() (string, error) httpClient *http.Client } @@ -46,12 +48,17 @@ func NewWhisperTranscriber(modelCfg *config.ModelConfig) *WhisperTranscriber { if tr == nil { return nil } + if modelCfg.AuthMethod == "oauth" && protocol == "openai" { + tr.tokenSource = createOpenAITranscriptionTokenSource() + } logger.DebugCF("voice", "Creating whisper transcriber", map[string]any{ - "api_base": tr.apiBase, - "has_key": tr.apiKey != "", - "model": tr.modelID, - "provider": tr.providerName, + "api_base": tr.apiBase, + "auth_method": modelCfg.AuthMethod, + "has_key": tr.apiKey != "", + "has_oauth": tr.tokenSource != nil, + "model": tr.modelID, + "provider": tr.providerName, }) return tr } @@ -86,6 +93,13 @@ func (t *WhisperTranscriber) transcriptionURL() string { return base + "/audio/transcriptions" } +func (t *WhisperTranscriber) authorizationToken() (string, error) { + if t.tokenSource != nil { + return t.tokenSource() + } + return t.apiKey, nil +} + func (t *WhisperTranscriber) TranscribeData( ctx context.Context, data []byte, @@ -189,8 +203,13 @@ func (t *WhisperTranscriber) doRequest( } req.Header.Set("Content-Type", contentType) - if t.apiKey != "" { - req.Header.Set("Authorization", "Bearer "+t.apiKey) + token, err := t.authorizationToken() + if err != nil { + logger.ErrorCF("voice", "Failed to load transcription auth token", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to load transcription auth token: %w", err) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) } logger.DebugCF("voice", "Sending whisper transcription request", map[string]any{ @@ -243,3 +262,10 @@ func (t *WhisperTranscriber) doRequest( func (t *WhisperTranscriber) Name() string { return "whisper" } + +func createOpenAITranscriptionTokenSource() func() (string, error) { + return func() (string, error) { + token, _, err := auth.GetOpenAIToken() + return token, err + } +} diff --git a/pkg/audio/asr/whisper_transcriber_test.go b/pkg/audio/asr/whisper_transcriber_test.go index a2a5178d1..ff506f974 100644 --- a/pkg/audio/asr/whisper_transcriber_test.go +++ b/pkg/audio/asr/whisper_transcriber_test.go @@ -100,3 +100,63 @@ func TestWhisperTranscriberUsesEndpointAPIBaseWithoutDoubleAppend(t *testing.T) t.Errorf("path = %q, want %q", gotPath, "/audio/transcriptions") } } + +func TestWhisperTranscriberUsesOAuthTokenSource(t *testing.T) { + var gotAuth string + var gotModel string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + + reader, err := r.MultipartReader() + if err != nil { + t.Fatalf("MultipartReader() error: %v", err) + } + for { + part, err := reader.NextPart() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("NextPart() error: %v", err) + } + data, err := io.ReadAll(part) + if err != nil { + t.Fatalf("ReadAll() error: %v", err) + } + if part.FormName() == "model" { + gotModel = string(data) + } + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(TranscriptionResponse{Text: "oauth transcription"}); err != nil { + t.Fatalf("Encode() error: %v", err) + } + })) + defer server.Close() + + tr := NewWhisperTranscriber(&config.ModelConfig{ + Model: "openai/gpt-4o-transcribe", + APIBase: server.URL, + AuthMethod: "oauth", + }) + tr.httpClient = server.Client() + tr.tokenSource = func() (string, error) { + return "oauth-token", nil + } + + resp, err := tr.TranscribeData(context.Background(), []byte("audio"), "clip.mp3") + if err != nil { + t.Fatalf("TranscribeData() error: %v", err) + } + if resp.Text != "oauth transcription" { + t.Errorf("Text = %q, want %q", resp.Text, "oauth transcription") + } + if gotAuth != "Bearer oauth-token" { + t.Errorf("Authorization = %q, want %q", gotAuth, "Bearer oauth-token") + } + if gotModel != "gpt-4o-transcribe" { + t.Errorf("model field = %q, want %q", gotModel, "gpt-4o-transcribe") + } +} diff --git a/pkg/auth/openai.go b/pkg/auth/openai.go new file mode 100644 index 000000000..2cad4537d --- /dev/null +++ b/pkg/auth/openai.go @@ -0,0 +1,32 @@ +package auth + +import "fmt" + +// GetOpenAIToken returns the current OpenAI credential, refreshing OAuth +// credentials when they are close to expiry. The account ID is returned for +// Codex backend calls that require the Chatgpt-Account-Id header. +func GetOpenAIToken() (accessToken, accountID string, err error) { + cred, err := GetCredential("openai") + if err != nil { + return "", "", fmt.Errorf("loading auth credentials: %w", err) + } + if cred == nil { + return "", "", fmt.Errorf("no credentials for openai. Run: picoclaw auth login --provider openai") + } + + if cred.AuthMethod == "oauth" && cred.NeedsRefresh() && cred.RefreshToken != "" { + refreshed, err := RefreshAccessToken(cred, OpenAIOAuthConfig()) + if err != nil { + return "", "", fmt.Errorf("refreshing token: %w", err) + } + if refreshed.AccountID == "" { + refreshed.AccountID = cred.AccountID + } + if err := SetCredential("openai", refreshed); err != nil { + return "", "", fmt.Errorf("saving refreshed token: %w", err) + } + return refreshed.AccessToken, refreshed.AccountID, nil + } + + return cred.AccessToken, cred.AccountID, nil +} diff --git a/pkg/providers/oauth/codex_provider.go b/pkg/providers/oauth/codex_provider.go index 0b125997b..88da42af7 100644 --- a/pkg/providers/oauth/codex_provider.go +++ b/pkg/providers/oauth/codex_provider.go @@ -104,8 +104,16 @@ func (p *CodexProvider) Chat( defer stream.Close() var resp *responses.Response + var streamText strings.Builder for stream.Next() { evt := stream.Current() + if evt.Type == "response.output_text.done" { + textDone := evt.AsResponseOutputTextDone() + if textDone.Text != "" { + streamText.Reset() + streamText.WriteString(textDone.Text) + } + } if evt.Type == "response.completed" || evt.Type == "response.failed" || evt.Type == "response.incomplete" { evtResp := evt.Response if evtResp.ID != "" { @@ -153,7 +161,11 @@ func (p *CodexProvider) Chat( return nil, fmt.Errorf("codex API call: stream ended without completed response") } - return orc.ParseResponseFromStruct(resp), nil + parsed := orc.ParseResponseFromStruct(resp) + if parsed.Content == "" && len(parsed.ToolCalls) == 0 && streamText.Len() > 0 { + parsed.Content = streamText.String() + } + return parsed, nil } func (p *CodexProvider) GetDefaultModel() string { @@ -242,29 +254,6 @@ func buildCodexParams( func CreateCodexTokenSource() func() (string, string, error) { return func() (string, string, error) { - cred, err := auth.GetCredential("openai") - if err != nil { - return "", "", fmt.Errorf("loading auth credentials: %w", err) - } - if cred == nil { - return "", "", fmt.Errorf("no credentials for openai. Run: picoclaw auth login --provider openai") - } - - if cred.AuthMethod == "oauth" && cred.NeedsRefresh() && cred.RefreshToken != "" { - oauthCfg := auth.OpenAIOAuthConfig() - refreshed, err := auth.RefreshAccessToken(cred, oauthCfg) - if err != nil { - return "", "", fmt.Errorf("refreshing token: %w", err) - } - if refreshed.AccountID == "" { - refreshed.AccountID = cred.AccountID - } - if err := auth.SetCredential("openai", refreshed); err != nil { - return "", "", fmt.Errorf("saving refreshed token: %w", err) - } - return refreshed.AccessToken, refreshed.AccountID, nil - } - - return cred.AccessToken, cred.AccountID, nil + return auth.GetOpenAIToken() } }