Merge 647e0db124 into 412705783d

pkg/audio/asr/asr.go

@@ -63,7 +63,7 @@ func supportsWhisperTranscription(modelCfg *config.ModelConfig) bool {
 }
 
 func whisperModelID(modelCfg *config.ModelConfig) string {
-	if modelCfg == nil || modelCfg.APIKey() == "" {
+	if modelCfg == nil {
 		return ""
 	}
 
@@ -72,7 +72,11 @@ func whisperModelID(modelCfg *config.ModelConfig) string {
 	}
 
 	_, modelID := providers.ExtractProtocol(modelCfg)
-	if strings.Contains(strings.ToLower(modelID), "whisper") {
+	normalized := strings.ToLower(modelID)
+	if strings.Contains(normalized, "whisper") || strings.Contains(normalized, "transcribe") {
+		if modelCfg.APIKey() == "" && modelCfg.AuthMethod != "oauth" {
+			return ""
+		}
 		return modelID
 	}
 	return ""

pkg/audio/asr/whisper_transcriber.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/sipeed/picoclaw/pkg/auth"
 	"github.com/sipeed/picoclaw/pkg/config"
 	"github.com/sipeed/picoclaw/pkg/logger"
 	"github.com/sipeed/picoclaw/pkg/providers"
@@ -24,6 +25,7 @@ type WhisperTranscriber struct {
 	apiBase      string
 	modelID      string
 	providerName string
+	tokenSource  func() (string, error)
 	httpClient   *http.Client
 }
 
@@ -46,10 +48,15 @@ func NewWhisperTranscriber(modelCfg *config.ModelConfig) *WhisperTranscriber {
 	if tr == nil {
 		return nil
 	}
+	if modelCfg.AuthMethod == "oauth" && protocol == "openai" {
+		tr.tokenSource = createOpenAITranscriptionTokenSource()
+	}
 
 	logger.DebugCF("voice", "Creating whisper transcriber", map[string]any{
 		"api_base":    tr.apiBase,
+		"auth_method": modelCfg.AuthMethod,
 		"has_key":     tr.apiKey != "",
+		"has_oauth":   tr.tokenSource != nil,
 		"model":       tr.modelID,
 		"provider":    tr.providerName,
 	})
@@ -86,6 +93,13 @@ func (t *WhisperTranscriber) transcriptionURL() string {
 	return base + "/audio/transcriptions"
 }
 
+func (t *WhisperTranscriber) authorizationToken() (string, error) {
+	if t.tokenSource != nil {
+		return t.tokenSource()
+	}
+	return t.apiKey, nil
+}
+
 func (t *WhisperTranscriber) TranscribeData(
 	ctx context.Context,
 	data []byte,
@@ -189,8 +203,13 @@ func (t *WhisperTranscriber) doRequest(
 	}
 
 	req.Header.Set("Content-Type", contentType)
-	if t.apiKey != "" {
+	token, err := t.authorizationToken()
-		req.Header.Set("Authorization", "Bearer "+t.apiKey)
+	if err != nil {
+		logger.ErrorCF("voice", "Failed to load transcription auth token", map[string]any{"error": err})
+		return nil, fmt.Errorf("failed to load transcription auth token: %w", err)
+	}
+	if token != "" {
+		req.Header.Set("Authorization", "Bearer "+token)
 	}
 
 	logger.DebugCF("voice", "Sending whisper transcription request", map[string]any{
@@ -243,3 +262,10 @@ func (t *WhisperTranscriber) doRequest(
 func (t *WhisperTranscriber) Name() string {
 	return "whisper"
 }
+
+func createOpenAITranscriptionTokenSource() func() (string, error) {
+	return func() (string, error) {
+		token, _, err := auth.GetOpenAIToken()
+		return token, err
+	}
+}

pkg/audio/asr/whisper_transcriber_test.go

@@ -100,3 +100,63 @@ func TestWhisperTranscriberUsesEndpointAPIBaseWithoutDoubleAppend(t *testing.T)
 		t.Errorf("path = %q, want %q", gotPath, "/audio/transcriptions")
 	}
 }
+
+func TestWhisperTranscriberUsesOAuthTokenSource(t *testing.T) {
+	var gotAuth string
+	var gotModel string
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotAuth = r.Header.Get("Authorization")
+
+		reader, err := r.MultipartReader()
+		if err != nil {
+			t.Fatalf("MultipartReader() error: %v", err)
+		}
+		for {
+			part, err := reader.NextPart()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				t.Fatalf("NextPart() error: %v", err)
+			}
+			data, err := io.ReadAll(part)
+			if err != nil {
+				t.Fatalf("ReadAll() error: %v", err)
+			}
+			if part.FormName() == "model" {
+				gotModel = string(data)
+			}
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		if err := json.NewEncoder(w).Encode(TranscriptionResponse{Text: "oauth transcription"}); err != nil {
+			t.Fatalf("Encode() error: %v", err)
+		}
+	}))
+	defer server.Close()
+
+	tr := NewWhisperTranscriber(&config.ModelConfig{
+		Model:      "openai/gpt-4o-transcribe",
+		APIBase:    server.URL,
+		AuthMethod: "oauth",
+	})
+	tr.httpClient = server.Client()
+	tr.tokenSource = func() (string, error) {
+		return "oauth-token", nil
+	}
+
+	resp, err := tr.TranscribeData(context.Background(), []byte("audio"), "clip.mp3")
+	if err != nil {
+		t.Fatalf("TranscribeData() error: %v", err)
+	}
+	if resp.Text != "oauth transcription" {
+		t.Errorf("Text = %q, want %q", resp.Text, "oauth transcription")
+	}
+	if gotAuth != "Bearer oauth-token" {
+		t.Errorf("Authorization = %q, want %q", gotAuth, "Bearer oauth-token")
+	}
+	if gotModel != "gpt-4o-transcribe" {
+		t.Errorf("model field = %q, want %q", gotModel, "gpt-4o-transcribe")
+	}
+}

pkg/auth/openai.go

@@ -0,0 +1,32 @@
+package auth
+
+import "fmt"
+
+// GetOpenAIToken returns the current OpenAI credential, refreshing OAuth
+// credentials when they are close to expiry. The account ID is returned for
+// Codex backend calls that require the Chatgpt-Account-Id header.
+func GetOpenAIToken() (accessToken, accountID string, err error) {
+	cred, err := GetCredential("openai")
+	if err != nil {
+		return "", "", fmt.Errorf("loading auth credentials: %w", err)
+	}
+	if cred == nil {
+		return "", "", fmt.Errorf("no credentials for openai. Run: picoclaw auth login --provider openai")
+	}
+
+	if cred.AuthMethod == "oauth" && cred.NeedsRefresh() && cred.RefreshToken != "" {
+		refreshed, err := RefreshAccessToken(cred, OpenAIOAuthConfig())
+		if err != nil {
+			return "", "", fmt.Errorf("refreshing token: %w", err)
+		}
+		if refreshed.AccountID == "" {
+			refreshed.AccountID = cred.AccountID
+		}
+		if err := SetCredential("openai", refreshed); err != nil {
+			return "", "", fmt.Errorf("saving refreshed token: %w", err)
+		}
+		return refreshed.AccessToken, refreshed.AccountID, nil
+	}
+
+	return cred.AccessToken, cred.AccountID, nil
+}

pkg/providers/oauth/codex_provider.go

@@ -2,6 +2,7 @@ package oauthprovider
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"strings"
@@ -9,6 +10,7 @@ import (
 	"github.com/openai/openai-go/v3"
 	"github.com/openai/openai-go/v3/option"
 	"github.com/openai/openai-go/v3/responses"
+	"github.com/openai/openai-go/v3/shared"
 
 	"github.com/sipeed/picoclaw/pkg/auth"
 	"github.com/sipeed/picoclaw/pkg/logger"
@@ -104,8 +106,23 @@ func (p *CodexProvider) Chat(
 	defer stream.Close()
 
 	var resp *responses.Response
+	var streamText strings.Builder
+	var streamToolCalls []ToolCall
 	for stream.Next() {
 		evt := stream.Current()
+		if evt.Type == "response.output_text.done" {
+			textDone := evt.AsResponseOutputTextDone()
+			if textDone.Text != "" {
+				streamText.Reset()
+				streamText.WriteString(textDone.Text)
+			}
+		}
+		if evt.Type == "response.output_item.done" {
+			done := evt.AsResponseOutputItemDone()
+			if tc, ok := codexToolCallFromOutputItem(done.Item); ok {
+				streamToolCalls = append(streamToolCalls, tc)
+			}
+		}
 		if evt.Type == "response.completed" || evt.Type == "response.failed" || evt.Type == "response.incomplete" {
 			evtResp := evt.Response
 			if evtResp.ID != "" {
@@ -153,7 +170,46 @@ func (p *CodexProvider) Chat(
 		return nil, fmt.Errorf("codex API call: stream ended without completed response")
 	}
 
-	return orc.ParseResponseFromStruct(resp), nil
+	parsed := orc.ParseResponseFromStruct(resp)
+	if parsed.Content == "" && len(parsed.ToolCalls) == 0 && streamText.Len() > 0 {
+		parsed.Content = streamText.String()
+	}
+	if len(parsed.ToolCalls) == 0 && len(streamToolCalls) > 0 {
+		parsed.ToolCalls = streamToolCalls
+		parsed.FinishReason = "tool_calls"
+	}
+	return parsed, nil
+}
+
+func codexToolCallFromOutputItem(item responses.ResponseOutputItemUnion) (ToolCall, bool) {
+	if item.Type != "function_call" {
+		return ToolCall{}, false
+	}
+
+	call := item.AsFunctionCall()
+	if call.Name == "" {
+		return ToolCall{}, false
+	}
+
+	var args map[string]any
+	if err := json.Unmarshal([]byte(call.Arguments), &args); err != nil {
+		args = map[string]any{"raw": call.Arguments}
+	}
+
+	id := call.CallID
+	if id == "" {
+		id = call.ID
+	}
+
+	return ToolCall{
+		ID:        id,
+		Name:      call.Name,
+		Arguments: args,
+		Function: &FunctionCall{
+			Name:      call.Name,
+			Arguments: call.Arguments,
+		},
+	}, true
 }
 
 func (p *CodexProvider) GetDefaultModel() string {
@@ -164,6 +220,10 @@ func (p *CodexProvider) SupportsNativeSearch() bool {
 	return p.enableWebSearch
 }
 
+func (p *CodexProvider) SupportsThinking() bool {
+	return true
+}
+
 func resolveCodexModel(model string) (string, string) {
 	m := strings.ToLower(strings.TrimSpace(model))
 	if m == "" {
@@ -217,6 +277,9 @@ func buildCodexParams(
 			OfInputItemList: inputItems,
 		},
 		Store: openai.Opt(false),
+		Reasoning: shared.ReasoningParam{
+			Effort: codexReasoningEffort(options["thinking_level"]),
+		},
 	}
 
 	if instructions != "" {
@@ -240,31 +303,24 @@ func buildCodexParams(
 	return params
 }
 
-func CreateCodexTokenSource() func() (string, string, error) {
+func codexReasoningEffort(raw any) shared.ReasoningEffort {
-	return func() (string, string, error) {
+	level, _ := raw.(string)
-		cred, err := auth.GetCredential("openai")
+	switch strings.ToLower(strings.TrimSpace(level)) {
-		if err != nil {
+	case "low":
-			return "", "", fmt.Errorf("loading auth credentials: %w", err)
+		return shared.ReasoningEffortLow
-		}
+	case "medium", "adaptive":
-		if cred == nil {
+		return shared.ReasoningEffortMedium
-			return "", "", fmt.Errorf("no credentials for openai. Run: picoclaw auth login --provider openai")
+	case "high":
-		}
+		return shared.ReasoningEffortHigh
-
+	case "xhigh", "max":
-		if cred.AuthMethod == "oauth" && cred.NeedsRefresh() && cred.RefreshToken != "" {
+		return shared.ReasoningEffortXhigh
-			oauthCfg := auth.OpenAIOAuthConfig()
+	default:
-			refreshed, err := auth.RefreshAccessToken(cred, oauthCfg)
+		return shared.ReasoningEffortNone
-			if err != nil {
+	}
-				return "", "", fmt.Errorf("refreshing token: %w", err)
+}
-			}
+
-			if refreshed.AccountID == "" {
+func CreateCodexTokenSource() func() (string, string, error) {
-				refreshed.AccountID = cred.AccountID
+	return func() (string, string, error) {
-			}
+		return auth.GetOpenAIToken()
-			if err := auth.SetCredential("openai", refreshed); err != nil {
-				return "", "", fmt.Errorf("saving refreshed token: %w", err)
-			}
-			return refreshed.AccessToken, refreshed.AccountID, nil
-		}
-
-		return cred.AccessToken, cred.AccountID, nil
 	}
 }

pkg/providers/oauth/codex_provider_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/openai/openai-go/v3"
 	openaiopt "github.com/openai/openai-go/v3/option"
 	"github.com/openai/openai-go/v3/responses"
+	"github.com/openai/openai-go/v3/shared"
 
 	orc "github.com/sipeed/picoclaw/pkg/providers/openai_responses_common"
 )
@@ -34,6 +35,9 @@ func TestBuildCodexParams_BasicMessage(t *testing.T) {
 	if params.MaxOutputTokens.Valid() {
 		t.Fatalf("MaxOutputTokens should not be set for Codex backend")
 	}
+	if params.Reasoning.Effort != shared.ReasoningEffortNone {
+		t.Fatalf("Reasoning.Effort = %q, want none", params.Reasoning.Effort)
+	}
 }
 
 func TestBuildCodexParams_SystemAsInstructions(t *testing.T) {
@@ -50,6 +54,44 @@ func TestBuildCodexParams_SystemAsInstructions(t *testing.T) {
 	}
 }
 
+func TestBuildCodexParams_ThinkingLevel(t *testing.T) {
+	tests := []struct {
+		name  string
+		level any
+		want  shared.ReasoningEffort
+	}{
+		{name: "default", level: nil, want: shared.ReasoningEffortNone},
+		{name: "off", level: "off", want: shared.ReasoningEffortNone},
+		{name: "low", level: "low", want: shared.ReasoningEffortLow},
+		{name: "medium", level: "medium", want: shared.ReasoningEffortMedium},
+		{name: "adaptive", level: "adaptive", want: shared.ReasoningEffortMedium},
+		{name: "high", level: "high", want: shared.ReasoningEffortHigh},
+		{name: "xhigh", level: "xhigh", want: shared.ReasoningEffortXhigh},
+		{name: "max", level: "max", want: shared.ReasoningEffortXhigh},
+		{name: "unknown", level: "banana", want: shared.ReasoningEffortNone},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			opts := map[string]any{}
+			if tt.level != nil {
+				opts["thinking_level"] = tt.level
+			}
+			params := buildCodexParams([]Message{{Role: "user", Content: "Hi"}}, nil, "gpt-5.4", opts, false)
+			if params.Reasoning.Effort != tt.want {
+				t.Fatalf("Reasoning.Effort = %q, want %q", params.Reasoning.Effort, tt.want)
+			}
+		})
+	}
+}
+
+func TestCodexProvider_SupportsThinking(t *testing.T) {
+	provider := NewCodexProvider("test-token", "acc-123")
+	if !provider.SupportsThinking() {
+		t.Fatal("CodexProvider should support thinking_level")
+	}
+}
+
 func TestBuildCodexParams_ToolCallConversation(t *testing.T) {
 	messages := []Message{
 		{Role: "user", Content: "What's the weather?"},
@@ -374,6 +416,83 @@ func TestCodexProvider_ChatRoundTrip(t *testing.T) {
 	}
 }
 
+func TestCodexProvider_ChatRoundTrip_ToolCallFromStreamItem(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/responses" {
+			http.Error(w, "not found: "+r.URL.Path, http.StatusNotFound)
+			return
+		}
+
+		var reqBody map[string]any
+		if err := json.NewDecoder(r.Body).Decode(&reqBody); err != nil {
+			http.Error(w, "invalid json", http.StatusBadRequest)
+			return
+		}
+		if reqBody["stream"] != true {
+			http.Error(w, "stream must be true", http.StatusBadRequest)
+			return
+		}
+
+		w.Header().Set("Content-Type", "text/event-stream")
+		itemDone := map[string]any{
+			"type":            "response.output_item.done",
+			"sequence_number": 1,
+			"output_index":    0,
+			"item": map[string]any{
+				"id":        "fc_1",
+				"type":      "function_call",
+				"call_id":   "call_abc",
+				"name":      "nutritiondb__list_weight_entries",
+				"arguments": `{"limit":5}`,
+				"status":    "completed",
+			},
+		}
+		b, _ := json.Marshal(itemDone)
+		fmt.Fprintf(w, "event: response.output_item.done\n")
+		fmt.Fprintf(w, "data: %s\n\n", string(b))
+
+		resp := map[string]any{
+			"id":     "resp_test",
+			"object": "response",
+			"status": "completed",
+			"output": []map[string]any{},
+			"usage": map[string]any{
+				"input_tokens":          10,
+				"output_tokens":         5,
+				"total_tokens":          15,
+				"input_tokens_details":  map[string]any{"cached_tokens": 0},
+				"output_tokens_details": map[string]any{"reasoning_tokens": 0},
+			},
+		}
+		writeCompletedSSE(w, resp)
+	}))
+	defer server.Close()
+
+	provider := NewCodexProvider("test-token", "acc-123")
+	provider.client = createOpenAITestClient(server.URL, "test-token", "acc-123")
+
+	resp, err := provider.Chat(t.Context(), []Message{{Role: "user", Content: "latest weights"}}, nil, "gpt-5.4", nil)
+	if err != nil {
+		t.Fatalf("Chat() error: %v", err)
+	}
+	if len(resp.ToolCalls) != 1 {
+		t.Fatalf("len(ToolCalls) = %d, want 1", len(resp.ToolCalls))
+	}
+	tc := resp.ToolCalls[0]
+	if tc.ID != "call_abc" {
+		t.Errorf("ToolCall.ID = %q, want call_abc", tc.ID)
+	}
+	if tc.Name != "nutritiondb__list_weight_entries" {
+		t.Errorf("ToolCall.Name = %q, want nutritiondb__list_weight_entries", tc.Name)
+	}
+	if tc.Arguments["limit"] != float64(5) {
+		t.Errorf("ToolCall.Arguments[limit] = %v, want 5", tc.Arguments["limit"])
+	}
+	if resp.FinishReason != "tool_calls" {
+		t.Errorf("FinishReason = %q, want tool_calls", resp.FinishReason)
+	}
+}
+
 func TestCodexProvider_ChatRoundTrip_WebSearchDisabled(t *testing.T) {
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path != "/responses" {