Add STT capability support in LLM agent
- Update the `ToMap` function in `capabilities.go` to include the `stt` capability from OpenAI. - Modify the `convertAnthropicCaps` function to map the `STT` field from Anthropic capabilities. - Enhance documentation in `llm.go` to describe the new `stt` capability for audio transcription models.
This commit is contained in:
parent
342c2f11a3
commit
85159f49dd
3 changed files with 224 additions and 1 deletions
|
|
@ -125,6 +125,7 @@ func ToMap(caps *openai.Capabilities) map[string]interface{} {
|
|||
}
|
||||
|
||||
result["audio"] = caps.Audio
|
||||
result["stt"] = caps.STT
|
||||
result["tool_calls"] = caps.ToolCalls
|
||||
result["reasoning"] = caps.Reasoning
|
||||
result["streaming"] = caps.Streaming
|
||||
|
|
@ -144,6 +145,7 @@ func convertAnthropicCaps(caps *anthropic.Capabilities) *openai.Capabilities {
|
|||
return &openai.Capabilities{
|
||||
Vision: caps.Vision,
|
||||
Audio: caps.Audio,
|
||||
STT: caps.STT,
|
||||
ToolCalls: caps.ToolCalls,
|
||||
Reasoning: caps.Reasoning,
|
||||
Streaming: caps.Streaming,
|
||||
|
|
|
|||
220
agent/llm/process.go
Normal file
220
agent/llm/process.go
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/yaoapp/gou/connector"
|
||||
gouHTTP "github.com/yaoapp/gou/http"
|
||||
"github.com/yaoapp/gou/process"
|
||||
"github.com/yaoapp/gou/runtime/v8/bridge"
|
||||
agentContext "github.com/yaoapp/yao/agent/context"
|
||||
"github.com/yaoapp/yao/agent/output/message"
|
||||
"github.com/yaoapp/yao/openapi/oauth/authorized"
|
||||
)
|
||||
|
||||
func init() {
|
||||
process.Register("llm.ChatCompletions", ProcessChatCompletions)
|
||||
}
|
||||
|
||||
// ProcessChatCompletions implements the llm.ChatCompletions Process.
|
||||
// A universal replacement for openai.chat.Completions that auto-detects
|
||||
// connector type (openai, anthropic, etc.) and routes accordingly.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// Process("llm.ChatCompletions", connector, messages)
|
||||
// Process("llm.ChatCompletions", connector, messages, opts)
|
||||
// Process("llm.ChatCompletions", connector, messages, opts, callback)
|
||||
//
|
||||
// Args:
|
||||
// - connector (string): Connector ID, any type (openai / anthropic / ...)
|
||||
// - messages ([]map): Message array, supports multimodal content (image_url, etc.)
|
||||
// - opts (map): Optional. temperature, max_tokens, etc.
|
||||
// - callback (func): Optional. Streaming callback func(data []byte) int
|
||||
//
|
||||
// Returns: OpenAI-compatible format { choices: [{ message: { role, content } }], ... }
|
||||
func ProcessChatCompletions(p *process.Process) interface{} {
|
||||
p.ValidateArgNums(2)
|
||||
|
||||
// 1. Parse connector ID
|
||||
connectorID := p.ArgsString(0)
|
||||
if connectorID == "" {
|
||||
return newErrorResponse("llm.ChatCompletions: connector is required")
|
||||
}
|
||||
|
||||
// 2. Parse messages
|
||||
rawMessages := p.ArgsArray(1)
|
||||
messages := make([]map[string]interface{}, 0, len(rawMessages))
|
||||
for i, v := range rawMessages {
|
||||
msg, ok := v.(map[string]interface{})
|
||||
if !ok {
|
||||
return newErrorResponse(fmt.Sprintf("llm.ChatCompletions: message %d is not an object", i))
|
||||
}
|
||||
messages = append(messages, msg)
|
||||
}
|
||||
|
||||
// 3. Parse optional opts
|
||||
var opts map[string]interface{}
|
||||
if p.NumOfArgs() > 2 && p.Args[2] != nil {
|
||||
if o, ok := p.Args[2].(map[string]interface{}); ok {
|
||||
opts = o
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Parse optional callback (for streaming)
|
||||
var callback func(data []byte) int
|
||||
if p.NumOfArgs() > 3 && p.Args[3] != nil {
|
||||
switch cb := p.Args[3].(type) {
|
||||
case func(data []byte) int:
|
||||
callback = cb
|
||||
case bridge.FunctionT:
|
||||
callback = func(data []byte) int {
|
||||
v, err := cb.Call(string(data))
|
||||
if err != nil {
|
||||
return gouHTTP.HandlerReturnError
|
||||
}
|
||||
ret, ok := v.(int)
|
||||
if !ok {
|
||||
return gouHTTP.HandlerReturnError
|
||||
}
|
||||
return ret
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Select connector
|
||||
conn, err := connector.Select(connectorID)
|
||||
if err != nil {
|
||||
return newErrorResponse(fmt.Sprintf("llm.ChatCompletions: connector %s not found: %v", connectorID, err))
|
||||
}
|
||||
|
||||
// 6. Build completion options (reuse jsapi.go logic)
|
||||
completionOptions := buildCompletionOptions(conn, opts)
|
||||
|
||||
// 7. Create LLM instance (auto-selects openai/anthropic provider)
|
||||
llmInstance, err := New(conn, completionOptions)
|
||||
if err != nil {
|
||||
return newErrorResponse(fmt.Sprintf("llm.ChatCompletions: failed to create LLM: %v", err))
|
||||
}
|
||||
|
||||
// 8. Parse messages to context.Message format (reuse jsapi.go logic)
|
||||
interfaceMessages := make([]interface{}, len(messages))
|
||||
for i, m := range messages {
|
||||
interfaceMessages[i] = m
|
||||
}
|
||||
ctxMessages, err := parseMessages(interfaceMessages)
|
||||
if err != nil {
|
||||
return newErrorResponse(fmt.Sprintf("llm.ChatCompletions: invalid messages: %v", err))
|
||||
}
|
||||
|
||||
// 8.1 Normalize multimodal content: convert []interface{} maps to []ContentPart
|
||||
// so that providers (especially Anthropic) can type-assert correctly.
|
||||
for i := range ctxMessages {
|
||||
if parts, ok := ctxMessages[i].Content.([]interface{}); ok {
|
||||
ctxMessages[i].Content = normalizeContentParts(parts)
|
||||
}
|
||||
}
|
||||
|
||||
// 9. Build a minimal headless context for LLM call
|
||||
parent := p.Context
|
||||
if parent == nil {
|
||||
parent = context.Background()
|
||||
}
|
||||
authInfo := authorized.ProcessAuthInfo(p)
|
||||
chatID := agentContext.GenChatID()
|
||||
ctx := agentContext.New(parent, authInfo, chatID)
|
||||
defer ctx.Release()
|
||||
|
||||
// 10. Create stream handler
|
||||
var streamHandler message.StreamFunc
|
||||
if callback != nil {
|
||||
// With callback: forward raw chunks to caller
|
||||
streamHandler = func(chunkType message.StreamChunkType, data []byte) int {
|
||||
if chunkType == message.ChunkText || chunkType == message.ChunkThinking {
|
||||
return callback(data)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
} else {
|
||||
// No callback: no-op handler, just collect final response
|
||||
streamHandler = func(chunkType message.StreamChunkType, data []byte) int {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// 11. Execute LLM stream call
|
||||
response, err := llmInstance.Stream(ctx, ctxMessages, completionOptions, streamHandler)
|
||||
if err != nil {
|
||||
return newErrorResponse(fmt.Sprintf("llm.ChatCompletions: LLM call failed: %v", err))
|
||||
}
|
||||
|
||||
// 12. Convert CompletionResponse to OpenAI-compatible format
|
||||
// { choices: [{ message: { role, content } }], id, model, ... }
|
||||
return toOpenAIFormat(response)
|
||||
}
|
||||
|
||||
// toOpenAIFormat converts CompletionResponse to OpenAI chat.completions format
|
||||
// for backward compatibility with code that consumed openai.chat.Completions.
|
||||
func toOpenAIFormat(resp *agentContext.CompletionResponse) map[string]interface{} {
|
||||
if resp == nil {
|
||||
return map[string]interface{}{
|
||||
"choices": []interface{}{},
|
||||
}
|
||||
}
|
||||
|
||||
msgMap := map[string]interface{}{
|
||||
"role": resp.Role,
|
||||
"content": resp.Content,
|
||||
}
|
||||
if len(resp.ToolCalls) > 0 {
|
||||
msgMap["tool_calls"] = resp.ToolCalls
|
||||
}
|
||||
|
||||
choice := map[string]interface{}{
|
||||
"index": 0,
|
||||
"message": msgMap,
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
|
||||
result := map[string]interface{}{
|
||||
"id": resp.ID,
|
||||
"object": "chat.completion",
|
||||
"created": resp.Created,
|
||||
"model": resp.Model,
|
||||
"choices": []interface{}{choice},
|
||||
}
|
||||
|
||||
if resp.Usage != nil {
|
||||
result["usage"] = resp.Usage
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// newErrorResponse creates an error response in OpenAI-compatible format
|
||||
func newErrorResponse(errMsg string) map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"error": map[string]interface{}{
|
||||
"message": errMsg,
|
||||
"type": "invalid_request_error",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeContentParts converts []interface{} (raw maps from Process args)
|
||||
// to []agentContext.ContentPart (strongly typed) via JSON round-trip.
|
||||
// This is essential for providers (e.g. Anthropic) that type-assert on
|
||||
// []ContentPart to apply format-specific conversions (image_url → image).
|
||||
func normalizeContentParts(parts []interface{}) []agentContext.ContentPart {
|
||||
raw, err := json.Marshal(parts)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var typed []agentContext.ContentPart
|
||||
if err := json.Unmarshal(raw, &typed); err != nil {
|
||||
return nil
|
||||
}
|
||||
return typed
|
||||
}
|
||||
|
|
@ -126,7 +126,8 @@ func getCapabilitiesWithModels(conn connector.Connector, modelCapabilities map[s
|
|||
// matchesFilters checks if capabilities match all requested filters
|
||||
// Filters are matched case-insensitively and support the following capability keys:
|
||||
// - vision: true or string value like "openai", "claude"
|
||||
// - audio: bool
|
||||
// - audio: bool (LLM supports audio input/understanding)
|
||||
// - stt: bool (Speech-to-Text / audio transcription model, e.g. Whisper)
|
||||
// - tool_calls: bool
|
||||
// - reasoning: bool
|
||||
// - streaming: bool
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue