diff --git a/agent/assistant/agent.go b/agent/assistant/agent.go index c3d9fa42..9cebdf5f 100644 --- a/agent/assistant/agent.go +++ b/agent/assistant/agent.go @@ -11,6 +11,7 @@ import ( "github.com/yaoapp/yao/agent/assistant/handlers" "github.com/yaoapp/yao/agent/context" "github.com/yaoapp/yao/agent/i18n" + "github.com/yaoapp/yao/agent/llm" "github.com/yaoapp/yao/agent/output/message" ) @@ -323,58 +324,12 @@ func (ast *Assistant) GetConnector(ctx *context.Context) (connector.Connector, * } // Get connector capabilities from settings - capabilities := ast.getConnectorCapabilities(conn) + // Uses unified capability getter: 1. User-defined models.yml, 2. connector's Setting()["capabilities"], 3. default + capabilities := llm.GetCapabilitiesFromConn(conn, modelCapabilities) return conn, capabilities, nil } -// getConnectorCapabilities get the capabilities of a connector from settings -// Priority: 1. modelCapabilities mapping, 2. connector's Setting()["capabilities"] -func (ast *Assistant) getConnectorCapabilities(conn connector.Connector) *openai.Capabilities { - if conn == nil { - return &openai.Capabilities{ - Vision: false, - ToolCalls: false, - Audio: false, - Reasoning: false, - Streaming: false, - JSON: false, - Multimodal: false, - TemperatureAdjustable: true, - } - } - - // Get connector ID - connectorID := conn.ID() - - // Priority 1: Check global modelCapabilities mapping - if modelCaps, exists := modelCapabilities[connectorID]; exists { - return &modelCaps - } - - // Priority 2: Get capabilities from connector's Setting() method - // Modern connectors (post-upgrade) provide default capabilities via Setting() - settings := conn.Setting() - if caps, ok := settings["capabilities"]; ok { - if capabilities, ok := caps.(*openai.Capabilities); ok { - return capabilities - } - } - - // Fallback: Return minimal default capabilities - // This should rarely happen with upgraded connectors - return &openai.Capabilities{ - Vision: false, - ToolCalls: false, - Audio: false, - Reasoning: false, - Streaming: false, - JSON: false, - Multimodal: false, - TemperatureAdjustable: true, // Default to true for non-reasoning models - } -} - // Info get the assistant information func (ast *Assistant) Info(locale ...string) *message.AssistantInfo { lc := "en" diff --git a/agent/llm/capabilities.go b/agent/llm/capabilities.go new file mode 100644 index 00000000..63072805 --- /dev/null +++ b/agent/llm/capabilities.go @@ -0,0 +1,127 @@ +package llm + +import ( + "github.com/yaoapp/gou/connector" + "github.com/yaoapp/gou/connector/openai" +) + +// GetCapabilities get the capabilities of a connector by connector ID +// This is a unified function to get connector capabilities with proper priority: +// 1. User-defined model capabilities from agent/models.yml (passed via modelCapabilities map) +// 2. Connector's Setting()["capabilities"] (default capabilities from connector) +// 3. Fallback to minimal default capabilities +// +// Usage in Agent with user-defined models: +// +// capabilities := llm.GetCapabilities(connectorID, modelCapabilities) +// +// Usage in API (without user-defined models): +// +// capabilities := llm.GetCapabilities(connectorID, nil) +func GetCapabilities(connectorID string, modelCapabilities map[string]openai.Capabilities) *openai.Capabilities { + if connectorID == "" { + return getDefaultCapabilities() + } + + // Priority 1: Check user-defined model capabilities from agent/models.yml + if modelCapabilities != nil { + if modelCaps, exists := modelCapabilities[connectorID]; exists { + return &modelCaps + } + } + + // Priority 2: Get connector and extract capabilities from Setting() + conn, err := connector.Select(connectorID) + if err != nil { + // If connector not found, return default + return getDefaultCapabilities() + } + + return GetCapabilitiesFromConn(conn, modelCapabilities) +} + +// GetCapabilitiesFromConn get the capabilities from a connector instance +// This is useful when you already have the connector object +func GetCapabilitiesFromConn(conn connector.Connector, modelCapabilities map[string]openai.Capabilities) *openai.Capabilities { + if conn == nil { + return getDefaultCapabilities() + } + + connectorID := conn.ID() + + // Priority 1: Check user-defined model capabilities from agent/models.yml + if modelCapabilities != nil { + if modelCaps, exists := modelCapabilities[connectorID]; exists { + return &modelCaps + } + } + + // Priority 2: Get capabilities from connector's Setting() method + settings := conn.Setting() + if settings != nil { + if caps, ok := settings["capabilities"]; ok { + // Try to convert to *openai.Capabilities + if capabilities, ok := caps.(*openai.Capabilities); ok { + return capabilities + } + // Try to convert to openai.Capabilities (value type) + if capabilities, ok := caps.(openai.Capabilities); ok { + return &capabilities + } + } + } + + // Priority 3: Fallback to minimal default capabilities + return getDefaultCapabilities() +} + +// getDefaultCapabilities returns minimal default capabilities +// This should rarely be used as modern connectors provide capabilities via Setting() +func getDefaultCapabilities() *openai.Capabilities { + return &openai.Capabilities{ + Vision: false, + ToolCalls: false, + Audio: false, + Reasoning: false, + Streaming: false, + JSON: false, + Multimodal: false, + TemperatureAdjustable: true, // Default to true for non-reasoning models + } +} + +// GetCapabilitiesMap get capabilities as map[string]interface{} for API responses +// This is useful for OpenAPI responses that need JSON-serializable format +func GetCapabilitiesMap(connectorID string, modelCapabilities map[string]openai.Capabilities) map[string]interface{} { + caps := GetCapabilities(connectorID, modelCapabilities) + if caps == nil { + return nil + } + + return ToMap(caps) +} + +// ToMap converts openai.Capabilities to map[string]interface{} +// This is useful for JSON serialization in API responses +func ToMap(caps *openai.Capabilities) map[string]interface{} { + if caps == nil { + return nil + } + + result := make(map[string]interface{}) + + // Handle Vision field specially as it can be bool or string + if caps.Vision != nil { + result["vision"] = caps.Vision + } + + result["audio"] = caps.Audio + result["tool_calls"] = caps.ToolCalls + result["reasoning"] = caps.Reasoning + result["streaming"] = caps.Streaming + result["json"] = caps.JSON + result["multimodal"] = caps.Multimodal + result["temperature_adjustable"] = caps.TemperatureAdjustable + + return result +} diff --git a/openapi/agent/agent.go b/openapi/agent/agent.go index 307f0745..cd6bf4b6 100644 --- a/openapi/agent/agent.go +++ b/openapi/agent/agent.go @@ -16,11 +16,12 @@ func Attach(group *gin.RouterGroup, oauth types.OAuth) { group.Use(oauth.Guard) // Assistant CRUD - Standard REST endpoints - group.GET("/assistants", ListAssistants) // GET /assistants - List assistants - group.POST("/assistants", CreateAssistant) // POST /assistants - Create assistant - group.GET("/assistants/tags", ListAssistantTags) // GET /assistants/tags - Get all assistant tags with permission filtering - group.GET("/assistants/:id", GetAssistant) // GET /assistants/:id - Get assistant details with permission verification - group.PUT("/assistants/:id", UpdateAssistant) // PUT /assistants/:id - Update assistant + group.GET("/assistants", ListAssistants) // GET /assistants - List assistants + group.POST("/assistants", CreateAssistant) // POST /assistants - Create assistant + group.GET("/assistants/tags", ListAssistantTags) // GET /assistants/tags - Get all assistant tags with permission filtering + group.GET("/assistants/:id", GetAssistant) // GET /assistants/:id - Get assistant details with permission verification + group.GET("/assistants/:id/info", GetAssistantInfo) // GET /assistants/:id/messages - Get assistant Information + group.PUT("/assistants/:id", UpdateAssistant) // PUT /assistants/:id - Update assistant // group.DELETE("/assistants/:id", agent.HandleAssistantDelete) // DELETE /assistants/:id - Delete assistant // Assistant Actions diff --git a/openapi/agent/assistant.go b/openapi/agent/assistant.go index ce2d10ea..d0c13487 100644 --- a/openapi/agent/assistant.go +++ b/openapi/agent/assistant.go @@ -512,6 +512,110 @@ func UpdateAssistant(c *gin.Context) { }) } +// GetAssistantInfo retrieves essential assistant information for InputArea component +// Returns only the fields needed for UI display: id, name, avatar, description, connector, connector_options, modes, default_mode +func GetAssistantInfo(c *gin.Context) { + + // Get authorized information + authInfo := authorized.GetInfo(c) + + // Get Agent instance from global variable + agentInstance := agent.GetAgent() + if agentInstance == nil || agentInstance.Store == nil { + errorResp := &response.ErrorResponse{ + Code: response.ErrServerError.Code, + ErrorDescription: "Agent store not initialized", + } + response.RespondWithError(c, response.StatusInternalServerError, errorResp) + return + } + + // Get assistant ID from URL parameter + assistantID := c.Param("id") + if assistantID == "" { + errorResp := &response.ErrorResponse{ + Code: response.ErrInvalidRequest.Code, + ErrorDescription: "assistant_id is required", + } + response.RespondWithError(c, response.StatusBadRequest, errorResp) + return + } + + // Parse locale (optional - defaults to "en-us") + locale := "en-us" + if loc := c.Query("locale"); loc != "" { + locale = strings.ToLower(strings.TrimSpace(loc)) + } + + // Define fields needed for InputArea + infoFields := []string{ + "assistant_id", + "name", + "avatar", + "description", + "connector", + "connector_options", + "modes", + "default_mode", + } + + // Get assistant with specific fields and locale + assistant, err := agentInstance.Store.GetAssistant(assistantID, infoFields, locale) + if err != nil { + log.Error("Failed to get assistant info %s: %v", assistantID, err) + errorResp := &response.ErrorResponse{ + Code: response.ErrInvalidRequest.Code, + ErrorDescription: "Assistant not found: " + err.Error(), + } + response.RespondWithError(c, response.StatusNotFound, errorResp) + return + } + + // Check read permission (same as GetAssistant) + hasPermission, err := checkAssistantPermission(authInfo, assistantID, true) + if err != nil { + log.Error("Failed to check permission for assistant %s: %v", assistantID, err) + errorResp := &response.ErrorResponse{ + Code: response.ErrServerError.Code, + ErrorDescription: "Failed to check permission: " + err.Error(), + } + response.RespondWithError(c, response.StatusForbidden, errorResp) + return + } + + if !hasPermission { + errorResp := &response.ErrorResponse{ + Code: response.ErrAccessDenied.Code, + ErrorDescription: "Forbidden: No permission to access this assistant", + } + response.RespondWithError(c, response.StatusForbidden, errorResp) + return + } + + // Build response with only the required fields + infoResponse := map[string]interface{}{ + "assistant_id": assistant.ID, + "name": assistant.Name, + "avatar": assistant.Avatar, + "description": assistant.Description, + "connector": assistant.Connector, + } + + // Add optional fields if they exist + if assistant.ConnectorOptions != nil { + infoResponse["connector_options"] = assistant.ConnectorOptions + } + if len(assistant.Modes) > 0 { + infoResponse["modes"] = assistant.Modes + } + if assistant.DefaultMode != "" { + infoResponse["default_mode"] = assistant.DefaultMode + } + + // Return the result with standard response format + response.RespondWithSuccess(c, response.StatusOK, infoResponse) +} + // checkAssistantPermission checks if the user has permission to access the assistant // Similar logic to checkCollectionPermission in openapi/kb/collection.go // readable: true for read permission, false for write permission diff --git a/openapi/llm/llm.go b/openapi/llm/llm.go index 14b55eff..30a8374a 100644 --- a/openapi/llm/llm.go +++ b/openapi/llm/llm.go @@ -1,18 +1,34 @@ package llm import ( + "strings" + "github.com/gin-gonic/gin" "github.com/yaoapp/gou/connector" + "github.com/yaoapp/gou/connector/openai" + "github.com/yaoapp/yao/agent" + agentllm "github.com/yaoapp/yao/agent/llm" oauthTypes "github.com/yaoapp/yao/openapi/oauth/types" "github.com/yaoapp/yao/openapi/response" ) // Provider represents an LLM provider option type Provider struct { - Label string `json:"label"` - Value string `json:"value"` - Type string `json:"type"` // "openai" - Builtin bool `json:"builtin"` // true for system built-in, false for user-defined + Label string `json:"label"` + Value string `json:"value"` + Type string `json:"type"` // "openai" + Builtin bool `json:"builtin"` // true for system built-in, false for user-defined + Capabilities map[string]interface{} `json:"capabilities"` // Model capabilities from connector settings +} + +// getModelCapabilities returns user-defined model capabilities from agent DSL +// Returns nil if agent not initialized or no models configured +func getModelCapabilities() map[string]openai.Capabilities { + agentDSL := agent.GetAgent() + if agentDSL != nil && agentDSL.Models != nil { + return agentDSL.Models + } + return nil } // Attach attaches the LLM management handlers to the router with OAuth protection @@ -26,52 +42,50 @@ func Attach(group *gin.RouterGroup, oauth oauthTypes.OAuth) { } // listProviders lists all available LLM providers (built-in + user-defined) +// Supports filtering by capabilities using query parameter: ?filters=vision,tool_calls,audio func listProviders(c *gin.Context) { allProviders := make([]Provider, 0) - // Track which connectors we've already added (to avoid duplicates) - added := make(map[string]bool) + // Parse filter parameters from query string + filtersParam := c.Query("filters") + var filters []string + if filtersParam != "" { + filters = strings.Split(filtersParam, ",") + for i, filter := range filters { + filters[i] = strings.TrimSpace(strings.ToLower(filter)) + } + } - // 1. Get system built-in OpenAI-compatible LLM connectors + // Get user-defined model capabilities once at the start of request + modelCapabilities := getModelCapabilities() + + // Get all OpenAI-compatible LLM connectors from AIConnectors + // Note: All openai type connectors are automatically added to AIConnectors during loading + // See gou/connector/connector.go LoadSource() for details for _, opt := range connector.AIConnectors { connType := getConnectorType(opt.Value) // Only include OpenAI-compatible LLM connectors if connType == "openai" { - allProviders = append(allProviders, Provider{ - Label: opt.Label, - Value: opt.Value, - Type: connType, - Builtin: true, - }) - added[opt.Value] = true - } - } + conn, ok := connector.Connectors[opt.Value] + if !ok { + continue + } - // 2. Get user-defined OpenAI-compatible LLM connectors from the global connector registry - // This includes all loaded connectors, both built-in and user-defined - // Only include OpenAI-compatible connectors (standard openai format) - for id, conn := range connector.Connectors { - // Skip if already added - if added[id] { - continue - } + // Get capabilities from connector settings + capabilities := getCapabilitiesWithModels(conn, modelCapabilities) - // Only include OpenAI-compatible LLM connectors - connType := getConnectorType(id) - if connType == "openai" { - meta := conn.GetMetaInfo() - label := meta.Label - if label == "" { - label = id + // Apply capability filters + if len(filters) > 0 && !matchesFilters(capabilities, filters) { + continue } allProviders = append(allProviders, Provider{ - Label: label, - Value: id, - Type: connType, - Builtin: meta.Builtin, + Label: opt.Label, + Value: opt.Value, + Type: connType, + Builtin: conn.GetMetaInfo().Builtin, + Capabilities: capabilities, }) - added[id] = true } } @@ -92,3 +106,70 @@ func getConnectorType(id string) string { return "unknown" } + +// getCapabilitiesWithModels extracts capabilities from connector settings +// Uses the unified capability getter from agent/llm package +// Takes modelCapabilities as parameter to avoid repeated calls to getModelCapabilities() +func getCapabilitiesWithModels(conn connector.Connector, modelCapabilities map[string]openai.Capabilities) map[string]interface{} { + if conn == nil { + return nil + } + + // Use unified capability getter with user-defined model capabilities + caps := agentllm.GetCapabilitiesFromConn(conn, modelCapabilities) + return agentllm.ToMap(caps) +} + +// matchesFilters checks if capabilities match all requested filters +// Filters are matched case-insensitively and support the following capability keys: +// - vision: true or string value like "openai", "claude" +// - audio: bool +// - tool_calls: bool +// - reasoning: bool +// - streaming: bool +// - json: bool +// - multimodal: bool +// - temperature_adjustable: bool +func matchesFilters(capabilities map[string]interface{}, filters []string) bool { + if capabilities == nil { + return false + } + + // All filters must match (AND logic) + for _, filter := range filters { + matched := false + + // Check each capability field + for key, value := range capabilities { + keyLower := strings.ToLower(key) + + // Match the filter against capability key + if keyLower == filter { + // For vision, check if it's true or a non-empty string + if filter == "vision" { + if boolVal, ok := value.(bool); ok && boolVal { + matched = true + break + } + if strVal, ok := value.(string); ok && strVal != "" { + matched = true + break + } + } else { + // For other capabilities, check if it's true + if boolVal, ok := value.(bool); ok && boolVal { + matched = true + break + } + } + } + } + + // If any filter doesn't match, return false + if !matched { + return false + } + } + + return true +}