Refactor assistant capabilities retrieval and enhance API endpoints

- Replaced the deprecated getConnectorCapabilities method with a unified capability getter in the Assistant model, improving capability retrieval logic.
- Added a new API endpoint to retrieve essential assistant information, including fields like id, name, avatar, and connector options, enhancing the assistant's data accessibility.
- Updated the LLM management to support filtering by capabilities, allowing for more flexible provider listings based on user-defined models.
- Improved overall structure and clarity in the assistant's capabilities and API responses, ensuring better maintainability and usability.

agent/assistant/agent.go

@@ -11,6 +11,7 @@ import (
 	"github.com/yaoapp/yao/agent/assistant/handlers"
 	"github.com/yaoapp/yao/agent/context"
 	"github.com/yaoapp/yao/agent/i18n"
+	"github.com/yaoapp/yao/agent/llm"
 	"github.com/yaoapp/yao/agent/output/message"
 )
 
@@ -323,58 +324,12 @@ func (ast *Assistant) GetConnector(ctx *context.Context) (connector.Connector, *
 	}
 
 	// Get connector capabilities from settings
-	capabilities := ast.getConnectorCapabilities(conn)
+	// Uses unified capability getter: 1. User-defined models.yml, 2. connector's Setting()["capabilities"], 3. default
+	capabilities := llm.GetCapabilitiesFromConn(conn, modelCapabilities)
 
 	return conn, capabilities, nil
 }
 
-// getConnectorCapabilities get the capabilities of a connector from settings
-// Priority: 1. modelCapabilities mapping, 2. connector's Setting()["capabilities"]
-func (ast *Assistant) getConnectorCapabilities(conn connector.Connector) *openai.Capabilities {
-	if conn == nil {
-		return &openai.Capabilities{
-			Vision:                false,
-			ToolCalls:             false,
-			Audio:                 false,
-			Reasoning:             false,
-			Streaming:             false,
-			JSON:                  false,
-			Multimodal:            false,
-			TemperatureAdjustable: true,
-		}
-	}
-
-	// Get connector ID
-	connectorID := conn.ID()
-
-	// Priority 1: Check global modelCapabilities mapping
-	if modelCaps, exists := modelCapabilities[connectorID]; exists {
-		return &modelCaps
-	}
-
-	// Priority 2: Get capabilities from connector's Setting() method
-	// Modern connectors (post-upgrade) provide default capabilities via Setting()
-	settings := conn.Setting()
-	if caps, ok := settings["capabilities"]; ok {
-		if capabilities, ok := caps.(*openai.Capabilities); ok {
-			return capabilities
-		}
-	}
-
-	// Fallback: Return minimal default capabilities
-	// This should rarely happen with upgraded connectors
-	return &openai.Capabilities{
-		Vision:                false,
-		ToolCalls:             false,
-		Audio:                 false,
-		Reasoning:             false,
-		Streaming:             false,
-		JSON:                  false,
-		Multimodal:            false,
-		TemperatureAdjustable: true, // Default to true for non-reasoning models
-	}
-}
-
 // Info get the assistant information
 func (ast *Assistant) Info(locale ...string) *message.AssistantInfo {
 	lc := "en"

agent/llm/capabilities.go

@@ -0,0 +1,127 @@
+package llm
+
+import (
+	"github.com/yaoapp/gou/connector"
+	"github.com/yaoapp/gou/connector/openai"
+)
+
+// GetCapabilities get the capabilities of a connector by connector ID
+// This is a unified function to get connector capabilities with proper priority:
+// 1. User-defined model capabilities from agent/models.yml (passed via modelCapabilities map)
+// 2. Connector's Setting()["capabilities"] (default capabilities from connector)
+// 3. Fallback to minimal default capabilities
+//
+// Usage in Agent with user-defined models:
+//
+//	capabilities := llm.GetCapabilities(connectorID, modelCapabilities)
+//
+// Usage in API (without user-defined models):
+//
+//	capabilities := llm.GetCapabilities(connectorID, nil)
+func GetCapabilities(connectorID string, modelCapabilities map[string]openai.Capabilities) *openai.Capabilities {
+	if connectorID == "" {
+		return getDefaultCapabilities()
+	}
+
+	// Priority 1: Check user-defined model capabilities from agent/models.yml
+	if modelCapabilities != nil {
+		if modelCaps, exists := modelCapabilities[connectorID]; exists {
+			return &modelCaps
+		}
+	}
+
+	// Priority 2: Get connector and extract capabilities from Setting()
+	conn, err := connector.Select(connectorID)
+	if err != nil {
+		// If connector not found, return default
+		return getDefaultCapabilities()
+	}
+
+	return GetCapabilitiesFromConn(conn, modelCapabilities)
+}
+
+// GetCapabilitiesFromConn get the capabilities from a connector instance
+// This is useful when you already have the connector object
+func GetCapabilitiesFromConn(conn connector.Connector, modelCapabilities map[string]openai.Capabilities) *openai.Capabilities {
+	if conn == nil {
+		return getDefaultCapabilities()
+	}
+
+	connectorID := conn.ID()
+
+	// Priority 1: Check user-defined model capabilities from agent/models.yml
+	if modelCapabilities != nil {
+		if modelCaps, exists := modelCapabilities[connectorID]; exists {
+			return &modelCaps
+		}
+	}
+
+	// Priority 2: Get capabilities from connector's Setting() method
+	settings := conn.Setting()
+	if settings != nil {
+		if caps, ok := settings["capabilities"]; ok {
+			// Try to convert to *openai.Capabilities
+			if capabilities, ok := caps.(*openai.Capabilities); ok {
+				return capabilities
+			}
+			// Try to convert to openai.Capabilities (value type)
+			if capabilities, ok := caps.(openai.Capabilities); ok {
+				return &capabilities
+			}
+		}
+	}
+
+	// Priority 3: Fallback to minimal default capabilities
+	return getDefaultCapabilities()
+}
+
+// getDefaultCapabilities returns minimal default capabilities
+// This should rarely be used as modern connectors provide capabilities via Setting()
+func getDefaultCapabilities() *openai.Capabilities {
+	return &openai.Capabilities{
+		Vision:                false,
+		ToolCalls:             false,
+		Audio:                 false,
+		Reasoning:             false,
+		Streaming:             false,
+		JSON:                  false,
+		Multimodal:            false,
+		TemperatureAdjustable: true, // Default to true for non-reasoning models
+	}
+}
+
+// GetCapabilitiesMap get capabilities as map[string]interface{} for API responses
+// This is useful for OpenAPI responses that need JSON-serializable format
+func GetCapabilitiesMap(connectorID string, modelCapabilities map[string]openai.Capabilities) map[string]interface{} {
+	caps := GetCapabilities(connectorID, modelCapabilities)
+	if caps == nil {
+		return nil
+	}
+
+	return ToMap(caps)
+}
+
+// ToMap converts openai.Capabilities to map[string]interface{}
+// This is useful for JSON serialization in API responses
+func ToMap(caps *openai.Capabilities) map[string]interface{} {
+	if caps == nil {
+		return nil
+	}
+
+	result := make(map[string]interface{})
+
+	// Handle Vision field specially as it can be bool or string
+	if caps.Vision != nil {
+		result["vision"] = caps.Vision
+	}
+
+	result["audio"] = caps.Audio
+	result["tool_calls"] = caps.ToolCalls
+	result["reasoning"] = caps.Reasoning
+	result["streaming"] = caps.Streaming
+	result["json"] = caps.JSON
+	result["multimodal"] = caps.Multimodal
+	result["temperature_adjustable"] = caps.TemperatureAdjustable
+
+	return result
+}

openapi/agent/agent.go

@@ -16,11 +16,12 @@ func Attach(group *gin.RouterGroup, oauth types.OAuth) {
 	group.Use(oauth.Guard)
 
 	// Assistant CRUD - Standard REST endpoints
-	group.GET("/assistants", ListAssistants)         // GET /assistants - List assistants
-	group.POST("/assistants", CreateAssistant)       // POST /assistants - Create assistant
-	group.GET("/assistants/tags", ListAssistantTags) // GET /assistants/tags - Get all assistant tags with permission filtering
-	group.GET("/assistants/:id", GetAssistant)       // GET /assistants/:id - Get assistant details with permission verification
-	group.PUT("/assistants/:id", UpdateAssistant)    // PUT /assistants/:id - Update assistant
+	group.GET("/assistants", ListAssistants)            // GET /assistants - List assistants
+	group.POST("/assistants", CreateAssistant)          // POST /assistants - Create assistant
+	group.GET("/assistants/tags", ListAssistantTags)    // GET /assistants/tags - Get all assistant tags with permission filtering
+	group.GET("/assistants/:id", GetAssistant)          // GET /assistants/:id - Get assistant details with permission verification
+	group.GET("/assistants/:id/info", GetAssistantInfo) // GET /assistants/:id/messages - Get assistant Information
+	group.PUT("/assistants/:id", UpdateAssistant)       // PUT /assistants/:id - Update assistant
 	// group.DELETE("/assistants/:id", agent.HandleAssistantDelete) // DELETE /assistants/:id - Delete assistant
 
 	// Assistant Actions

openapi/agent/assistant.go

@@ -512,6 +512,110 @@ func UpdateAssistant(c *gin.Context) {
 	})
 }
 
+// GetAssistantInfo retrieves essential assistant information for InputArea component
+// Returns only the fields needed for UI display: id, name, avatar, description, connector, connector_options, modes, default_mode
+func GetAssistantInfo(c *gin.Context) {
+
+	// Get authorized information
+	authInfo := authorized.GetInfo(c)
+
+	// Get Agent instance from global variable
+	agentInstance := agent.GetAgent()
+	if agentInstance == nil || agentInstance.Store == nil {
+		errorResp := &response.ErrorResponse{
+			Code:             response.ErrServerError.Code,
+			ErrorDescription: "Agent store not initialized",
+		}
+		response.RespondWithError(c, response.StatusInternalServerError, errorResp)
+		return
+	}
+
+	// Get assistant ID from URL parameter
+	assistantID := c.Param("id")
+	if assistantID == "" {
+		errorResp := &response.ErrorResponse{
+			Code:             response.ErrInvalidRequest.Code,
+			ErrorDescription: "assistant_id is required",
+		}
+		response.RespondWithError(c, response.StatusBadRequest, errorResp)
+		return
+	}
+
+	// Parse locale (optional - defaults to "en-us")
+	locale := "en-us"
+	if loc := c.Query("locale"); loc != "" {
+		locale = strings.ToLower(strings.TrimSpace(loc))
+	}
+
+	// Define fields needed for InputArea
+	infoFields := []string{
+		"assistant_id",
+		"name",
+		"avatar",
+		"description",
+		"connector",
+		"connector_options",
+		"modes",
+		"default_mode",
+	}
+
+	// Get assistant with specific fields and locale
+	assistant, err := agentInstance.Store.GetAssistant(assistantID, infoFields, locale)
+	if err != nil {
+		log.Error("Failed to get assistant info %s: %v", assistantID, err)
+		errorResp := &response.ErrorResponse{
+			Code:             response.ErrInvalidRequest.Code,
+			ErrorDescription: "Assistant not found: " + err.Error(),
+		}
+		response.RespondWithError(c, response.StatusNotFound, errorResp)
+		return
+	}
+
+	// Check read permission (same as GetAssistant)
+	hasPermission, err := checkAssistantPermission(authInfo, assistantID, true)
+	if err != nil {
+		log.Error("Failed to check permission for assistant %s: %v", assistantID, err)
+		errorResp := &response.ErrorResponse{
+			Code:             response.ErrServerError.Code,
+			ErrorDescription: "Failed to check permission: " + err.Error(),
+		}
+		response.RespondWithError(c, response.StatusForbidden, errorResp)
+		return
+	}
+
+	if !hasPermission {
+		errorResp := &response.ErrorResponse{
+			Code:             response.ErrAccessDenied.Code,
+			ErrorDescription: "Forbidden: No permission to access this assistant",
+		}
+		response.RespondWithError(c, response.StatusForbidden, errorResp)
+		return
+	}
+
+	// Build response with only the required fields
+	infoResponse := map[string]interface{}{
+		"assistant_id": assistant.ID,
+		"name":         assistant.Name,
+		"avatar":       assistant.Avatar,
+		"description":  assistant.Description,
+		"connector":    assistant.Connector,
+	}
+
+	// Add optional fields if they exist
+	if assistant.ConnectorOptions != nil {
+		infoResponse["connector_options"] = assistant.ConnectorOptions
+	}
+	if len(assistant.Modes) > 0 {
+		infoResponse["modes"] = assistant.Modes
+	}
+	if assistant.DefaultMode != "" {
+		infoResponse["default_mode"] = assistant.DefaultMode
+	}
+
+	// Return the result with standard response format
+	response.RespondWithSuccess(c, response.StatusOK, infoResponse)
+}
+
 // checkAssistantPermission checks if the user has permission to access the assistant
 // Similar logic to checkCollectionPermission in openapi/kb/collection.go
 // readable: true for read permission, false for write permission

openapi/llm/llm.go

@@ -1,18 +1,34 @@
 package llm
 
 import (
+	"strings"
+
 	"github.com/gin-gonic/gin"
 	"github.com/yaoapp/gou/connector"
+	"github.com/yaoapp/gou/connector/openai"
+	"github.com/yaoapp/yao/agent"
+	agentllm "github.com/yaoapp/yao/agent/llm"
 	oauthTypes "github.com/yaoapp/yao/openapi/oauth/types"
 	"github.com/yaoapp/yao/openapi/response"
 )
 
 // Provider represents an LLM provider option
 type Provider struct {
-	Label   string `json:"label"`
-	Value   string `json:"value"`
-	Type    string `json:"type"`    // "openai"
-	Builtin bool   `json:"builtin"` // true for system built-in, false for user-defined
+	Label        string                 `json:"label"`
+	Value        string                 `json:"value"`
+	Type         string                 `json:"type"`         // "openai"
+	Builtin      bool                   `json:"builtin"`      // true for system built-in, false for user-defined
+	Capabilities map[string]interface{} `json:"capabilities"` // Model capabilities from connector settings
+}
+
+// getModelCapabilities returns user-defined model capabilities from agent DSL
+// Returns nil if agent not initialized or no models configured
+func getModelCapabilities() map[string]openai.Capabilities {
+	agentDSL := agent.GetAgent()
+	if agentDSL != nil && agentDSL.Models != nil {
+		return agentDSL.Models
+	}
+	return nil
 }
 
 // Attach attaches the LLM management handlers to the router with OAuth protection
@@ -26,52 +42,50 @@ func Attach(group *gin.RouterGroup, oauth oauthTypes.OAuth) {
 }
 
 // listProviders lists all available LLM providers (built-in + user-defined)
+// Supports filtering by capabilities using query parameter: ?filters=vision,tool_calls,audio
 func listProviders(c *gin.Context) {
 	allProviders := make([]Provider, 0)
 
-	// Track which connectors we've already added (to avoid duplicates)
-	added := make(map[string]bool)
+	// Parse filter parameters from query string
+	filtersParam := c.Query("filters")
+	var filters []string
+	if filtersParam != "" {
+		filters = strings.Split(filtersParam, ",")
+		for i, filter := range filters {
+			filters[i] = strings.TrimSpace(strings.ToLower(filter))
+		}
+	}
 
-	// 1. Get system built-in OpenAI-compatible LLM connectors
+	// Get user-defined model capabilities once at the start of request
+	modelCapabilities := getModelCapabilities()
+
+	// Get all OpenAI-compatible LLM connectors from AIConnectors
+	// Note: All openai type connectors are automatically added to AIConnectors during loading
+	// See gou/connector/connector.go LoadSource() for details
 	for _, opt := range connector.AIConnectors {
 		connType := getConnectorType(opt.Value)
 		// Only include OpenAI-compatible LLM connectors
 		if connType == "openai" {
-			allProviders = append(allProviders, Provider{
-				Label:   opt.Label,
-				Value:   opt.Value,
-				Type:    connType,
-				Builtin: true,
-			})
-			added[opt.Value] = true
-		}
-	}
+			conn, ok := connector.Connectors[opt.Value]
+			if !ok {
+				continue
+			}
 
-	// 2. Get user-defined OpenAI-compatible LLM connectors from the global connector registry
-	// This includes all loaded connectors, both built-in and user-defined
-	// Only include OpenAI-compatible connectors (standard openai format)
-	for id, conn := range connector.Connectors {
-		// Skip if already added
-		if added[id] {
-			continue
-		}
+			// Get capabilities from connector settings
+			capabilities := getCapabilitiesWithModels(conn, modelCapabilities)
 
-		// Only include OpenAI-compatible LLM connectors
-		connType := getConnectorType(id)
-		if connType == "openai" {
-			meta := conn.GetMetaInfo()
-			label := meta.Label
-			if label == "" {
-				label = id
+			// Apply capability filters
+			if len(filters) > 0 && !matchesFilters(capabilities, filters) {
+				continue
 			}
 
 			allProviders = append(allProviders, Provider{
-				Label:   label,
-				Value:   id,
-				Type:    connType,
-				Builtin: meta.Builtin,
+				Label:        opt.Label,
+				Value:        opt.Value,
+				Type:         connType,
+				Builtin:      conn.GetMetaInfo().Builtin,
+				Capabilities: capabilities,
 			})
-			added[id] = true
 		}
 	}
 
@@ -92,3 +106,70 @@ func getConnectorType(id string) string {
 
 	return "unknown"
 }
+
+// getCapabilitiesWithModels extracts capabilities from connector settings
+// Uses the unified capability getter from agent/llm package
+// Takes modelCapabilities as parameter to avoid repeated calls to getModelCapabilities()
+func getCapabilitiesWithModels(conn connector.Connector, modelCapabilities map[string]openai.Capabilities) map[string]interface{} {
+	if conn == nil {
+		return nil
+	}
+
+	// Use unified capability getter with user-defined model capabilities
+	caps := agentllm.GetCapabilitiesFromConn(conn, modelCapabilities)
+	return agentllm.ToMap(caps)
+}
+
+// matchesFilters checks if capabilities match all requested filters
+// Filters are matched case-insensitively and support the following capability keys:
+// - vision: true or string value like "openai", "claude"
+// - audio: bool
+// - tool_calls: bool
+// - reasoning: bool
+// - streaming: bool
+// - json: bool
+// - multimodal: bool
+// - temperature_adjustable: bool
+func matchesFilters(capabilities map[string]interface{}, filters []string) bool {
+	if capabilities == nil {
+		return false
+	}
+
+	// All filters must match (AND logic)
+	for _, filter := range filters {
+		matched := false
+
+		// Check each capability field
+		for key, value := range capabilities {
+			keyLower := strings.ToLower(key)
+
+			// Match the filter against capability key
+			if keyLower == filter {
+				// For vision, check if it's true or a non-empty string
+				if filter == "vision" {
+					if boolVal, ok := value.(bool); ok && boolVal {
+						matched = true
+						break
+					}
+					if strVal, ok := value.(string); ok && strVal != "" {
+						matched = true
+						break
+					}
+				} else {
+					// For other capabilities, check if it's true
+					if boolVal, ok := value.(bool); ok && boolVal {
+						matched = true
+						break
+					}
+				}
+			}
+		}
+
+		// If any filter doesn't match, return false
+		if !matched {
+			return false
+		}
+	}
+
+	return true
+}