diff --git a/agent/assistant/agent.go b/agent/assistant/agent.go index 5675aef0..85e32ec3 100644 --- a/agent/assistant/agent.go +++ b/agent/assistant/agent.go @@ -6,6 +6,7 @@ import ( "github.com/yaoapp/gou/connector" "github.com/yaoapp/yao/agent/context" "github.com/yaoapp/yao/agent/llm" + "github.com/yaoapp/yao/utils/jsonschema" ) // Stream stream the agent @@ -188,7 +189,10 @@ func (ast *Assistant) BuildRequest(ctx *context.Context, messages []context.Mess } // Build completion options from createResponse and ctx - options := ast.buildCompletionOptions(ctx, createResponse) + options, err := ast.buildCompletionOptions(ctx, createResponse) + if err != nil { + return nil, nil, err + } return finalMessages, options, nil } @@ -210,11 +214,13 @@ func (ast *Assistant) buildMessages(ctx *context.Context, messages []context.Mes // buildCompletionOptions builds completion options from multiple sources // Priority (lowest to highest, later overrides earlier): ast > ctx > createResponse // The priority means: if createResponse has a value, use it; else use ctx; else use ast -func (ast *Assistant) buildCompletionOptions(ctx *context.Context, createResponse *context.HookCreateResponse) *context.CompletionOptions { +func (ast *Assistant) buildCompletionOptions(ctx *context.Context, createResponse *context.HookCreateResponse) (*context.CompletionOptions, error) { options := &context.CompletionOptions{} // Layer 1 (base): Apply ast - Assistant configuration - ast.applyAssistantOptions(options) + if err := ast.applyAssistantOptions(options); err != nil { + return nil, err + } // Layer 2 (middle): Apply ctx - Context configuration (overrides ast) ast.applyContextOptions(options, ctx) @@ -224,14 +230,15 @@ func (ast *Assistant) buildCompletionOptions(ctx *context.Context, createRespons ast.applyCreateResponseOptions(options, createResponse) } - return options + return options, nil } // applyAssistantOptions applies options from ast.Options to CompletionOptions // ast.Options can contain any OpenAI API parameters (temperature, top_p, stop, etc.) -func (ast *Assistant) applyAssistantOptions(options *context.CompletionOptions) { +// Returns error if any option validation fails (e.g., invalid JSON Schema) +func (ast *Assistant) applyAssistantOptions(options *context.CompletionOptions) error { if ast.Options == nil { - return + return nil } // Temperature @@ -302,8 +309,53 @@ func (ast *Assistant) applyAssistantOptions(options *context.CompletionOptions) } // ResponseFormat - if v, ok := ast.Options["response_format"].(map[string]interface{}); ok { - options.ResponseFormat = v + // @todo: Assistant should have a default response format + if v, ok := ast.Options["response_format"]; ok { + // Try to convert to *context.ResponseFormat + if rf, ok := v.(*context.ResponseFormat); ok { + // Validate JSONSchema if present - reject if invalid + if rf.JSONSchema != nil && rf.JSONSchema.Schema != nil { + if _, err := jsonschema.New(rf.JSONSchema.Schema); err != nil { + return fmt.Errorf("invalid JSON Schema in response_format: %w", err) + } + } + options.ResponseFormat = rf + } else if rfMap, ok := v.(map[string]interface{}); ok { + // Handle legacy map[string]interface{} format + // Try to parse into ResponseFormat struct + rf := &context.ResponseFormat{} + + // Parse type + if typeStr, ok := rfMap["type"].(string); ok { + rf.Type = context.ResponseFormatType(typeStr) + } + + // Parse json_schema if present + if jsonSchemaMap, ok := rfMap["json_schema"].(map[string]interface{}); ok { + jsonSchema := &context.JSONSchema{} + + if name, ok := jsonSchemaMap["name"].(string); ok { + jsonSchema.Name = name + } + if desc, ok := jsonSchemaMap["description"].(string); ok { + jsonSchema.Description = desc + } + if schema, ok := jsonSchemaMap["schema"]; ok { + // Validate schema format - reject if invalid + if _, err := jsonschema.New(schema); err != nil { + return fmt.Errorf("invalid JSON Schema in response_format: %w", err) + } + jsonSchema.Schema = schema + } + if strict, ok := jsonSchemaMap["strict"].(bool); ok { + jsonSchema.Strict = &strict + } + + rf.JSONSchema = jsonSchema + } + + options.ResponseFormat = rf + } } // Seed @@ -336,6 +388,8 @@ func (ast *Assistant) applyAssistantOptions(options *context.CompletionOptions) if v, ok := ast.Options["stream"].(bool); ok { options.Stream = &v } + + return nil } // applyContextOptions applies options from ctx to CompletionOptions @@ -345,14 +399,9 @@ func (ast *Assistant) applyContextOptions(options *context.CompletionOptions, ct options.Route = ctx.Route options.Metadata = ctx.Metadata - // Set wrapper configurations (assistant.Uses has priority over global settings) + // Set Uses configurations (assistant.Uses has priority over global settings) // These can be overridden by createResponse - if visionWrapper := ast.getVisionWrapper(); visionWrapper != "" { - options.VisionWrapper = visionWrapper - } - if audioWrapper := ast.getAudioWrapper(); audioWrapper != "" { - options.AudioWrapper = audioWrapper - } + options.Uses = ast.getUses() } // applyCreateResponseOptions applies options from createResponse to CompletionOptions @@ -396,34 +445,40 @@ func (ast *Assistant) applyCreateResponseOptions(options *context.CompletionOpti } } -// getVisionWrapper get the vision wrapper with priority: assistant.Uses > global settings -func (ast *Assistant) getVisionWrapper() string { +// getUses get the Uses configuration with priority: assistant.Uses > global settings +func (ast *Assistant) getUses() *context.Uses { // Priority 1: Assistant-specific Uses configuration - if ast.Uses != nil && ast.Uses.Vision != "" { - return ast.Uses.Vision + if ast.Uses != nil { + // Create a merged Uses by starting with global, then override with assistant-specific + merged := &context.Uses{} + + // Start with global settings + if globalUses != nil { + merged.Vision = globalUses.Vision + merged.Audio = globalUses.Audio + merged.Search = globalUses.Search + merged.Fetch = globalUses.Fetch + } + + // Override with assistant-specific settings (only if not empty) + if ast.Uses.Vision != "" { + merged.Vision = ast.Uses.Vision + } + if ast.Uses.Audio != "" { + merged.Audio = ast.Uses.Audio + } + if ast.Uses.Search != "" { + merged.Search = ast.Uses.Search + } + if ast.Uses.Fetch != "" { + merged.Fetch = ast.Uses.Fetch + } + + return merged } - // Priority 2: Global settings from globalUses - if globalUses != nil && globalUses.Vision != "" { - return globalUses.Vision - } - - return "" -} - -// getAudioWrapper get the audio wrapper with priority: assistant.Uses > global settings -func (ast *Assistant) getAudioWrapper() string { - // Priority 1: Assistant-specific Uses configuration - if ast.Uses != nil && ast.Uses.Audio != "" { - return ast.Uses.Audio - } - - // Priority 2: Global settings from globalUses - if globalUses != nil && globalUses.Audio != "" { - return globalUses.Audio - } - - return "" + // Priority 2: Global settings only + return globalUses } // WithHistory with the history messages diff --git a/agent/assistant/agent_test.go b/agent/assistant/agent_test.go index ac47e145..ec0874a1 100644 --- a/agent/assistant/agent_test.go +++ b/agent/assistant/agent_test.go @@ -259,4 +259,156 @@ func TestBuildRequest(t *testing.T) { t.Log("✓ Nil createResponse: ast.Options and ctx values used") }) + + // Test 6: ResponseFormat with *context.ResponseFormat + t.Run("ResponseFormatStruct", func(t *testing.T) { + freshCtx := newTestContext("chat-test-response-format", "tests.buildrequest") + inputMessages := []context.Message{{Role: "user", Content: "test message"}} + + // Create a test agent with response_format in Options + testAgent := *agent + strict := true + testAgent.Options = map[string]interface{}{ + "temperature": 0.7, + "response_format": &context.ResponseFormat{ + Type: context.ResponseFormatJSONSchema, + JSONSchema: &context.JSONSchema{ + Name: "test_schema", + Description: "Test schema description", + Schema: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "name": map[string]interface{}{ + "type": "string", + }, + }, + }, + Strict: &strict, + }, + }, + } + + _, options, err := testAgent.BuildRequest(freshCtx, inputMessages, nil) + if err != nil { + t.Fatalf("Failed to build LLM request: %s", err.Error()) + } + + // Verify ResponseFormat + if options.ResponseFormat == nil { + t.Fatal("Expected ResponseFormat, got nil") + } + + if options.ResponseFormat.Type != context.ResponseFormatJSONSchema { + t.Errorf("Expected type 'json_schema', got: %s", options.ResponseFormat.Type) + } + + if options.ResponseFormat.JSONSchema == nil { + t.Fatal("Expected JSONSchema, got nil") + } + + if options.ResponseFormat.JSONSchema.Name != "test_schema" { + t.Errorf("Expected schema name 'test_schema', got: %s", options.ResponseFormat.JSONSchema.Name) + } + + if options.ResponseFormat.JSONSchema.Description != "Test schema description" { + t.Errorf("Expected schema description 'Test schema description', got: %s", options.ResponseFormat.JSONSchema.Description) + } + + if options.ResponseFormat.JSONSchema.Strict == nil || *options.ResponseFormat.JSONSchema.Strict != true { + t.Errorf("Expected strict = true, got: %v", options.ResponseFormat.JSONSchema.Strict) + } + + t.Log("✓ ResponseFormat with *context.ResponseFormat struct works correctly") + }) + + // Test 7: ResponseFormat with legacy map[string]interface{} + t.Run("ResponseFormatLegacyMap", func(t *testing.T) { + freshCtx := newTestContext("chat-test-response-format-map", "tests.buildrequest") + inputMessages := []context.Message{{Role: "user", Content: "test message"}} + + // Create a test agent with legacy map format + testAgent := *agent + testAgent.Options = map[string]interface{}{ + "temperature": 0.7, + "response_format": map[string]interface{}{ + "type": "json_schema", + "json_schema": map[string]interface{}{ + "name": "legacy_schema", + "description": "Legacy schema format", + "schema": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "email": map[string]interface{}{ + "type": "string", + }, + }, + }, + "strict": true, + }, + }, + } + + _, options, err := testAgent.BuildRequest(freshCtx, inputMessages, nil) + if err != nil { + t.Fatalf("Failed to build LLM request: %s", err.Error()) + } + + // Verify ResponseFormat was converted from map + if options.ResponseFormat == nil { + t.Fatal("Expected ResponseFormat, got nil") + } + + if options.ResponseFormat.Type != context.ResponseFormatJSONSchema { + t.Errorf("Expected type 'json_schema', got: %s", options.ResponseFormat.Type) + } + + if options.ResponseFormat.JSONSchema == nil { + t.Fatal("Expected JSONSchema, got nil") + } + + if options.ResponseFormat.JSONSchema.Name != "legacy_schema" { + t.Errorf("Expected schema name 'legacy_schema', got: %s", options.ResponseFormat.JSONSchema.Name) + } + + if options.ResponseFormat.JSONSchema.Description != "Legacy schema format" { + t.Errorf("Expected schema description 'Legacy schema format', got: %s", options.ResponseFormat.JSONSchema.Description) + } + + t.Log("✓ ResponseFormat with legacy map[string]interface{} format works correctly") + }) + + // Test 8: ResponseFormat with simple type (text or json_object) + t.Run("ResponseFormatSimpleType", func(t *testing.T) { + freshCtx := newTestContext("chat-test-response-format-simple", "tests.buildrequest") + inputMessages := []context.Message{{Role: "user", Content: "test message"}} + + // Create a test agent with simple response_format + testAgent := *agent + testAgent.Options = map[string]interface{}{ + "temperature": 0.7, + "response_format": map[string]interface{}{ + "type": "json_object", + }, + } + + _, options, err := testAgent.BuildRequest(freshCtx, inputMessages, nil) + if err != nil { + t.Fatalf("Failed to build LLM request: %s", err.Error()) + } + + // Verify ResponseFormat + if options.ResponseFormat == nil { + t.Fatal("Expected ResponseFormat, got nil") + } + + if options.ResponseFormat.Type != context.ResponseFormatJSON { + t.Errorf("Expected type 'json_object', got: %s", options.ResponseFormat.Type) + } + + if options.ResponseFormat.JSONSchema != nil { + t.Errorf("Expected JSONSchema to be nil for simple type, got: %v", options.ResponseFormat.JSONSchema) + } + + t.Log("✓ ResponseFormat with simple type (json_object) works correctly") + }) } diff --git a/agent/assistant/load.go b/agent/assistant/load.go index 5f721606..aa9f71ac 100644 --- a/agent/assistant/load.go +++ b/agent/assistant/load.go @@ -14,6 +14,7 @@ import ( "github.com/yaoapp/gou/fs" v8 "github.com/yaoapp/gou/runtime/v8" "github.com/yaoapp/yao/agent/assistant/hook" + "github.com/yaoapp/yao/agent/context" "github.com/yaoapp/yao/agent/i18n" store "github.com/yaoapp/yao/agent/store/types" agentvision "github.com/yaoapp/yao/agent/vision" @@ -29,7 +30,7 @@ var search interface{} = nil var connectorSettings map[string]ConnectorSetting = map[string]ConnectorSetting{} var vision *agentvision.Vision = nil var defaultConnector string = "" // default connector -var globalUses *store.Uses = nil // global uses configuration from agent.yml +var globalUses *context.Uses = nil // global uses configuration from agent.yml // LoadBuiltIn load the built-in assistants func LoadBuiltIn() error { @@ -147,7 +148,7 @@ func SetConnector(c string) { } // SetGlobalUses set the global uses configuration -func SetGlobalUses(uses *store.Uses) { +func SetGlobalUses(uses *context.Uses) { globalUses = uses } diff --git a/agent/context/interfaces.go b/agent/context/interfaces.go index ddf3197d..ac7c1a36 100644 --- a/agent/context/interfaces.go +++ b/agent/context/interfaces.go @@ -2,8 +2,28 @@ package context import "net/http" -// StreamFunc the streaming function -type StreamFunc func(data []byte) int +// StreamChunkType represents the type of content in a streaming chunk +type StreamChunkType string + +// Stream chunk type constants - indicates what type of content is in the current chunk +const ( + ChunkText StreamChunkType = "text" // Regular text content + ChunkThinking StreamChunkType = "thinking" // Reasoning/thinking content (o1, DeepSeek R1) + ChunkToolCall StreamChunkType = "tool_call" // Tool/function call + ChunkRefusal StreamChunkType = "refusal" // Model refusal + ChunkMetadata StreamChunkType = "metadata" // Metadata (usage, finish_reason, etc.) + ChunkError StreamChunkType = "error" // Error chunk + ChunkUnknown StreamChunkType = "unknown" // Unknown/unrecognized chunk type +) + +// StreamFunc the streaming function callback +// Parameters: +// - chunkType: the type of content in this chunk (text, thinking, tool_call, etc.) +// - data: the actual chunk data (could be text, JSON, or other format) +// +// Returns: +// - int: status code (0 = continue, non-zero = stop streaming) +type StreamFunc func(chunkType StreamChunkType, data []byte) int // Writer is an alias for http.ResponseWriter interface used by an agent to construct a response. // A Writer may not be used after the agent execution has completed. diff --git a/agent/context/types_llm.go b/agent/context/types_llm.go index 57e8cf32..c78dc69b 100644 --- a/agent/context/types_llm.go +++ b/agent/context/types_llm.go @@ -1,5 +1,14 @@ package context +// Uses represents the wrapper configurations for assistant +// Used to specify which assistant or MCP server to use for vision, audio, search, and fetch operations +type Uses struct { + Vision string `json:"vision,omitempty"` // Vision processing tool. Format: "agent" or "mcp:server_id" + Audio string `json:"audio,omitempty"` // Audio processing tool. Format: "agent" or "mcp:server_id" + Search string `json:"search,omitempty"` // Search tool. Format: "agent" or "mcp:server_id" + Fetch string `json:"fetch,omitempty"` // Fetch/retrieval tool. Format: "agent" or "mcp:server_id" +} + // ModelCapabilities defines the capabilities of a language model // Used by LLM to select appropriate provider and validate requests type ModelCapabilities struct { @@ -20,10 +29,8 @@ type CompletionOptions struct { // nil means capabilities are not specified/checked Capabilities *ModelCapabilities `json:"capabilities,omitempty"` - // Wrapper configurations for vision and audio processing - // Format: "agent" (default) or "mcp:mcp_server_id" - VisionWrapper string `json:"vision_wrapper,omitempty"` // Vision processing wrapper (for image/video description) - AudioWrapper string `json:"audio_wrapper,omitempty"` // Audio processing wrapper (for speech-to-text/text-to-speech) + // User-specified tools for vision, audio, search, and fetch processing + Uses *Uses `json:"uses,omitempty"` // Audio configuration (for models that support audio output) Audio *AudioConfig `json:"audio,omitempty"` @@ -42,9 +49,9 @@ type CompletionOptions struct { LogitBias map[string]float64 `json:"logit_bias,omitempty"` // Modify likelihood of specified tokens appearing // User and response format - User string `json:"user,omitempty"` // Unique identifier representing end-user - ResponseFormat map[string]interface{} `json:"response_format,omitempty"` // Format of the response (e.g., {"type": "json_object"}) - Seed *int `json:"seed,omitempty"` // Seed for deterministic sampling + User string `json:"user,omitempty"` // Unique identifier representing end-user + ResponseFormat *ResponseFormat `json:"response_format,omitempty"` // Format of the model's output + Seed *int `json:"seed,omitempty"` // Seed for deterministic sampling // Tool calling Tools []map[string]interface{} `json:"tools,omitempty"` // List of tools the model may call @@ -59,8 +66,8 @@ type CompletionOptions struct { Metadata map[string]interface{} `json:"metadata,omitempty"` // Metadata to pass to the page for CUI context } -// CompletionResponse represents the unified completion response -// Compatible with OpenAI chat completion response format +// CompletionResponse represents the unified LLM completion response +// This is Yao's internal representation that works with multiple LLM providers (OpenAI, Claude, DeepSeek, etc.) type CompletionResponse struct { // Response metadata ID string `json:"id"` // Unique identifier for the completion @@ -68,79 +75,90 @@ type CompletionResponse struct { Created int64 `json:"created"` // Unix timestamp of creation Model string `json:"model"` // Model used for completion - // Completion content (these fields can coexist) - Content string `json:"content"` // Text content (regular response text) - ReasoningContent string `json:"reasoning_content,omitempty"` // Reasoning/thinking content (for o1, DeepSeek R1, etc.) - ToolCalls []ToolCallResult `json:"tool_calls,omitempty"` // Tool calls made by the model - Refusal string `json:"refusal,omitempty"` // Refusal message if model refused to answer - ContentTypes []ContentType `json:"content_types"` // Types of content present (can have multiple simultaneously) + // Response message (similar to OpenAI's message structure) + Role string `json:"role"` // Role of the response, typically "assistant" + Content interface{} `json:"content,omitempty"` // string (text) or []ContentPart (multimodal: text, image, audio) - // Raw response data - Raw interface{} `json:"raw,omitempty"` // Original raw response from the LLM provider (for debugging and special cases) + // Tool calls (when model calls functions/tools) + ToolCalls []ToolCall `json:"tool_calls,omitempty"` // Tool calls made by the model + + // Refusal (when model refuses to respond due to policy) + Refusal string `json:"refusal,omitempty"` // Refusal message if model refused to answer + + // Reasoning content (for reasoning models like o1, DeepSeek R1) + ReasoningContent string `json:"reasoning_content,omitempty"` // Thinking/reasoning process // Completion metadata - FinishReason string `json:"finish_reason"` // Reason for completion (stop, length, tool_calls, content_filter, etc.) + FinishReason string `json:"finish_reason"` // Why generation stopped (stop, length, tool_calls, content_filter, etc.) // Usage statistics Usage *UsageInfo `json:"usage,omitempty"` // Token usage statistics // Additional metadata SystemFingerprint string `json:"system_fingerprint,omitempty"` // System fingerprint for reproducibility - Metadata map[string]interface{} `json:"metadata,omitempty"` // Additional metadata + Metadata map[string]interface{} `json:"metadata,omitempty"` // Additional provider-specific metadata + + // Raw response data (for debugging and special cases) + Raw interface{} `json:"raw,omitempty"` // Original raw response from the LLM provider } -// ContentType represents the type of content in the response -// A response can contain multiple content types simultaneously -type ContentType string - -// Content type constants - a response can have multiple types simultaneously -// For example: text + reasoning, or text + tool_call, or all three -const ( - ContentTypeText ContentType = "text" // Regular text content - ContentTypeReasoning ContentType = "reasoning" // Reasoning/thinking content (o1, DeepSeek R1, etc.) - ContentTypeToolCall ContentType = "tool_call" // Tool/function call - ContentTypeRefusal ContentType = "refusal" // Model refused to answer - ContentTypeEmpty ContentType = "empty" // Empty response (no content) -) - // UsageInfo represents token usage statistics +// Structure matches OpenAI API: https://platform.openai.com/docs/api-reference/chat/object#chat-object-usage type UsageInfo struct { - PromptTokens int `json:"prompt_tokens"` // Tokens in the prompt - CompletionTokens int `json:"completion_tokens"` // Tokens in the completion - TotalTokens int `json:"total_tokens"` // Total tokens used + PromptTokens int `json:"prompt_tokens"` // Number of tokens in the prompt + CompletionTokens int `json:"completion_tokens"` // Number of tokens in the generated completion + TotalTokens int `json:"total_tokens"` // Total number of tokens used (prompt + completion) - // Detailed token breakdown (for models with reasoning) - PromptTokensDetails *TokenDetails `json:"prompt_tokens_details,omitempty"` // Detailed prompt token breakdown - CompletionTokensDetails *TokenDetails `json:"completion_tokens_details,omitempty"` // Detailed completion token breakdown + // Detailed token breakdown + PromptTokensDetails *PromptTokensDetails `json:"prompt_tokens_details,omitempty"` // Breakdown of tokens used in the prompt + CompletionTokensDetails *CompletionTokensDetails `json:"completion_tokens_details,omitempty"` // Breakdown of tokens used in the completion } -// TokenDetails provides detailed token usage breakdown -type TokenDetails struct { - CachedTokens int `json:"cached_tokens,omitempty"` // Tokens from cache - ReasoningTokens int `json:"reasoning_tokens,omitempty"` // Tokens used for reasoning/thinking - AudioTokens int `json:"audio_tokens,omitempty"` // Tokens used for audio - TextTokens int `json:"text_tokens,omitempty"` // Tokens used for text +// PromptTokensDetails provides detailed breakdown of tokens used in the prompt +type PromptTokensDetails struct { + AudioTokens int `json:"audio_tokens,omitempty"` // Audio input tokens present in the prompt + CachedTokens int `json:"cached_tokens,omitempty"` // Cached tokens present in the prompt } -// ToolCallResult represents a tool call result in the completion -type ToolCallResult struct { - ID string `json:"id"` // Tool call ID - Type string `json:"type"` // Tool call type (usually "function") - Function FunctionCallResult `json:"function"` // Function call details +// CompletionTokensDetails provides detailed breakdown of tokens used in the completion +type CompletionTokensDetails struct { + AcceptedPredictionTokens int `json:"accepted_prediction_tokens,omitempty"` // Tokens from predictions that appeared in the completion + AudioTokens int `json:"audio_tokens,omitempty"` // Audio input tokens generated by the model + ReasoningTokens int `json:"reasoning_tokens,omitempty"` // Tokens generated by the model for reasoning (o1, o1-mini, DeepSeek R1) + RejectedPredictionTokens int `json:"rejected_prediction_tokens,omitempty"` // Tokens from predictions that did not appear in the completion } -// FunctionCallResult represents a function call result -type FunctionCallResult struct { - Name string `json:"name"` // Function name - Arguments string `json:"arguments"` // Function arguments as JSON string -} - -// FinishReason constants +// FinishReason constants - why the model stopped generating tokens const ( - FinishReasonStop = "stop" // Natural stop point - FinishReasonLength = "length" // Max tokens reached - FinishReasonToolCalls = "tool_calls" // Tool calls made - FinishReasonContentFilter = "content_filter" // Content filtered - FinishReasonFunctionCall = "function_call" // Function call (deprecated) - FinishReasonError = "error" // Error occurred + FinishReasonStop = "stop" // Natural stop point or provided stop sequence reached + FinishReasonLength = "length" // Max tokens limit reached + FinishReasonToolCalls = "tool_calls" // Model called a tool + FinishReasonContentFilter = "content_filter" // Content filtered due to safety + FinishReasonFunctionCall = "function_call" // Model called a function (deprecated, use tool_calls) ) + +// ResponseFormat specifies the format of the model's output +// Reference: https://platform.openai.com/docs/api-reference/chat/create#chat_create-response_format +type ResponseFormat struct { + Type ResponseFormatType `json:"type"` // Required: type of response format + JSONSchema *JSONSchema `json:"json_schema,omitempty"` // Optional: for type="json_schema", defines the schema +} + +// ResponseFormatType represents the type of response format +type ResponseFormatType string + +// Response format type constants +const ( + ResponseFormatText ResponseFormatType = "text" // Default text format + ResponseFormatJSON ResponseFormatType = "json_object" // JSON object format (no schema) + ResponseFormatJSONSchema ResponseFormatType = "json_schema" // JSON with strict schema validation +) + +// JSONSchema defines a JSON schema for structured output +// Used when ResponseFormat.Type is "json_schema" +type JSONSchema struct { + Name string `json:"name"` // Required: name of the schema + Description string `json:"description,omitempty"` // Optional: description of the schema + Schema interface{} `json:"schema"` // Required: JSON schema (*jsonschema.Schema or map[string]interface{}) + Strict *bool `json:"strict,omitempty"` // Optional: whether to enforce strict schema validation (default: true) +} diff --git a/agent/context/types_wrapper.go b/agent/context/types_wrapper.go deleted file mode 100644 index a9909301..00000000 --- a/agent/context/types_wrapper.go +++ /dev/null @@ -1,49 +0,0 @@ -package context - -import "strings" - -// WrapperType represents the type of wrapper for processing -type WrapperType string - -const ( - WrapperTypeAgent WrapperType = "agent" // Use agent for processing - WrapperTypeMCP WrapperType = "mcp" // Use MCP server for processing -) - -// ParseWrapper parses a wrapper string and returns the type and ID -// Format: "agent" or "mcp:mcp_server_id" -func ParseWrapper(wrapper string) (WrapperType, string) { - if wrapper == "" || wrapper == "agent" { - return WrapperTypeAgent, "" - } - - if strings.HasPrefix(wrapper, "mcp:") { - mcpID := strings.TrimPrefix(wrapper, "mcp:") - return WrapperTypeMCP, mcpID - } - - // Default to agent if format is unknown - return WrapperTypeAgent, "" -} - -// IsAgentWrapper checks if the wrapper is an agent wrapper -func IsAgentWrapper(wrapper string) bool { - wrapperType, _ := ParseWrapper(wrapper) - return wrapperType == WrapperTypeAgent -} - -// IsMCPWrapper checks if the wrapper is an MCP wrapper -func IsMCPWrapper(wrapper string) bool { - wrapperType, _ := ParseWrapper(wrapper) - return wrapperType == WrapperTypeMCP -} - -// GetMCPServerID extracts the MCP server ID from wrapper string -// Returns empty string if not an MCP wrapper -func GetMCPServerID(wrapper string) string { - wrapperType, id := ParseWrapper(wrapper) - if wrapperType == WrapperTypeMCP { - return id - } - return "" -} diff --git a/agent/llm/handlers/handlers.go b/agent/llm/handlers/handlers.go deleted file mode 100644 index 53fb4a93..00000000 --- a/agent/llm/handlers/handlers.go +++ /dev/null @@ -1,51 +0,0 @@ -package handlers - -import ( - "github.com/yaoapp/yao/agent/context" -) - -// Handler interface for stream handlers -type Handler interface { - OnChunk(chunk *StreamChunk) error - OnComplete() error - OnError(err error) error -} - -// NewDefaultHandler creates a default handler that sends chunks via context -func NewDefaultHandler(ctx *context.Context) Handler { - return &DefaultHandler{ - ctx: ctx, - } -} - -// DefaultHandler default stream handler implementation -type DefaultHandler struct { - ctx *context.Context -} - -// OnChunk handles a streaming chunk -func (h *DefaultHandler) OnChunk(chunk *StreamChunk) error { - // TODO: Implement chunk handling - // - Send chunk via ctx - // - Handle different chunk types - // - Aggregate content for final response - return SendStreamChunk(h.ctx, chunk) -} - -// OnComplete handles stream completion -func (h *DefaultHandler) OnComplete() error { - // TODO: Implement completion handling - // - Send final message - // - Close stream - // - Return aggregated response - return nil -} - -// OnError handles stream errors -func (h *DefaultHandler) OnError(err error) error { - // TODO: Implement error handling - // - Send error message to client - // - Log error - // - Clean up resources - return err -} diff --git a/agent/llm/handlers/stream.go b/agent/llm/handlers/stream.go index 22fc3aea..5e1be574 100644 --- a/agent/llm/handlers/stream.go +++ b/agent/llm/handlers/stream.go @@ -7,20 +7,35 @@ import ( // DefaultStreamHandler creates a default stream handler that sends messages via context // This handler is used when no custom handler is provided func DefaultStreamHandler(ctx *context.Context) context.StreamFunc { - return func(data []byte) int { + return func(chunkType context.StreamChunkType, data []byte) int { // TODO: Implement default stream handling - // - Parse streaming chunk data + // - Parse streaming chunk data based on chunkType // - Extract content from chunk // - Send message via ctx (SSE, WebSocket, etc.) - // - Handle different chunk types (content, tool_calls, reasoning) - // - Return 1 to continue streaming, 0 to stop - return 1 + // - Handle different chunk types (text, thinking, tool_calls, etc.) + // - Return 0 to continue streaming, non-zero to stop + + switch chunkType { + case context.ChunkText: + // Handle text content + case context.ChunkThinking: + // Handle reasoning/thinking content + case context.ChunkToolCall: + // Handle tool calls + case context.ChunkMetadata: + // Handle metadata (usage, finish_reason) + case context.ChunkError: + // Handle error + return 1 // Stop on error + } + + return 0 // Continue streaming } } // SendStreamChunk sends a stream chunk via context // Used internally by DefaultStreamHandler -func SendStreamChunk(ctx *context.Context, chunk *StreamChunk) error { +func SendStreamChunk(ctx *context.Context, chunkType context.StreamChunkType, data []byte) error { // TODO: Implement sending stream chunk // - Format chunk for transport (SSE, WebSocket) // - Send via ctx's connection @@ -28,58 +43,20 @@ func SendStreamChunk(ctx *context.Context, chunk *StreamChunk) error { return nil } -// StreamChunk represents a parsed streaming chunk -type StreamChunk struct { - Type ChunkType `json:"type"` // Type of chunk (content, reasoning, tool_call, etc.) - Content string `json:"content,omitempty"` // Text content - - // For reasoning chunks - ReasoningContent string `json:"reasoning_content,omitempty"` - - // For tool call chunks - ToolCallID string `json:"tool_call_id,omitempty"` - ToolCallFunction string `json:"tool_call_function,omitempty"` - ToolCallArgs string `json:"tool_call_args,omitempty"` - - // Metadata - Done bool `json:"done"` // Whether this is the final chunk - FinishReason string `json:"finish_reason,omitempty"` // Reason for completion (if done) -} - -// ChunkType represents the type of streaming chunk -type ChunkType string - -const ( - ChunkTypeContent ChunkType = "content" // Regular text content - ChunkTypeReasoning ChunkType = "reasoning" // Reasoning/thinking content - ChunkTypeToolCall ChunkType = "tool_call" // Tool call chunk - ChunkTypeDone ChunkType = "done" // Final chunk (completion) - ChunkTypeError ChunkType = "error" // Error chunk -) - -// ParseStreamChunk parses raw streaming data into StreamChunk -func ParseStreamChunk(data []byte) (*StreamChunk, error) { - // TODO: Implement stream chunk parsing - // - Parse SSE format (data: {...}) - // - Handle different provider formats (OpenAI, DeepSeek, etc.) - // - Extract content, reasoning, tool calls - // - Detect completion (done: true) - return nil, nil -} - -// FormatSSE formats a StreamChunk as Server-Sent Events format -func FormatSSE(chunk *StreamChunk) string { +// FormatSSE formats streaming data as Server-Sent Events format +func FormatSSE(chunkType context.StreamChunkType, data []byte) string { // TODO: Implement SSE formatting // - Format as "data: {...}\n\n" + // - Include chunk type in the message // - Handle special cases (done, error) // - Ensure proper JSON encoding return "" } -// FormatWebSocket formats a StreamChunk as WebSocket message -func FormatWebSocket(chunk *StreamChunk) []byte { +// FormatWebSocket formats streaming data as WebSocket message +func FormatWebSocket(chunkType context.StreamChunkType, data []byte) []byte { // TODO: Implement WebSocket formatting - // - Format as JSON message + // - Format as JSON message with chunk type // - Add message type/metadata // - Handle binary vs text frames return nil diff --git a/agent/llm/providers/legacy/legacy.go b/agent/llm/providers/legacy/legacy.go index 1bb387d8..a266452b 100644 --- a/agent/llm/providers/legacy/legacy.go +++ b/agent/llm/providers/legacy/legacy.go @@ -55,7 +55,7 @@ func (p *Provider) InjectToolInstructions(messages []context.Message, tools []ma } // ExtractToolCallsFromText extract tool calls from model's text response -func (p *Provider) ExtractToolCallsFromText(text string) []context.ToolCallResult { +func (p *Provider) ExtractToolCallsFromText(text string) []context.ToolCall { // TODO: Implement tool call extraction // - Look for JSON blocks or specific patterns // - Parse tool name and arguments diff --git a/agent/llm/providers/reasoning/reasoning.go b/agent/llm/providers/reasoning/reasoning.go index e9a2cb7d..15793d3f 100644 --- a/agent/llm/providers/reasoning/reasoning.go +++ b/agent/llm/providers/reasoning/reasoning.go @@ -92,7 +92,7 @@ func (p *Provider) injectToolInstructions(messages []context.Message, tools []ma // extractToolCallsFromText extract tool calls from reasoning model's text response // Used when model doesn't support native tool calls -func (p *Provider) extractToolCallsFromText(text string) []context.ToolCallResult { +func (p *Provider) extractToolCallsFromText(text string) []context.ToolCall { // TODO: Implement tool call extraction from text // - Look for JSON blocks or specific patterns // - Parse tool name and arguments diff --git a/agent/load.go b/agent/load.go index 7a6eed93..3680b895 100644 --- a/agent/load.go +++ b/agent/load.go @@ -9,6 +9,7 @@ import ( "github.com/yaoapp/kun/exception" "github.com/yaoapp/yao/agent/api" "github.com/yaoapp/yao/agent/assistant" + "github.com/yaoapp/yao/agent/context" "github.com/yaoapp/yao/agent/i18n" mongoStore "github.com/yaoapp/yao/agent/store/mongo" redisStore "github.com/yaoapp/yao/agent/store/redis" @@ -173,7 +174,7 @@ func initAssistant() error { // Set global Uses configuration if api.Agent.DSL.Use != nil { - globalUses := &store.Uses{ + globalUses := &context.Uses{ Vision: api.Agent.DSL.Use.Vision, Audio: api.Agent.DSL.Use.Audio, Search: api.Agent.DSL.Use.Search, diff --git a/agent/store/types/convert.go b/agent/store/types/convert.go index eb06af03..43b50260 100644 --- a/agent/store/types/convert.go +++ b/agent/store/types/convert.go @@ -173,15 +173,6 @@ func ToMySQLTime(v interface{}) string { } } -// Uses the wrapper configurations for assistant -// Used to specify which assistant or MCP server to use for vision, audio, etc. -type Uses struct { - Vision string `json:"vision,omitempty"` // Vision processing wrapper. Format: "agent" or "mcp:mcp_server_id" - Audio string `json:"audio,omitempty"` // Audio processing wrapper. Format: "agent" or "mcp:mcp_server_id" - Search string `json:"search,omitempty"` // Search wrapper. Format: "agent" or "mcp:mcp_server_id" - Fetch string `json:"fetch,omitempty"` // Fetch wrapper. Format: "agent" or "mcp:mcp_server_id" -} - // ToAssistantModel converts various types to AssistantModel func ToAssistantModel(v interface{}) (*AssistantModel, error) { if v == nil { diff --git a/agent/store/types/types.go b/agent/store/types/types.go index 7bf2e45a..3f3f7867 100644 --- a/agent/store/types/types.go +++ b/agent/store/types/types.go @@ -2,6 +2,7 @@ package types import ( "github.com/yaoapp/xun/dbal/query" + "github.com/yaoapp/yao/agent/context" "github.com/yaoapp/yao/agent/i18n" ) @@ -155,7 +156,7 @@ type AssistantModel struct { Workflow *Workflow `json:"workflow,omitempty"` // Workflow configuration Placeholder *Placeholder `json:"placeholder,omitempty"` // Assistant Placeholder Locales i18n.Map `json:"locales,omitempty"` // Assistant Locales - Uses *Uses `json:"uses,omitempty"` // Assistant-specific wrapper configurations for vision, audio, etc. If not set, use global settings + Uses *context.Uses `json:"uses,omitempty"` // Assistant-specific wrapper configurations for vision, audio, etc. If not set, use global settings CreatedAt int64 `json:"created_at"` // Creation timestamp UpdatedAt int64 `json:"updated_at"` // Last update timestamp diff --git a/agent/store/xun/assistant.go b/agent/store/xun/assistant.go index aa8382dc..d1339bcb 100644 --- a/agent/store/xun/assistant.go +++ b/agent/store/xun/assistant.go @@ -8,6 +8,7 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/yaoapp/kun/log" "github.com/yaoapp/xun/dbal/query" + "github.com/yaoapp/yao/agent/context" "github.com/yaoapp/yao/agent/i18n" "github.com/yaoapp/yao/agent/store/types" ) @@ -582,7 +583,7 @@ func (conv *Xun) GetAssistant(assistantID string, locale ...string) (*types.Assi if uses, has := data["uses"]; has && uses != nil { raw, err := jsoniter.Marshal(uses) if err == nil { - var u types.Uses + var u context.Uses if err := jsoniter.Unmarshal(raw, &u); err == nil { model.Uses = &u } diff --git a/agent/store/xun/assistant_test.go b/agent/store/xun/assistant_test.go index 4d812203..ce50c548 100644 --- a/agent/store/xun/assistant_test.go +++ b/agent/store/xun/assistant_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/yaoapp/xun/dbal/query" + "github.com/yaoapp/yao/agent/context" "github.com/yaoapp/yao/agent/i18n" "github.com/yaoapp/yao/agent/store/types" "github.com/yaoapp/yao/config" @@ -205,7 +206,7 @@ func TestSaveAssistant(t *testing.T) { Type: "assistant", Connector: "openai", Share: "private", - Uses: &types.Uses{ + Uses: &context.Uses{ Vision: "mcp:vision-server", Audio: "agent", Search: "mcp:search-server", @@ -279,7 +280,7 @@ func TestSaveAssistant(t *testing.T) { Type: "assistant", Connector: "openai", Share: "private", - Uses: &types.Uses{ + Uses: &context.Uses{ Vision: "mcp:vision-only", // Audio, Search, Fetch not set }, @@ -2088,7 +2089,7 @@ func TestUpdateAssistant(t *testing.T) { // Update with uses configuration updates := map[string]interface{}{ - "uses": &types.Uses{ + "uses": &context.Uses{ Vision: "mcp:new-vision", Audio: "mcp:new-audio", Search: "agent", @@ -2126,7 +2127,7 @@ func TestUpdateAssistant(t *testing.T) { // Update to change uses updates2 := map[string]interface{}{ - "uses": &types.Uses{ + "uses": &context.Uses{ Vision: "agent", Audio: "agent", }, diff --git a/go.mod b/go.mod index d6819720..b2e2ef35 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/yaoapp/yao -go 1.24.0 - -toolchain go1.24.3 +go 1.25 require ( github.com/PuerkitoBio/goquery v1.10.3 @@ -25,6 +23,7 @@ require ( github.com/joho/godotenv v1.5.1 github.com/json-iterator/go v1.1.12 github.com/kaptinlin/jsonrepair v0.1.1 + github.com/kaptinlin/jsonschema v0.5.2 github.com/matoous/go-nanoid/v2 v2.1.0 github.com/mozillazg/go-pinyin v0.20.0 github.com/pkoukk/tiktoken-go v0.1.7 @@ -32,7 +31,7 @@ require ( github.com/rhysd/go-github-selfupdate v1.2.3 github.com/spf13/cast v1.9.2 github.com/spf13/cobra v1.9.1 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 github.com/xuri/excelize/v2 v2.9.1 github.com/yaoapp/gou v0.10.3 github.com/yaoapp/kun v0.9.0 @@ -75,6 +74,7 @@ require ( github.com/gabriel-vasile/mimetype v1.4.9 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-errors/errors v1.5.1 // indirect + github.com/go-json-experiment/json v0.0.0-20250910080747-cc2cfa0554c3 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.1 // indirect @@ -84,6 +84,7 @@ require ( github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect github.com/go-sql-driver/mysql v1.9.2 // indirect github.com/goccy/go-json v0.10.5 // indirect + github.com/goccy/go-yaml v1.18.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/flatbuffers v25.2.10+incompatible // indirect @@ -101,6 +102,8 @@ require ( github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmoiron/sqlx v1.4.0 // indirect + github.com/kaptinlin/go-i18n v0.2.0 // indirect + github.com/kaptinlin/messageformat-go v0.4.5 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/leodido/go-urn v1.4.0 // indirect diff --git a/go.sum b/go.sum index b8f1663b..49e1a48c 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,8 @@ github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-json-experiment/json v0.0.0-20250910080747-cc2cfa0554c3 h1:02WINGfSX5w0Mn+F28UyRoSt9uvMhKguwWMlOAh6U/0= +github.com/go-json-experiment/json v0.0.0-20250910080747-cc2cfa0554c3/go.mod h1:uNVvRXArCGbZ508SxYYTC5v1JWoz2voff5pm25jU1Ok= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -119,6 +121,8 @@ github.com/go-sql-driver/mysql v1.9.2 h1:4cNKDYQ1I84SXslGddlsrMhc8k4LeDVj6Ad6WRj github.com/go-sql-driver/mysql v1.9.2/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -177,8 +181,14 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kaptinlin/go-i18n v0.2.0 h1:8iwjAERQbCVF78c3HxC4MxUDxDRFvQVQlMDvlsO43hU= +github.com/kaptinlin/go-i18n v0.2.0/go.mod h1:gRHEMrTHtQLsAFwulPbJG71TwHjXxkagn88O8FI8FuA= github.com/kaptinlin/jsonrepair v0.1.1 h1:Ddn1sN1cZXuXeKA9vpaHAtBETnGSFBZFaaYfoN2Uo8c= github.com/kaptinlin/jsonrepair v0.1.1/go.mod h1:SivjE7np/GsSrk7UX/9mibH6VF8cVpD2aUmg7vceg2k= +github.com/kaptinlin/jsonschema v0.5.2 h1:ipUBEv1/RnT+ErwdqXZ3Xtwkwp6uqp/Q9lFILrwhUfc= +github.com/kaptinlin/jsonschema v0.5.2/go.mod h1:HuWb90460GwFxRe0i9Ni3Z7YXwkjpqjeccWTB9gTZZE= +github.com/kaptinlin/messageformat-go v0.4.5 h1:Y1CTf38O6lKKXX/UZTwb2Xw7c6DPk7kjQEHPJW6qxTI= +github.com/kaptinlin/messageformat-go v0.4.5/go.mod h1:r0PH7FsxJX8jS/n6LAYZon5w3X+yfCLUrquqYd2H7ks= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -285,8 +295,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tcnksm/go-gitconfig v0.1.2 h1:iiDhRitByXAEyjgBqsKi9QU4o2TNtv9kPP3RgPgXBPw= github.com/tcnksm/go-gitconfig v0.1.2/go.mod h1:/8EhP4H7oJZdIPyT+/UIsG87kTzrzM4UsLGSItWYCpE= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= diff --git a/utils/jsonschema/jsonschema.go b/utils/jsonschema/jsonschema.go new file mode 100644 index 00000000..528b5041 --- /dev/null +++ b/utils/jsonschema/jsonschema.go @@ -0,0 +1,170 @@ +package jsonschema + +import ( + "encoding/json" + "fmt" + + "github.com/kaptinlin/jsonschema" + "github.com/yaoapp/gou/process" +) + +// Validator wraps a compiled JSON Schema for validation +type Validator struct { + schema *jsonschema.Schema +} + +// New compiles a JSON Schema and returns a validator +// Returns error if the schema is invalid +// +// Args: +// - schema: can be map[string]interface{}, []byte, string, or any JSON-serializable type +// +// Usage: +// +// // From map +// schemaMap := map[string]interface{}{ +// "type": "object", +// "properties": map[string]interface{}{ +// "name": map[string]interface{}{"type": "string"}, +// }, +// "required": []string{"name"}, +// } +// validator, err := jsonschema.New(schemaMap) +// +// // From JSON string +// validator, err := jsonschema.New(`{"type": "object", "properties": {...}}`) +// +// // From JSON bytes +// validator, err := jsonschema.New([]byte(`{"type": "object", ...}`)) +func New(schema interface{}) (*Validator, error) { + var schemaBytes []byte + var err error + + // Handle different input types + switch v := schema.(type) { + case string: + // Already a JSON string + schemaBytes = []byte(v) + case []byte: + // Already JSON bytes + schemaBytes = v + default: + // Marshal to JSON + schemaBytes, err = json.Marshal(schema) + if err != nil { + return nil, fmt.Errorf("failed to marshal schema: %w", err) + } + } + + // Compile the schema - this validates the schema structure + compiler := jsonschema.NewCompiler() + compiledSchema, err := compiler.Compile(schemaBytes) + if err != nil { + return nil, fmt.Errorf("invalid JSON Schema: %w", err) + } + + return &Validator{ + schema: compiledSchema, + }, nil +} + +// Validate validates data against the compiled JSON Schema +// Returns nil if data is valid, error with validation details otherwise +// +// Usage: +// +// validator, _ := jsonschema.New(schemaMap) +// data := map[string]interface{}{"name": "John"} +// if err := validator.Validate(data); err != nil { +// log.Printf("Validation failed: %v", err) +// } +func (v *Validator) Validate(data interface{}) error { + result := v.schema.Validate(data) + if !result.IsValid() { + // Collect all validation errors + var errMsg string + for field, err := range result.Errors { + if errMsg != "" { + errMsg += "; " + } + errMsg += fmt.Sprintf("%s: %s", field, err.Message) + } + return fmt.Errorf("validation failed: %s", errMsg) + } + + return nil +} + +// ValidateSchema validates a JSON Schema structure without compiling it +// Returns error if the schema is invalid +func ValidateSchema(schema interface{}) error { + _, err := New(schema) + return err +} + +// ValidateData validates data against a JSON Schema (one-shot validation) +// Returns error if schema is invalid or data doesn't match the schema +// +// Usage: +// +// err := jsonschema.ValidateData(schemaMap, data) +// if err != nil { +// log.Printf("Validation failed: %v", err) +// } +func ValidateData(schema interface{}, data interface{}) error { + validator, err := New(schema) + if err != nil { + return err + } + return validator.Validate(data) +} + +// **************************************** +// * Process Handlers for JS/DSL +// **************************************** + +// ProcessValidateSchema utils.jsonschema.ValidateSchema +// Validates a JSON Schema structure +// Args: schema interface{} - The JSON Schema to validate +// Returns: nil if valid, error message string if invalid +func ProcessValidateSchema(process *process.Process) interface{} { + process.ValidateArgNums(1) + schema := process.Args[0] + + err := ValidateSchema(schema) + if err != nil { + return err.Error() + } + return nil +} + +// ProcessValidate utils.jsonschema.Validate +// Validates data against a JSON Schema +// Args: +// - schema interface{} - The JSON Schema (map, string, or []byte) +// - data interface{} - The data to validate +// +// Returns: nil if valid, error message string if invalid +// +// Usage in JS/DSL: +// +// // Validate with schema map +// var result = Process("utils.jsonschema.Validate", schema, data) +// if result != null { +// log.Error("Validation failed: " + result) +// } +// +// // Validate with JSON string +// var schemaStr = '{"type": "object", "properties": {"name": {"type": "string"}}}' +// var result = Process("utils.jsonschema.Validate", schemaStr, {"name": "John"}) +func ProcessValidate(process *process.Process) interface{} { + process.ValidateArgNums(2) + schema := process.Args[0] + data := process.Args[1] + + err := ValidateData(schema, data) + if err != nil { + return err.Error() + } + return nil +} diff --git a/utils/jsonschema/jsonschema_test.go b/utils/jsonschema/jsonschema_test.go new file mode 100644 index 00000000..f4b1d30a --- /dev/null +++ b/utils/jsonschema/jsonschema_test.go @@ -0,0 +1,718 @@ +package jsonschema + +import ( + "strings" + "testing" + + "github.com/yaoapp/gou/process" +) + +// TestNew tests the New function +func TestNew(t *testing.T) { + t.Run("ValidSimpleSchema", func(t *testing.T) { + schema := map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "name": map[string]interface{}{ + "type": "string", + }, + }, + } + + validator, err := New(schema) + if err != nil { + t.Fatalf("Expected valid schema to compile, got error: %v", err) + } + + if validator == nil { + t.Fatal("Expected non-nil validator") + } + + if validator.schema == nil { + t.Fatal("Expected validator to have compiled schema") + } + + t.Log("✓ Valid simple schema compiled successfully") + }) + + t.Run("ValidComplexSchema", func(t *testing.T) { + schema := map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "name": map[string]interface{}{ + "type": "string", + "minLength": 1, + "maxLength": 100, + }, + "age": map[string]interface{}{ + "type": "integer", + "minimum": 0, + "maximum": 150, + }, + "email": map[string]interface{}{ + "type": "string", + "format": "email", + }, + }, + "required": []string{"name", "email"}, + } + + validator, err := New(schema) + if err != nil { + t.Fatalf("Expected valid complex schema to compile, got error: %v", err) + } + + if validator == nil { + t.Fatal("Expected non-nil validator") + } + + t.Log("✓ Valid complex schema compiled successfully") + }) + + t.Run("InvalidSchema_BadMinimum", func(t *testing.T) { + schema := map[string]interface{}{ + "type": "integer", + "minimum": "not a number", + } + + _, err := New(schema) + if err == nil { + t.Fatal("Expected error for invalid minimum value, got nil") + } + + t.Log("✓ Invalid schema (bad minimum value) rejected correctly") + }) + + t.Run("SchemaFromJSONString", func(t *testing.T) { + schemaJSON := `{ + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"} + }, + "required": ["name"] + }` + + validator, err := New(schemaJSON) + if err != nil { + t.Fatalf("Expected valid JSON string schema to compile, got error: %v", err) + } + + if validator == nil { + t.Fatal("Expected non-nil validator") + } + + // Test validation with the validator + data := map[string]interface{}{ + "name": "John", + "age": 30, + } + + err = validator.Validate(data) + if err != nil { + t.Fatalf("Expected valid data to pass validation, got error: %v", err) + } + + t.Log("✓ Schema from JSON string compiled and validated successfully") + }) + + t.Run("SchemaFromJSONBytes", func(t *testing.T) { + schemaJSON := []byte(`{ + "type": "object", + "properties": { + "email": {"type": "string", "format": "email"} + }, + "required": ["email"] + }`) + + validator, err := New(schemaJSON) + if err != nil { + t.Fatalf("Expected valid JSON bytes schema to compile, got error: %v", err) + } + + if validator == nil { + t.Fatal("Expected non-nil validator") + } + + // Test validation with the validator + data := map[string]interface{}{ + "email": "test@example.com", + } + + err = validator.Validate(data) + if err != nil { + t.Fatalf("Expected valid data to pass validation, got error: %v", err) + } + + t.Log("✓ Schema from JSON bytes compiled and validated successfully") + }) + + t.Run("InvalidJSONString", func(t *testing.T) { + schemaJSON := `{invalid json}` + + _, err := New(schemaJSON) + if err == nil { + t.Fatal("Expected error for invalid JSON string, got nil") + } + + t.Log("✓ Invalid JSON string rejected correctly") + }) +} + +// TestValidator_Validate tests the Validate method +func TestValidator_Validate(t *testing.T) { + schema := map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "name": map[string]interface{}{ + "type": "string", + "minLength": 1, + }, + "age": map[string]interface{}{ + "type": "integer", + "minimum": 0, + }, + }, + "required": []string{"name"}, + } + + validator, err := New(schema) + if err != nil { + t.Fatalf("Failed to compile schema for testing: %v", err) + } + + t.Run("ValidData", func(t *testing.T) { + data := map[string]interface{}{ + "name": "John Doe", + "age": 30, + } + + err := validator.Validate(data) + if err != nil { + t.Fatalf("Expected valid data to pass validation, got error: %v", err) + } + + t.Log("✓ Valid data passed validation") + }) + + t.Run("InvalidData_MissingRequired", func(t *testing.T) { + data := map[string]interface{}{ + "age": 25, + } + + err := validator.Validate(data) + if err == nil { + t.Fatal("Expected validation error for missing required field, got nil") + } + + if !strings.Contains(err.Error(), "validation failed") { + t.Errorf("Expected error message to contain 'validation failed', got: %v", err) + } + + t.Log("✓ Invalid data (missing required) rejected correctly") + }) + + t.Run("InvalidData_WrongType", func(t *testing.T) { + data := map[string]interface{}{ + "name": "Alice", + "age": "not a number", + } + + err := validator.Validate(data) + if err == nil { + t.Fatal("Expected validation error for wrong type, got nil") + } + + t.Log("✓ Invalid data (wrong type) rejected correctly") + }) +} + +// TestValidateSchema tests the ValidateSchema function +func TestValidateSchema(t *testing.T) { + t.Run("ValidSchema", func(t *testing.T) { + schema := map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "name": map[string]interface{}{"type": "string"}, + }, + } + + err := ValidateSchema(schema) + if err != nil { + t.Fatalf("Expected valid schema, got error: %v", err) + } + + t.Log("✓ Valid schema validated successfully") + }) + + t.Run("InvalidSchema", func(t *testing.T) { + schema := map[string]interface{}{ + "type": "integer", + "minimum": "invalid", + } + + err := ValidateSchema(schema) + if err == nil { + t.Fatal("Expected error for invalid schema, got nil") + } + + t.Log("✓ Invalid schema rejected correctly") + }) +} + +// TestValidateData tests the ValidateData function +func TestValidateData(t *testing.T) { + schema := map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "name": map[string]interface{}{ + "type": "string", + "minLength": 1, + }, + }, + "required": []string{"name"}, + } + + t.Run("ValidData", func(t *testing.T) { + data := map[string]interface{}{ + "name": "John", + } + + err := ValidateData(schema, data) + if err != nil { + t.Fatalf("Expected valid data, got error: %v", err) + } + + t.Log("✓ Valid data validated successfully") + }) + + t.Run("InvalidData", func(t *testing.T) { + data := map[string]interface{}{ + "name": "", + } + + err := ValidateData(schema, data) + if err == nil { + t.Fatal("Expected validation error, got nil") + } + + t.Log("✓ Invalid data rejected correctly") + }) + + t.Run("InvalidSchema", func(t *testing.T) { + invalidSchema := map[string]interface{}{ + "type": "integer", + "minimum": "invalid", + } + + data := map[string]interface{}{"value": 10} + + err := ValidateData(invalidSchema, data) + if err == nil { + t.Fatal("Expected error for invalid schema, got nil") + } + + t.Log("✓ Invalid schema rejected correctly") + }) +} + +// TestArraySchema tests validation with array schema +func TestArraySchema(t *testing.T) { + schema := map[string]interface{}{ + "type": "array", + "items": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "id": map[string]interface{}{ + "type": "integer", + }, + "name": map[string]interface{}{ + "type": "string", + }, + }, + "required": []string{"id"}, + }, + "minItems": 1, + } + + validator, err := New(schema) + if err != nil { + t.Fatalf("Failed to compile array schema: %v", err) + } + + t.Run("ValidArray", func(t *testing.T) { + data := []interface{}{ + map[string]interface{}{"id": 1, "name": "Item 1"}, + map[string]interface{}{"id": 2, "name": "Item 2"}, + } + + err := validator.Validate(data) + if err != nil { + t.Fatalf("Expected valid array to pass validation, got error: %v", err) + } + + t.Log("✓ Valid array data passed validation") + }) + + t.Run("InvalidArray_Empty", func(t *testing.T) { + data := []interface{}{} + + err := validator.Validate(data) + if err == nil { + t.Fatal("Expected validation error for empty array (minItems: 1), got nil") + } + + t.Log("✓ Invalid array (empty) rejected correctly") + }) + + t.Run("InvalidArray_MissingRequiredInItem", func(t *testing.T) { + data := []interface{}{ + map[string]interface{}{"name": "Item without ID"}, + } + + err := validator.Validate(data) + if err == nil { + t.Fatal("Expected validation error for item missing required field, got nil") + } + + t.Log("✓ Invalid array (item missing required field) rejected correctly") + }) +} + +// TestNestedSchema tests validation with nested objects +func TestNestedSchema(t *testing.T) { + schema := map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "user": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "profile": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "bio": map[string]interface{}{ + "type": "string", + "maxLength": 500, + }, + }, + }, + }, + }, + }, + } + + validator, err := New(schema) + if err != nil { + t.Fatalf("Failed to compile nested schema: %v", err) + } + + t.Run("ValidNestedData", func(t *testing.T) { + data := map[string]interface{}{ + "user": map[string]interface{}{ + "profile": map[string]interface{}{ + "bio": "This is a short bio", + }, + }, + } + + err := validator.Validate(data) + if err != nil { + t.Fatalf("Expected valid nested data to pass validation, got error: %v", err) + } + + t.Log("✓ Valid nested data passed validation") + }) + + t.Run("InvalidNestedData_ViolatesConstraint", func(t *testing.T) { + longBio := strings.Repeat("a", 501) + data := map[string]interface{}{ + "user": map[string]interface{}{ + "profile": map[string]interface{}{ + "bio": longBio, + }, + }, + } + + err := validator.Validate(data) + if err == nil { + t.Fatal("Expected validation error for bio exceeding maxLength, got nil") + } + + t.Log("✓ Invalid nested data (violates constraint) rejected correctly") + }) +} + +// TestProcessValidateSchema tests the ProcessValidateSchema handler +func TestProcessValidateSchema(t *testing.T) { + t.Run("ValidSchema", func(t *testing.T) { + schema := map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "name": map[string]interface{}{"type": "string"}, + }, + } + + p := process.New("test.process", nil) + p.Args = []interface{}{schema} + + result := ProcessValidateSchema(p) + if result != nil { + t.Fatalf("Expected nil for valid schema, got: %v", result) + } + + t.Log("✓ ProcessValidateSchema: valid schema passed") + }) + + t.Run("InvalidSchema", func(t *testing.T) { + schema := map[string]interface{}{ + "type": "integer", + "minimum": "not a number", + } + + p := process.New("test.process", nil) + p.Args = []interface{}{schema} + + result := ProcessValidateSchema(p) + if result == nil { + t.Fatal("Expected error for invalid schema, got nil") + } + + errMsg, ok := result.(string) + if !ok { + t.Fatalf("Expected error message string, got: %T", result) + } + + if !strings.Contains(errMsg, "invalid JSON Schema") { + t.Errorf("Expected error message to contain 'invalid JSON Schema', got: %s", errMsg) + } + + t.Log("✓ ProcessValidateSchema: invalid schema rejected correctly") + }) + + t.Run("SchemaFromJSONString", func(t *testing.T) { + schemaJSON := `{ + "type": "object", + "properties": { + "email": {"type": "string"} + } + }` + + p := process.New("test.process", nil) + p.Args = []interface{}{schemaJSON} + + result := ProcessValidateSchema(p) + if result != nil { + t.Fatalf("Expected nil for valid JSON string schema, got: %v", result) + } + + t.Log("✓ ProcessValidateSchema: JSON string schema passed") + }) +} + +// TestProcessValidate tests the ProcessValidate handler +func TestProcessValidate(t *testing.T) { + schema := map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "name": map[string]interface{}{ + "type": "string", + "minLength": 1, + }, + "age": map[string]interface{}{ + "type": "integer", + "minimum": 0, + }, + }, + "required": []string{"name"}, + } + + t.Run("ValidData", func(t *testing.T) { + data := map[string]interface{}{ + "name": "John Doe", + "age": 30, + } + + p := process.New("test.process", nil) + p.Args = []interface{}{schema, data} + + result := ProcessValidate(p) + if result != nil { + t.Fatalf("Expected nil for valid data, got: %v", result) + } + + t.Log("✓ ProcessValidate: valid data passed") + }) + + t.Run("InvalidData_MissingRequired", func(t *testing.T) { + data := map[string]interface{}{ + "age": 25, + } + + p := process.New("test.process", nil) + p.Args = []interface{}{schema, data} + + result := ProcessValidate(p) + if result == nil { + t.Fatal("Expected error for missing required field, got nil") + } + + errMsg, ok := result.(string) + if !ok { + t.Fatalf("Expected error message string, got: %T", result) + } + + if !strings.Contains(errMsg, "validation failed") { + t.Errorf("Expected error message to contain 'validation failed', got: %s", errMsg) + } + + t.Log("✓ ProcessValidate: missing required field rejected correctly") + }) + + t.Run("InvalidData_WrongType", func(t *testing.T) { + data := map[string]interface{}{ + "name": "Alice", + "age": "not a number", + } + + p := process.New("test.process", nil) + p.Args = []interface{}{schema, data} + + result := ProcessValidate(p) + if result == nil { + t.Fatal("Expected error for wrong type, got nil") + } + + _, ok := result.(string) + if !ok { + t.Fatalf("Expected error message string, got: %T", result) + } + + t.Log("✓ ProcessValidate: wrong type rejected correctly") + }) + + t.Run("InvalidData_ViolatesConstraint", func(t *testing.T) { + data := map[string]interface{}{ + "name": "", + } + + p := process.New("test.process", nil) + p.Args = []interface{}{schema, data} + + result := ProcessValidate(p) + if result == nil { + t.Fatal("Expected error for constraint violation, got nil") + } + + t.Log("✓ ProcessValidate: constraint violation rejected correctly") + }) + + t.Run("InvalidSchema", func(t *testing.T) { + invalidSchema := map[string]interface{}{ + "type": "integer", + "minimum": "not a number", + } + + data := map[string]interface{}{"value": 10} + + p := process.New("test.process", nil) + p.Args = []interface{}{invalidSchema, data} + + result := ProcessValidate(p) + if result == nil { + t.Fatal("Expected error for invalid schema, got nil") + } + + t.Log("✓ ProcessValidate: invalid schema rejected correctly") + }) + + t.Run("SchemaFromJSONString", func(t *testing.T) { + schemaJSON := `{ + "type": "object", + "properties": { + "username": {"type": "string", "minLength": 3} + }, + "required": ["username"] + }` + + data := map[string]interface{}{ + "username": "john", + } + + p := process.New("test.process", nil) + p.Args = []interface{}{schemaJSON, data} + + result := ProcessValidate(p) + if result != nil { + t.Fatalf("Expected nil for valid data with JSON string schema, got: %v", result) + } + + t.Log("✓ ProcessValidate: JSON string schema with valid data passed") + }) + + t.Run("SchemaFromJSONBytes", func(t *testing.T) { + schemaJSON := []byte(`{ + "type": "object", + "properties": { + "email": {"type": "string"} + }, + "required": ["email"] + }`) + + data := map[string]interface{}{ + "email": "test@example.com", + } + + p := process.New("test.process", nil) + p.Args = []interface{}{schemaJSON, data} + + result := ProcessValidate(p) + if result != nil { + t.Fatalf("Expected nil for valid data with JSON bytes schema, got: %v", result) + } + + t.Log("✓ ProcessValidate: JSON bytes schema with valid data passed") + }) + + t.Run("ComplexNestedValidation", func(t *testing.T) { + complexSchema := map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "user": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "profile": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "age": map[string]interface{}{ + "type": "integer", + "minimum": 18, + "maximum": 100, + }, + }, + "required": []string{"age"}, + }, + }, + "required": []string{"profile"}, + }, + }, + } + + data := map[string]interface{}{ + "user": map[string]interface{}{ + "profile": map[string]interface{}{ + "age": 25, + }, + }, + } + + p := process.New("test.process", nil) + p.Args = []interface{}{complexSchema, data} + + result := ProcessValidate(p) + if result != nil { + t.Fatalf("Expected nil for valid nested data, got: %v", result) + } + + t.Log("✓ ProcessValidate: complex nested validation passed") + }) +} diff --git a/utils/process.go b/utils/process.go index 46d74688..696a03b0 100644 --- a/utils/process.go +++ b/utils/process.go @@ -6,6 +6,7 @@ import ( "github.com/yaoapp/yao/utils/datetime" "github.com/yaoapp/yao/utils/fmt" "github.com/yaoapp/yao/utils/json" + "github.com/yaoapp/yao/utils/jsonschema" "github.com/yaoapp/yao/utils/otp" "github.com/yaoapp/yao/utils/str" "github.com/yaoapp/yao/utils/throw" @@ -111,6 +112,12 @@ func Init() { // JSON process.Register("utils.json.Validate", json.ProcessValidate) + // JSON Schema + process.RegisterGroup("utils.jsonschema", map[string]process.Handler{ + "ValidateSchema": jsonschema.ProcessValidateSchema, + "Validate": jsonschema.ProcessValidate, + }) + // **************************************** // * New Processes Version 0.10.5+ // ****************************************