Add tests for JSON response formatting and schema validation in OpenAI provider
- Introduced TestOpenAIJSONMode and TestOpenAIJSONModePost to validate JSON response formatting for streaming and non-streaming requests. - Added TestOpenAIJSONSchema and TestOpenAIJSONSchemaPost to ensure compliance with strict JSON schema validation. - Enhanced request body construction in the OpenAI provider to support JSON schema details in response formats.
This commit is contained in:
Max
parent
68ed9ece6d
commit
d705fcc484
3 changed files with 439 additions and 11 deletions
|
|
@ -559,7 +559,17 @@ func (p *Provider) buildRequestBody(messages []context.Message, options *context
|
|||
}
|
||||
|
||||
if options.ResponseFormat != nil {
|
||||
body["response_format"] = options.ResponseFormat
|
||||
// Build response_format according to OpenAI API requirements
|
||||
responseFormat := map[string]interface{}{
|
||||
"type": options.ResponseFormat.Type,
|
||||
}
|
||||
|
||||
// For json_schema type, include the schema details
|
||||
if options.ResponseFormat.Type == context.ResponseFormatJSONSchema && options.ResponseFormat.JSONSchema != nil {
|
||||
responseFormat["json_schema"] = options.ResponseFormat.JSONSchema
|
||||
}
|
||||
|
||||
body["response_format"] = responseFormat
|
||||
}
|
||||
|
||||
if options.Seed != nil {
|
||||
|
|
|
|||
|
|
@ -803,6 +803,424 @@ func TestOpenAIToolCallValidationRetry(t *testing.T) {
|
|||
t.Log("Automatic tool call validation retry test completed")
|
||||
}
|
||||
|
||||
// TestOpenAIJSONMode tests JSON mode response formatting
|
||||
func TestOpenAIJSONMode(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
conn, err := connector.Select("openai.gpt-4o")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to select connector: %v", err)
|
||||
}
|
||||
|
||||
trueVal := true
|
||||
options := &context.CompletionOptions{
|
||||
Capabilities: &context.ModelCapabilities{
|
||||
Streaming: &trueVal,
|
||||
ToolCalls: &trueVal,
|
||||
},
|
||||
ResponseFormat: &context.ResponseFormat{
|
||||
Type: context.ResponseFormatJSON,
|
||||
},
|
||||
}
|
||||
|
||||
llmInstance, err := llm.New(conn, options)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create LLM instance: %v", err)
|
||||
}
|
||||
|
||||
messages := []context.Message{
|
||||
{
|
||||
Role: context.RoleUser,
|
||||
Content: "Generate a JSON object with fields: name (string), age (number), city (string). Use values: John, 30, New York",
|
||||
},
|
||||
}
|
||||
|
||||
ctx := newTestContext("test-json-mode", "openai.gpt-4o")
|
||||
|
||||
// Test streaming with JSON mode
|
||||
response, err := llmInstance.Stream(ctx, messages, options, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Stream with JSON mode failed: %v", err)
|
||||
}
|
||||
|
||||
if response == nil {
|
||||
t.Fatal("Response is nil")
|
||||
}
|
||||
|
||||
// Validate response
|
||||
contentStr, ok := response.Content.(string)
|
||||
if !ok || contentStr == "" {
|
||||
t.Error("Response content is empty or not a string")
|
||||
}
|
||||
|
||||
// Try to parse as JSON
|
||||
var jsonData map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(contentStr), &jsonData); err != nil {
|
||||
t.Errorf("Response is not valid JSON: %v\nContent: %s", err, contentStr)
|
||||
} else {
|
||||
t.Logf("✓ Response is valid JSON: %+v", jsonData)
|
||||
|
||||
// Verify expected fields exist
|
||||
if _, hasName := jsonData["name"]; !hasName {
|
||||
t.Error("JSON response missing 'name' field")
|
||||
}
|
||||
if _, hasAge := jsonData["age"]; !hasAge {
|
||||
t.Error("JSON response missing 'age' field")
|
||||
}
|
||||
if _, hasCity := jsonData["city"]; !hasCity {
|
||||
t.Error("JSON response missing 'city' field")
|
||||
}
|
||||
}
|
||||
|
||||
// Validate metadata
|
||||
if response.ID == "" {
|
||||
t.Error("Response ID is empty")
|
||||
}
|
||||
if response.Model == "" {
|
||||
t.Error("Response Model is empty")
|
||||
}
|
||||
if response.Usage == nil {
|
||||
t.Error("Response Usage is nil")
|
||||
} else {
|
||||
t.Logf("Usage: prompt=%d, completion=%d, total=%d",
|
||||
response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens)
|
||||
}
|
||||
|
||||
t.Log("JSON mode test completed successfully")
|
||||
}
|
||||
|
||||
// TestOpenAIJSONModePost tests JSON mode with non-streaming
|
||||
func TestOpenAIJSONModePost(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
conn, err := connector.Select("openai.gpt-4o")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to select connector: %v", err)
|
||||
}
|
||||
|
||||
trueVal := true
|
||||
options := &context.CompletionOptions{
|
||||
Capabilities: &context.ModelCapabilities{
|
||||
ToolCalls: &trueVal,
|
||||
},
|
||||
ResponseFormat: &context.ResponseFormat{
|
||||
Type: context.ResponseFormatJSON,
|
||||
},
|
||||
}
|
||||
|
||||
llmInstance, err := llm.New(conn, options)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create LLM instance: %v", err)
|
||||
}
|
||||
|
||||
messages := []context.Message{
|
||||
{
|
||||
Role: context.RoleUser,
|
||||
Content: "Return a JSON with: status='success', count=42",
|
||||
},
|
||||
}
|
||||
|
||||
ctx := newTestContext("test-json-mode-post", "openai.gpt-4o")
|
||||
|
||||
// Test non-streaming with JSON mode
|
||||
response, err := llmInstance.Post(ctx, messages, options)
|
||||
if err != nil {
|
||||
t.Fatalf("Post with JSON mode failed: %v", err)
|
||||
}
|
||||
|
||||
if response == nil {
|
||||
t.Fatal("Response is nil")
|
||||
}
|
||||
|
||||
// Validate response content is JSON
|
||||
contentStr, ok := response.Content.(string)
|
||||
if !ok || contentStr == "" {
|
||||
t.Error("Response content is empty or not a string")
|
||||
}
|
||||
|
||||
var jsonData map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(contentStr), &jsonData); err != nil {
|
||||
t.Errorf("Response is not valid JSON: %v\nContent: %s", err, contentStr)
|
||||
} else {
|
||||
t.Logf("✓ Response is valid JSON: %+v", jsonData)
|
||||
}
|
||||
|
||||
// Validate metadata
|
||||
if response.Usage == nil {
|
||||
t.Error("Response Usage is nil")
|
||||
} else {
|
||||
if response.Usage.TotalTokens == 0 {
|
||||
t.Error("Response Usage.TotalTokens is 0")
|
||||
}
|
||||
t.Logf("Usage: prompt=%d, completion=%d, total=%d",
|
||||
response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens)
|
||||
}
|
||||
|
||||
t.Log("JSON mode Post test completed successfully")
|
||||
}
|
||||
|
||||
// TestOpenAIJSONSchema tests JSON mode with strict schema validation
|
||||
func TestOpenAIJSONSchema(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
conn, err := connector.Select("openai.gpt-4o")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to select connector: %v", err)
|
||||
}
|
||||
|
||||
trueVal := true
|
||||
|
||||
// Define a strict JSON schema
|
||||
// Note: For OpenAI strict mode, 'required' must include ALL properties
|
||||
schema := map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"user": map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"name": map[string]interface{}{
|
||||
"type": "string",
|
||||
"description": "User's full name",
|
||||
},
|
||||
"email": map[string]interface{}{
|
||||
"type": "string",
|
||||
"description": "User's email address",
|
||||
},
|
||||
"age": map[string]interface{}{
|
||||
"type": "integer",
|
||||
"description": "User's age",
|
||||
},
|
||||
"isActive": map[string]interface{}{
|
||||
"type": "boolean",
|
||||
"description": "Whether user is active",
|
||||
},
|
||||
},
|
||||
"required": []string{"name", "email", "age", "isActive"},
|
||||
"additionalProperties": false,
|
||||
},
|
||||
},
|
||||
"required": []string{"user"},
|
||||
"additionalProperties": false,
|
||||
}
|
||||
|
||||
options := &context.CompletionOptions{
|
||||
Capabilities: &context.ModelCapabilities{
|
||||
Streaming: &trueVal,
|
||||
ToolCalls: &trueVal,
|
||||
},
|
||||
ResponseFormat: &context.ResponseFormat{
|
||||
Type: context.ResponseFormatJSONSchema,
|
||||
JSONSchema: &context.JSONSchema{
|
||||
Name: "user_info",
|
||||
Description: "User information schema",
|
||||
Schema: schema,
|
||||
Strict: &trueVal,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
llmInstance, err := llm.New(conn, options)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create LLM instance: %v", err)
|
||||
}
|
||||
|
||||
messages := []context.Message{
|
||||
{
|
||||
Role: context.RoleUser,
|
||||
Content: "Create user info for: Alice Smith, alice@example.com, age 28, active user",
|
||||
},
|
||||
}
|
||||
|
||||
ctx := newTestContext("test-json-schema", "openai.gpt-4o")
|
||||
|
||||
// Test streaming with JSON schema
|
||||
response, err := llmInstance.Stream(ctx, messages, options, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Stream with JSON schema failed: %v", err)
|
||||
}
|
||||
|
||||
if response == nil {
|
||||
t.Fatal("Response is nil")
|
||||
}
|
||||
|
||||
// Validate response content
|
||||
contentStr, ok := response.Content.(string)
|
||||
if !ok || contentStr == "" {
|
||||
t.Fatal("Response content is empty or not a string")
|
||||
}
|
||||
|
||||
// Parse as JSON
|
||||
var jsonData map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(contentStr), &jsonData); err != nil {
|
||||
t.Fatalf("Response is not valid JSON: %v\nContent: %s", err, contentStr)
|
||||
}
|
||||
|
||||
t.Logf("✓ Response is valid JSON: %+v", jsonData)
|
||||
|
||||
// Verify structure matches schema
|
||||
user, hasUser := jsonData["user"].(map[string]interface{})
|
||||
if !hasUser {
|
||||
t.Fatal("JSON response missing 'user' object")
|
||||
}
|
||||
|
||||
// Verify required fields
|
||||
if _, hasName := user["name"]; !hasName {
|
||||
t.Error("User object missing required 'name' field")
|
||||
}
|
||||
if _, hasEmail := user["email"]; !hasEmail {
|
||||
t.Error("User object missing required 'email' field")
|
||||
}
|
||||
|
||||
// Verify field types
|
||||
if name, ok := user["name"].(string); ok {
|
||||
t.Logf("✓ name: %s (string)", name)
|
||||
} else {
|
||||
t.Error("name is not a string")
|
||||
}
|
||||
|
||||
if email, ok := user["email"].(string); ok {
|
||||
t.Logf("✓ email: %s (string)", email)
|
||||
} else {
|
||||
t.Error("email is not a string")
|
||||
}
|
||||
|
||||
if age, ok := user["age"].(float64); ok {
|
||||
if age < 0 || age > 150 {
|
||||
t.Errorf("age %v is out of range [0, 150]", age)
|
||||
}
|
||||
t.Logf("✓ age: %v (integer, in range)", age)
|
||||
}
|
||||
|
||||
if isActive, ok := user["isActive"].(bool); ok {
|
||||
t.Logf("✓ isActive: %v (boolean)", isActive)
|
||||
}
|
||||
|
||||
// Validate metadata
|
||||
if response.Usage == nil {
|
||||
t.Error("Response Usage is nil")
|
||||
} else {
|
||||
t.Logf("Usage: prompt=%d, completion=%d, total=%d",
|
||||
response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens)
|
||||
}
|
||||
|
||||
t.Log("JSON schema test completed successfully")
|
||||
}
|
||||
|
||||
// TestOpenAIJSONSchemaPost tests JSON schema with non-streaming
|
||||
func TestOpenAIJSONSchemaPost(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
conn, err := connector.Select("openai.gpt-4o")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to select connector: %v", err)
|
||||
}
|
||||
|
||||
trueVal := true
|
||||
|
||||
// Simple schema for testing
|
||||
// Note: For OpenAI strict mode, 'required' must include ALL properties
|
||||
schema := map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"status": map[string]interface{}{
|
||||
"type": "string",
|
||||
"enum": []string{"success", "error", "pending"},
|
||||
},
|
||||
"message": map[string]interface{}{
|
||||
"type": "string",
|
||||
},
|
||||
"code": map[string]interface{}{
|
||||
"type": "integer",
|
||||
},
|
||||
},
|
||||
"required": []string{"status", "message", "code"},
|
||||
"additionalProperties": false,
|
||||
}
|
||||
|
||||
options := &context.CompletionOptions{
|
||||
Capabilities: &context.ModelCapabilities{
|
||||
ToolCalls: &trueVal,
|
||||
},
|
||||
ResponseFormat: &context.ResponseFormat{
|
||||
Type: context.ResponseFormatJSONSchema,
|
||||
JSONSchema: &context.JSONSchema{
|
||||
Name: "api_response",
|
||||
Description: "API response format",
|
||||
Schema: schema,
|
||||
Strict: &trueVal,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
llmInstance, err := llm.New(conn, options)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create LLM instance: %v", err)
|
||||
}
|
||||
|
||||
messages := []context.Message{
|
||||
{
|
||||
Role: context.RoleUser,
|
||||
Content: "Generate an API response with status 'success', message 'Operation completed', and code 200",
|
||||
},
|
||||
}
|
||||
|
||||
ctx := newTestContext("test-json-schema-post", "openai.gpt-4o")
|
||||
|
||||
// Test non-streaming with JSON schema
|
||||
response, err := llmInstance.Post(ctx, messages, options)
|
||||
if err != nil {
|
||||
t.Fatalf("Post with JSON schema failed: %v", err)
|
||||
}
|
||||
|
||||
if response == nil {
|
||||
t.Fatal("Response is nil")
|
||||
}
|
||||
|
||||
// Validate response content
|
||||
contentStr, ok := response.Content.(string)
|
||||
if !ok || contentStr == "" {
|
||||
t.Fatal("Response content is empty or not a string")
|
||||
}
|
||||
|
||||
// Parse and validate JSON
|
||||
var jsonData map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(contentStr), &jsonData); err != nil {
|
||||
t.Fatalf("Response is not valid JSON: %v\nContent: %s", err, contentStr)
|
||||
}
|
||||
|
||||
t.Logf("✓ Response is valid JSON: %+v", jsonData)
|
||||
|
||||
// Verify required fields
|
||||
status, hasStatus := jsonData["status"].(string)
|
||||
if !hasStatus {
|
||||
t.Fatal("Missing required 'status' field")
|
||||
}
|
||||
|
||||
// Verify enum constraint
|
||||
validStatuses := map[string]bool{"success": true, "error": true, "pending": true}
|
||||
if !validStatuses[status] {
|
||||
t.Errorf("status '%s' is not in enum [success, error, pending]", status)
|
||||
}
|
||||
|
||||
if _, hasMessage := jsonData["message"].(string); !hasMessage {
|
||||
t.Error("Missing required 'message' field")
|
||||
}
|
||||
|
||||
// Validate metadata
|
||||
if response.Usage == nil {
|
||||
t.Error("Response Usage is nil")
|
||||
} else {
|
||||
t.Logf("Usage: prompt=%d, completion=%d, total=%d",
|
||||
response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens)
|
||||
}
|
||||
|
||||
t.Log("JSON schema Post test completed successfully")
|
||||
}
|
||||
|
||||
// TestOpenAIProxySupport tests that HTTP proxy configuration is respected
|
||||
func TestOpenAIProxySupport(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
|
|
|
|||
|
|
@ -29,23 +29,23 @@ func TestVision_Make(t *testing.T) {
|
|||
// Error is expected because connector is not loaded in test environment
|
||||
})
|
||||
|
||||
t.Run("option with connector should return error due to missing connector", func(t *testing.T) {
|
||||
t.Run("option with non-existent connector should return error", func(t *testing.T) {
|
||||
option := &kbtypes.ProviderOption{
|
||||
Properties: map[string]interface{}{
|
||||
"connector": "openai.gpt-4o-mini",
|
||||
"connector": "non-existent.connector",
|
||||
},
|
||||
}
|
||||
_, err := vision.Make(option)
|
||||
if err == nil {
|
||||
t.Error("Expected error due to missing connector")
|
||||
t.Error("Expected error for non-existent connector")
|
||||
}
|
||||
// Error is expected because openai.gpt-4o-mini connector is not loaded
|
||||
// Error is expected because non-existent.connector is not loaded
|
||||
})
|
||||
|
||||
t.Run("option with all properties should return error due to missing connector", func(t *testing.T) {
|
||||
t.Run("option with all properties but non-existent connector should return error", func(t *testing.T) {
|
||||
option := &kbtypes.ProviderOption{
|
||||
Properties: map[string]interface{}{
|
||||
"connector": "openai.gpt-4o",
|
||||
"connector": "fake.vision.connector",
|
||||
"model": "gpt-4o",
|
||||
"prompt": "Describe this image",
|
||||
"compress_size": 1024,
|
||||
|
|
@ -58,21 +58,21 @@ func TestVision_Make(t *testing.T) {
|
|||
}
|
||||
_, err := vision.Make(option)
|
||||
if err == nil {
|
||||
t.Error("Expected error due to missing connector")
|
||||
t.Error("Expected error for non-existent connector")
|
||||
}
|
||||
// Error is expected because openai.gpt-4o connector is not loaded
|
||||
// Error is expected because fake.vision.connector is not loaded
|
||||
})
|
||||
|
||||
t.Run("compress_size as float64 should be converted to int64 but still return error", func(t *testing.T) {
|
||||
option := &kbtypes.ProviderOption{
|
||||
Properties: map[string]interface{}{
|
||||
"connector": "openai.gpt-4o-mini",
|
||||
"connector": "invalid.test.connector",
|
||||
"compress_size": 512.0, // float64
|
||||
},
|
||||
}
|
||||
_, err := vision.Make(option)
|
||||
if err == nil {
|
||||
t.Error("Expected error due to missing connector")
|
||||
t.Error("Expected error for invalid connector")
|
||||
}
|
||||
// Error is expected because connector is not loaded
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue