Add Detailed Validation for Agent Assertions in Dynamic Testing

- Introduced `ValidateWithDetails` method in the Asserter to provide comprehensive results for agent assertions, including failure messages and validation criteria.
- Updated `checkCheckpoints` to utilize the new validation method, enhancing the checkpoint validation process with detailed agent validation results.
- Enhanced `CheckpointResult` structure to include `AgentValidationResult`, capturing the agent's response and validation details.
- Expanded documentation in README.md to explain the new validation process and output structure for agent assertions, ensuring clarity on the validation workflow and expected results.

agent/test/README.md

@@ -607,6 +607,33 @@ For semantic or fuzzy validation using an LLM:
 
 The validator agent receives the output and criteria, then returns `{"passed": true/false, "reason": "..."}`.
 
+**How it works:**
+
+1. The framework builds a validation request with the agent's response (including tool result messages)
+2. The validator agent evaluates the response against the criteria
+3. The validator returns a JSON response with `passed` and `reason`
+
+**Output in test report (for checkpoints):**
+
+```json
+{
+  "agent_validation": {
+    "passed": true,
+    "reason": "Response explicitly confirms setup completion",
+    "criteria": "Response should be friendly and helpful",
+    "input": "Hello! How can I help you today?",
+    "response": {
+      "passed": true,
+      "reason": "Response explicitly confirms setup completion"
+    }
+  }
+}
+```
+
+- `input`: The content sent to the validator (agent response + tool result messages)
+- `response`: The raw JSON response from the validator agent
+- `criteria`: The validation criteria from the test case
+
 ### Script Assertions
 
 For custom validation logic:
@@ -1037,6 +1064,35 @@ interface TurnResult {
 }
 ```
 
+### Checkpoint Result Structure
+
+Each checkpoint in the output includes:
+
+```typescript
+interface CheckpointResult {
+  id: string; // Checkpoint identifier
+  reached: boolean; // Whether checkpoint was reached
+  reached_at_turn?: number; // Turn number when reached (if reached)
+  required: boolean; // Whether checkpoint is required
+  passed: boolean; // Whether assertion passed
+  message?: string; // Assertion result message
+  agent_validation?: {
+    // Agent assertion details (for type: "agent")
+    passed: boolean; // Validator's determination
+    reason: string; // Explanation from validator
+    criteria: string; // Validation criteria checked
+    input: any; // Content sent to validator
+    response: {
+      // Raw validator response
+      passed: boolean;
+      reason: string;
+    };
+  };
+}
+```
+
+**Note**: For agent-based assertions (`type: "agent"`), the `agent_validation` field provides full transparency into the validation process. The `input` field contains the combined output (agent text response + tool result messages) that was validated.
+
 ## Output Formats
 
 Determined by `-o` file extension:

agent/test/assert.go

@@ -42,6 +42,45 @@ func (a *Asserter) Validate(tc *Case, output interface{}) (bool, string) {
 	return true, ""
 }
 
+// ValidateWithDetails validates the output and returns detailed results
+// This is useful for agent assertions where we want to capture the validator's response
+func (a *Asserter) ValidateWithDetails(tc *Case, output interface{}) *AssertionResult {
+	if tc.Assert == nil {
+		return &AssertionResult{Passed: true}
+	}
+
+	assertions := a.parseAssertions(tc.Assert)
+	if len(assertions) == 0 {
+		return &AssertionResult{Passed: true}
+	}
+
+	// For single assertion, return its full result
+	if len(assertions) == 1 {
+		return a.evaluateAssertion(assertions[0], output, tc.Input)
+	}
+
+	// For multiple assertions, combine results
+	var failures []string
+	for _, assertion := range assertions {
+		result := a.evaluateAssertion(assertion, output, tc.Input)
+		if !result.Passed {
+			msg := result.Message
+			if assertion.Message != "" {
+				msg = assertion.Message
+			}
+			failures = append(failures, msg)
+		}
+	}
+
+	if len(failures) > 0 {
+		return &AssertionResult{
+			Passed:  false,
+			Message: strings.Join(failures, "; "),
+		}
+	}
+	return &AssertionResult{Passed: true}
+}
+
 // validateAssertions validates output against assertion rules
 func (a *Asserter) validateAssertions(tc *Case, output interface{}) (bool, string) {
 	assertions := a.parseAssertions(tc.Assert)

agent/test/dynamic_runner.go

@@ -194,8 +194,8 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s
 		// Add assistant response to messages, including tool calls if any
 		messages = appendAssistantMessages(messages, response)
 
-		// Check checkpoints against this response
+		// Check checkpoints against this response (including tool results)
-		reachedIDs := r.checkCheckpoints(tc.Checkpoints, output, result)
+		reachedIDs := r.checkCheckpoints(tc.Checkpoints, response, result)
 		turnResult.CheckpointsReached = reachedIDs
 
 		if r.opts.Verbose && len(reachedIDs) > 0 {
@@ -368,10 +368,15 @@ func (r *DynamicRunner) parseSimulatorResponse(response *context.Response) (*Sim
 	return output, nil
 }
 
-// checkCheckpoints validates checkpoints against current output
+// checkCheckpoints validates checkpoints against current response
-func (r *DynamicRunner) checkCheckpoints(checkpoints []*Checkpoint, output interface{}, result *DynamicResult) []string {
+// It checks both the completion content and tool results for comprehensive validation
+func (r *DynamicRunner) checkCheckpoints(checkpoints []*Checkpoint, response *context.Response, result *DynamicResult) []string {
 	reachedIDs := make([]string, 0)
 
+	// Build combined output for checkpoint validation
+	// This includes both content and tool result messages
+	combinedOutput := buildCombinedOutput(response)
+
 	for _, cp := range checkpoints {
 		cpResult := result.Checkpoints[cp.ID]
 		if cpResult.Reached {
@@ -394,22 +399,161 @@ func (r *DynamicRunner) checkCheckpoints(checkpoints []*Checkpoint, output inter
 			}
 		}
 
-		// Validate using asserter
+		// Validate using asserter against combined output with full details
 		tempCase := &Case{Assert: cp.Assert}
-		passed, msg := r.asserter.Validate(tempCase, output)
+		assertResult := r.asserter.ValidateWithDetails(tempCase, combinedOutput)
 
-		if passed {
+		if assertResult.Passed {
 			cpResult.Reached = true
 			cpResult.Passed = true
 			cpResult.ReachedAtTurn = len(result.Turns) + 1
-			cpResult.Message = msg
+			cpResult.Message = assertResult.Message
 			reachedIDs = append(reachedIDs, cp.ID)
 		}
+
+		// Store agent validation details if this is an agent assertion
+		if isAgentAssertion(cp.Assert) {
+			// Extract criteria from assertion value
+			var criteria string
+			if assertMap, ok := cp.Assert.(map[string]interface{}); ok {
+				if c, ok := assertMap["value"].(string); ok {
+					criteria = c
+				}
+			}
+
+			cpResult.AgentValidation = &AgentValidationResult{
+				Passed:   assertResult.Passed,
+				Criteria: criteria,
+				Input:    combinedOutput, // Content sent to validator for checking
+			}
+
+			// Extract reason and store full response from validator
+			if assertResult.Expected != nil {
+				if validatorResponse, ok := assertResult.Expected.(map[string]interface{}); ok {
+					if reason, ok := validatorResponse["reason"].(string); ok {
+						cpResult.AgentValidation.Reason = reason
+					}
+					// Store the full validator response
+					cpResult.AgentValidation.Response = validatorResponse
+				}
+			}
+		}
 	}
 
 	return reachedIDs
 }
 
+// isAgentAssertion checks if the assertion is an agent-based assertion
+func isAgentAssertion(assert interface{}) bool {
+	if assertMap, ok := assert.(map[string]interface{}); ok {
+		if assertType, ok := assertMap["type"].(string); ok {
+			return assertType == "agent"
+		}
+	}
+	return false
+}
+
+// truncateForReport truncates content for report output
+func truncateForReport(content interface{}, maxLen int) interface{} {
+	if content == nil {
+		return nil
+	}
+
+	str, ok := content.(string)
+	if !ok {
+		return content
+	}
+
+	if len(str) <= maxLen {
+		return str
+	}
+
+	return str[:maxLen] + "... (truncated)"
+}
+
+// buildCombinedOutput builds a combined output string from response
+// that includes both completion content and tool result messages
+func buildCombinedOutput(response *context.Response) string {
+	if response == nil {
+		return ""
+	}
+
+	var parts []string
+
+	// Add completion content
+	if response.Completion != nil && response.Completion.Content != nil {
+		if content := extractContentString(response.Completion.Content); content != "" {
+			parts = append(parts, content)
+		}
+	}
+
+	// Add tool result messages
+	if len(response.Tools) > 0 {
+		for _, tool := range response.Tools {
+			if tool.Result != nil {
+				// Try to extract message from result
+				if resultMap, ok := tool.Result.(map[string]interface{}); ok {
+					if msg, exists := resultMap["message"]; exists && msg != nil {
+						if msgStr, ok := msg.(string); ok && msgStr != "" {
+							parts = append(parts, msgStr)
+						}
+					}
+				}
+			}
+		}
+	}
+
+	// Join all parts with newline for comprehensive matching
+	return joinNonEmpty(parts, "\n")
+}
+
+// extractContentString extracts string content from various types
+func extractContentString(content interface{}) string {
+	if content == nil {
+		return ""
+	}
+
+	switch v := content.(type) {
+	case string:
+		return v
+	case []interface{}:
+		// Handle array content (e.g., multimodal content)
+		var texts []string
+		for _, item := range v {
+			if m, ok := item.(map[string]interface{}); ok {
+				if text, ok := m["text"].(string); ok {
+					texts = append(texts, text)
+				}
+			}
+		}
+		return joinNonEmpty(texts, "\n")
+	default:
+		// Try to marshal to string
+		if data, err := jsoniter.MarshalToString(content); err == nil {
+			return data
+		}
+		return ""
+	}
+}
+
+// joinNonEmpty joins non-empty strings with separator
+func joinNonEmpty(parts []string, sep string) string {
+	var nonEmpty []string
+	for _, p := range parts {
+		if p != "" {
+			nonEmpty = append(nonEmpty, p)
+		}
+	}
+	if len(nonEmpty) == 0 {
+		return ""
+	}
+	result := nonEmpty[0]
+	for i := 1; i < len(nonEmpty); i++ {
+		result += sep + nonEmpty[i]
+	}
+	return result
+}
+
 // allRequiredCheckpointsReached checks if all required checkpoints are reached
 func (r *DynamicRunner) allRequiredCheckpointsReached(result *DynamicResult) bool {
 	for _, cp := range result.Checkpoints {

agent/test/dynamic_types.go

@@ -93,6 +93,27 @@ type CheckpointResult struct {
 
 	// Message contains assertion result message
 	Message string `json:"message,omitempty"`
+
+	// AgentValidation contains the agent validator's response (for agent assertions)
+	AgentValidation *AgentValidationResult `json:"agent_validation,omitempty"`
+}
+
+// AgentValidationResult contains the result from an agent-based assertion
+type AgentValidationResult struct {
+	// Passed indicates if the agent validator determined the assertion passed
+	Passed bool `json:"passed"`
+
+	// Reason is the explanation from the agent validator
+	Reason string `json:"reason,omitempty"`
+
+	// Criteria is the validation criteria that was checked
+	Criteria string `json:"criteria,omitempty"`
+
+	// Input is the content that was sent to validator for checking
+	Input interface{} `json:"input,omitempty"`
+
+	// Response is the raw response from the validator agent
+	Response interface{} `json:"response,omitempty"`
 }
 
 // SimulatorInput is the input sent to the simulator agent