Merge pull request #1399 from trheyi/main

Enhance Agent Response Structure and Dynamic Test Functionality

agent/assistant/agent.go

@@ -264,6 +264,12 @@ func (ast *Assistant) Stream(ctx *context.Context, inputMessages []context.Messa
 			// Execute all tool calls
 			toolResults, hasErrors := ast.executeToolCalls(ctx, currentResponse.ToolCalls, attempt)
 
+			// Build a map of tool call ID to arguments for quick lookup
+			toolCallArgsMap := make(map[string]interface{})
+			for _, tc := range currentResponse.ToolCalls {
+				toolCallArgsMap[tc.ID] = tc.Function.Arguments
+			}
+
 			// Convert toolResults to toolCallResponses
 			toolCallResponses = make([]context.ToolCallResponse, len(toolResults))
 			for i, result := range toolResults {
@@ -272,7 +278,7 @@ func (ast *Assistant) Stream(ctx *context.Context, inputMessages []context.Messa
 					ToolCallID: result.ToolCallID,
 					Server:     result.Server(),
 					Tool:       result.Tool(),
-					Arguments:  nil,
+					Arguments:  toolCallArgsMap[result.ToolCallID],
 					Result:     parsedContent,
 					Error:      "",
 				}

agent/assistant/next.go

@@ -30,6 +30,7 @@ func (ast *Assistant) processNextResponse(npc *NextProcessContext) (*agentContex
 			Create:      npc.CreateResponse,
 			Next:        npc.NextResponse.Data, // Put custom data in Next field
 			Completion:  npc.CompletionResponse,
+			Tools:       npc.ToolCallResponses,
 		}, nil
 	}
 
@@ -72,5 +73,6 @@ func (ast *Assistant) buildStandardResponse(npc *NextProcessContext) *agentConte
 		Create:      npc.CreateResponse,
 		Next:        npc.NextResponse,
 		Completion:  npc.CompletionResponse,
+		Tools:       npc.ToolCallResponses,
 	}
 }

agent/context/types.go

@@ -349,6 +349,7 @@ type Response struct {
 	Create      *HookCreateResponse `json:"create,omitempty"`     // Create response from the create hook
 	Next        interface{}         `json:"next,omitempty"`       // Next response from the next hook
 	Completion  *CompletionResponse `json:"completion,omitempty"` // Completion response from the completion hook
+	Tools       []ToolCallResponse  `json:"tools,omitempty"`      // Tool call results (if any tools were executed)
 }
 
 // HookCreateResponse the response of the create hook

agent/test/README.md

@@ -203,6 +203,33 @@ Simulator-driven testing with checkpoint validation. A simulator agent generates
 | `--fail-fast` | Stop on first failure                                    | false                      |
 | `--dry-run`   | Generate test cases without running them                 | false                      |
 
+## Custom Context File
+
+Create a JSON file for custom authorization:
+
+```json
+{
+  "chat_id": "test-chat-001",
+  "authorized": {
+    "user_id": "test-user-123",
+    "team_id": "test-team-456",
+    "constraints": {
+      "owner_only": true,
+      "extra": { "department": "engineering" }
+    }
+  },
+  "metadata": {
+    "mode": "test"
+  }
+}
+```
+
+Use with `--ctx`:
+
+```bash
+yao agent test -i scripts.expense.setup --ctx tests/context.json -v
+```
+
 ## Input Format (JSONL)
 
 Each line is a JSON object. Below are examples organized by scenario.
@@ -548,6 +575,8 @@ Use `assert` for flexible validation. If `assert` is defined, it takes precedenc
 | `json_path`    | Extract JSON path and compare | `{"type": "json_path", "path": "$.field", "value": true}` |
 | `regex`        | Match regex pattern           | `{"type": "regex", "value": "\\d+"}`                      |
 | `type`         | Check output type             | `{"type": "type", "value": "object"}`                     |
+| `tool_called`  | Check if a tool was called    | `{"type": "tool_called", "value": "setup"}`               |
+| `tool_result`  | Check tool execution result   | `{"type": "tool_result", "value": {"tool": "setup", "result": {"success": true}}}` |
 
 ### Assertion Fields
 
@@ -580,6 +609,103 @@ For semantic or fuzzy validation using an LLM:
 
 The validator agent receives the output and criteria, then returns `{"passed": true/false, "reason": "..."}`.
 
+**How it works:**
+
+1. The framework builds a validation request with the agent's response (including tool result messages)
+2. The validator agent evaluates the response against the criteria
+3. The validator returns a JSON response with `passed` and `reason`
+
+**Output in test report (for checkpoints):**
+
+```json
+{
+  "agent_validation": {
+    "passed": true,
+    "reason": "Response explicitly confirms setup completion",
+    "criteria": "Response should be friendly and helpful",
+    "input": "Hello! How can I help you today?",
+    "response": {
+      "passed": true,
+      "reason": "Response explicitly confirms setup completion"
+    }
+  }
+}
+```
+
+- `input`: The content sent to the validator (agent response + tool result messages)
+- `response`: The raw JSON response from the validator agent
+- `criteria`: The validation criteria from the test case
+
+### Tool Assertions
+
+For validating that specific tools were called and their results:
+
+#### tool_called
+
+Check if a specific tool was called:
+
+```jsonl
+{
+  "id": "T001",
+  "input": "Set up my expense system",
+  "assert": {
+    "type": "tool_called",
+    "value": "setup"
+  }
+}
+```
+
+**Value formats:**
+
+- **String**: Tool name (supports suffix matching, e.g., `"setup"` matches `"agents_expense_tools__setup"`)
+- **Array**: Any of the specified tools must be called
+- **Object**: Match tool name and optionally arguments
+
+```jsonl
+// Match any of these tools
+{"type": "tool_called", "value": ["setup", "init"]}
+
+// Match tool with specific arguments
+{"type": "tool_called", "value": {"name": "setup", "arguments": {"action": "init"}}}
+```
+
+#### tool_result
+
+Check the result of a tool execution:
+
+```jsonl
+{
+  "id": "T001",
+  "input": "Set up my expense system",
+  "assert": {
+    "type": "tool_result",
+    "value": {
+      "tool": "setup",
+      "result": {
+        "success": true
+      }
+    }
+  }
+}
+```
+
+**Result matching:**
+
+- If `result` is omitted, only checks that the tool executed without error
+- Supports partial matching (only specified fields are checked)
+- Supports regex patterns with `regex:` prefix for string values
+
+```jsonl
+// Just check tool executed without error
+{"type": "tool_result", "value": {"tool": "setup"}}
+
+// Check specific result fields
+{"type": "tool_result", "value": {"tool": "setup", "result": {"success": true}}}
+
+// Use regex for message matching
+{"type": "tool_result", "value": {"tool": "setup", "result": {"message": "regex:(?i)setup.*complete"}}}
+```
+
 ### Script Assertions
 
 For custom validation logic:
@@ -761,9 +887,12 @@ export function AfterAll(ctx: Context, results: TestResult[], beforeData: any) {
 
 ```typescript
 interface Context {
-  user_id: string; // Test user ID
-  team_id: string; // Test team ID
   locale: string; // Locale (e.g., "en-us")
+  authorized: {
+    user_id: string; // Test user ID
+    team_id: string; // Test team ID
+    constraints?: object; // Access constraints
+  };
   metadata: object; // Custom metadata from test case
 }
 ```
@@ -980,6 +1109,62 @@ For testing complex conversation flows where the path is unpredictable:
   └─ PASSED (3 turns, 3 checkpoints, 8.5s)
 ```
 
+### Dynamic Mode Output Structure
+
+Each turn in the output includes:
+
+```typescript
+interface TurnResult {
+  turn: number; // Turn number (1-based)
+  input: string; // User message
+  output: any; // Agent response summary (for display)
+  response: {
+    // Full agent response (for detailed analysis)
+    content: string; // LLM text content
+    tool_calls: [
+      {
+        // Tool calls made
+        tool: string; // Tool name
+        arguments: any; // Call arguments
+        result: any; // Execution result
+      }
+    ];
+    next: any; // Next hook data
+  };
+  checkpoints_reached: string[]; // Checkpoint IDs reached
+  duration_ms: number; // Execution time
+}
+```
+
+### Checkpoint Result Structure
+
+Each checkpoint in the output includes:
+
+```typescript
+interface CheckpointResult {
+  id: string; // Checkpoint identifier
+  reached: boolean; // Whether checkpoint was reached
+  reached_at_turn?: number; // Turn number when reached (if reached)
+  required: boolean; // Whether checkpoint is required
+  passed: boolean; // Whether assertion passed
+  message?: string; // Assertion result message
+  agent_validation?: {
+    // Agent assertion details (for type: "agent")
+    passed: boolean; // Validator's determination
+    reason: string; // Explanation from validator
+    criteria: string; // Validation criteria checked
+    input: any; // Content sent to validator
+    response: {
+      // Raw validator response
+      passed: boolean;
+      reason: string;
+    };
+  };
+}
+```
+
+**Note**: For agent-based assertions (`type: "agent"`), the `agent_validation` field provides full transparency into the validation process. The `input` field contains the combined output (agent text response + tool result messages) that was validated.
+
 ## Output Formats
 
 Determined by `-o` file extension:

agent/test/assert.go

@@ -15,13 +15,22 @@ import (
 )
 
 // Asserter handles test assertions
-type Asserter struct{}
+type Asserter struct {
+	// response holds the current response for tool-related assertions
+	response *context.Response
+}
 
 // NewAsserter creates a new asserter
 func NewAsserter() *Asserter {
 	return &Asserter{}
 }
 
+// WithResponse sets the response for tool-related assertions
+func (a *Asserter) WithResponse(response *context.Response) *Asserter {
+	a.response = response
+	return a
+}
+
 // Validate validates the output against the test case's assertions
 // Returns (passed, error message)
 func (a *Asserter) Validate(tc *Case, output interface{}) (bool, string) {
@@ -42,6 +51,45 @@ func (a *Asserter) Validate(tc *Case, output interface{}) (bool, string) {
 	return true, ""
 }
 
+// ValidateWithDetails validates the output and returns detailed results
+// This is useful for agent assertions where we want to capture the validator's response
+func (a *Asserter) ValidateWithDetails(tc *Case, output interface{}) *AssertionResult {
+	if tc.Assert == nil {
+		return &AssertionResult{Passed: true}
+	}
+
+	assertions := a.parseAssertions(tc.Assert)
+	if len(assertions) == 0 {
+		return &AssertionResult{Passed: true}
+	}
+
+	// For single assertion, return its full result
+	if len(assertions) == 1 {
+		return a.evaluateAssertion(assertions[0], output, tc.Input)
+	}
+
+	// For multiple assertions, combine results
+	var failures []string
+	for _, assertion := range assertions {
+		result := a.evaluateAssertion(assertion, output, tc.Input)
+		if !result.Passed {
+			msg := result.Message
+			if assertion.Message != "" {
+				msg = assertion.Message
+			}
+			failures = append(failures, msg)
+		}
+	}
+
+	if len(failures) > 0 {
+		return &AssertionResult{
+			Passed:  false,
+			Message: strings.Join(failures, "; "),
+		}
+	}
+	return &AssertionResult{Passed: true}
+}
+
 // validateAssertions validates output against assertion rules
 func (a *Asserter) validateAssertions(tc *Case, output interface{}) (bool, string) {
 	assertions := a.parseAssertions(tc.Assert)
@@ -166,6 +214,10 @@ func (a *Asserter) evaluateAssertion(assertion *Assertion, output, input interfa
 		result = a.assertScript(assertion, output, input)
 	case "agent":
 		result = a.assertAgent(assertion, output, input)
+	case "tool_called":
+		result = a.assertToolCalled(assertion)
+	case "tool_result":
+		result = a.assertToolResult(assertion)
 	default:
 		result.Passed = false
 		result.Message = fmt.Sprintf("unknown assertion type: %s", assertion.Type)
@@ -708,6 +760,268 @@ func (a *Asserter) assertScript(assertion *Assertion, output, input interface{})
 	return result
 }
 
+// assertToolCalled checks if a specific tool was called
+// value can be:
+// - string: exact tool name to match
+// - []string: any of the tool names
+// - map with "name" and optional "arguments" for more specific matching
+func (a *Asserter) assertToolCalled(assertion *Assertion) *AssertionResult {
+	result := &AssertionResult{
+		Assertion: assertion,
+		Expected:  assertion.Value,
+	}
+
+	if a.response == nil {
+		result.Passed = false
+		result.Message = "no response available for tool_called assertion"
+		return result
+	}
+
+	if len(a.response.Tools) == 0 {
+		result.Passed = false
+		result.Message = "no tools were called"
+		return result
+	}
+
+	// Get tool names that were called
+	calledTools := make([]string, 0, len(a.response.Tools))
+	for _, tool := range a.response.Tools {
+		calledTools = append(calledTools, tool.Tool)
+	}
+	result.Actual = calledTools
+
+	switch v := assertion.Value.(type) {
+	case string:
+		// Simple case: check if tool name matches (supports prefix matching)
+		for _, tool := range a.response.Tools {
+			if matchToolName(tool.Tool, v) {
+				result.Passed = true
+				result.Message = fmt.Sprintf("tool '%s' was called", v)
+				return result
+			}
+		}
+		result.Passed = false
+		result.Message = fmt.Sprintf("tool '%s' was not called, called: %v", v, calledTools)
+
+	case []interface{}:
+		// Check if any of the specified tools were called
+		for _, expected := range v {
+			if expectedStr, ok := expected.(string); ok {
+				for _, tool := range a.response.Tools {
+					if matchToolName(tool.Tool, expectedStr) {
+						result.Passed = true
+						result.Message = fmt.Sprintf("tool '%s' was called", expectedStr)
+						return result
+					}
+				}
+			}
+		}
+		result.Passed = false
+		result.Message = fmt.Sprintf("none of the expected tools were called, called: %v", calledTools)
+
+	case map[string]interface{}:
+		// Advanced case: match name and optionally arguments
+		expectedName, _ := v["name"].(string)
+		expectedArgs := v["arguments"]
+
+		for _, tool := range a.response.Tools {
+			if matchToolName(tool.Tool, expectedName) {
+				// If arguments specified, check them too
+				if expectedArgs != nil {
+					if matchArguments(tool.Arguments, expectedArgs) {
+						result.Passed = true
+						result.Message = fmt.Sprintf("tool '%s' was called with matching arguments", expectedName)
+						return result
+					}
+				} else {
+					result.Passed = true
+					result.Message = fmt.Sprintf("tool '%s' was called", expectedName)
+					return result
+				}
+			}
+		}
+		result.Passed = false
+		if expectedArgs != nil {
+			result.Message = fmt.Sprintf("tool '%s' was not called with expected arguments", expectedName)
+		} else {
+			result.Message = fmt.Sprintf("tool '%s' was not called, called: %v", expectedName, calledTools)
+		}
+
+	default:
+		result.Passed = false
+		result.Message = fmt.Sprintf("invalid tool_called value type: %T", assertion.Value)
+	}
+
+	return result
+}
+
+// assertToolResult checks the result of a tool call
+// value should be a map with "tool" (name) and "result" (expected result pattern)
+func (a *Asserter) assertToolResult(assertion *Assertion) *AssertionResult {
+	result := &AssertionResult{
+		Assertion: assertion,
+		Expected:  assertion.Value,
+	}
+
+	if a.response == nil {
+		result.Passed = false
+		result.Message = "no response available for tool_result assertion"
+		return result
+	}
+
+	if len(a.response.Tools) == 0 {
+		result.Passed = false
+		result.Message = "no tools were called"
+		return result
+	}
+
+	spec, ok := assertion.Value.(map[string]interface{})
+	if !ok {
+		result.Passed = false
+		result.Message = "tool_result assertion requires a map with 'tool' and 'result' fields"
+		return result
+	}
+
+	toolName, _ := spec["tool"].(string)
+	expectedResult := spec["result"]
+
+	if toolName == "" {
+		result.Passed = false
+		result.Message = "tool_result assertion requires 'tool' field"
+		return result
+	}
+
+	// Find the tool call
+	for _, tool := range a.response.Tools {
+		if matchToolName(tool.Tool, toolName) {
+			result.Actual = tool.Result
+
+			// Check if there was an error
+			if tool.Error != "" {
+				result.Passed = false
+				result.Message = fmt.Sprintf("tool '%s' returned error: %s", toolName, tool.Error)
+				return result
+			}
+
+			// If no expected result specified, just check success (no error)
+			if expectedResult == nil {
+				result.Passed = true
+				result.Message = fmt.Sprintf("tool '%s' executed successfully", toolName)
+				return result
+			}
+
+			// Match result
+			if matchResult(tool.Result, expectedResult) {
+				result.Passed = true
+				result.Message = fmt.Sprintf("tool '%s' result matches expected", toolName)
+				return result
+			}
+
+			result.Passed = false
+			result.Message = fmt.Sprintf("tool '%s' result does not match expected", toolName)
+			return result
+		}
+	}
+
+	result.Passed = false
+	result.Message = fmt.Sprintf("tool '%s' was not called", toolName)
+	return result
+}
+
+// matchToolName checks if a tool name matches the expected pattern
+// Supports exact match and suffix match (e.g., "setup" matches "agents_expense_tools__setup")
+func matchToolName(actual, expected string) bool {
+	if actual == expected {
+		return true
+	}
+	// Support suffix matching (tool name without namespace prefix)
+	if strings.HasSuffix(actual, "__"+expected) || strings.HasSuffix(actual, "."+expected) {
+		return true
+	}
+	// Support contains matching for partial names
+	if strings.Contains(actual, expected) {
+		return true
+	}
+	return false
+}
+
+// matchArguments checks if tool arguments match expected pattern
+func matchArguments(actual, expected interface{}) bool {
+	expectedMap, ok := expected.(map[string]interface{})
+	if !ok {
+		return false
+	}
+
+	actualMap, ok := actual.(map[string]interface{})
+	if !ok {
+		// Try parsing as JSON string
+		if actualStr, ok := actual.(string); ok {
+			var parsed map[string]interface{}
+			if err := jsoniter.UnmarshalFromString(actualStr, &parsed); err == nil {
+				actualMap = parsed
+			} else {
+				return false
+			}
+		} else {
+			return false
+		}
+	}
+
+	// Check that all expected keys exist and match
+	for key, expectedVal := range expectedMap {
+		actualVal, exists := actualMap[key]
+		if !exists {
+			return false
+		}
+		if !validateOutput(actualVal, expectedVal) {
+			return false
+		}
+	}
+	return true
+}
+
+// matchResult checks if tool result matches expected pattern
+func matchResult(actual, expected interface{}) bool {
+	switch exp := expected.(type) {
+	case map[string]interface{}:
+		actualMap, ok := actual.(map[string]interface{})
+		if !ok {
+			return false
+		}
+		// Check that all expected keys exist and match
+		for key, expectedVal := range exp {
+			actualVal, exists := actualMap[key]
+			if !exists {
+				return false
+			}
+			if !matchResult(actualVal, expectedVal) {
+				return false
+			}
+		}
+		return true
+
+	case string:
+		// Support regex pattern matching for strings
+		if strings.HasPrefix(exp, "regex:") {
+			pattern := strings.TrimPrefix(exp, "regex:")
+			re, err := regexp.Compile(pattern)
+			if err != nil {
+				return false
+			}
+			actualStr := fmt.Sprintf("%v", actual)
+			return re.MatchString(actualStr)
+		}
+		return fmt.Sprintf("%v", actual) == exp
+
+	case bool:
+		actualBool, ok := actual.(bool)
+		return ok && actualBool == exp
+
+	default:
+		return validateOutput(actual, expected)
+	}
+}
+
 // toString converts a value to string for comparison
 func (a *Asserter) toString(v interface{}) string {
 	if v == nil {

agent/test/assert_test.go

@@ -2,6 +2,8 @@ package test
 
 import (
 	"testing"
+
+	"github.com/yaoapp/yao/agent/context"
 )
 
 func TestAsserter_JSONPath_ArrayEquality(t *testing.T) {
@@ -303,3 +305,552 @@ func TestAsserter_Regex(t *testing.T) {
 		})
 	}
 }
+
+func TestAsserter_ToolCalled(t *testing.T) {
+	tests := []struct {
+		name     string
+		tc       *Case
+		response *context.Response
+		expected bool
+		errMsg   string
+	}{
+		{
+			name: "tool called - exact match",
+			tc: &Case{
+				Assert: map[string]interface{}{
+					"type":  "tool_called",
+					"value": "agents_expense_tools__setup",
+				},
+			},
+			response: &context.Response{
+				Tools: []context.ToolCallResponse{
+					{Tool: "agents_expense_tools__setup", Result: map[string]interface{}{"success": true}},
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "tool called - suffix match",
+			tc: &Case{
+				Assert: map[string]interface{}{
+					"type":  "tool_called",
+					"value": "setup",
+				},
+			},
+			response: &context.Response{
+				Tools: []context.ToolCallResponse{
+					{Tool: "agents_expense_tools__setup", Result: map[string]interface{}{"success": true}},
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "tool not called",
+			tc: &Case{
+				Assert: map[string]interface{}{
+					"type":  "tool_called",
+					"value": "setup",
+				},
+			},
+			response: &context.Response{
+				Tools: []context.ToolCallResponse{},
+			},
+			expected: false,
+		},
+		{
+			name: "tool called - wrong tool",
+			tc: &Case{
+				Assert: map[string]interface{}{
+					"type":  "tool_called",
+					"value": "setup",
+				},
+			},
+			response: &context.Response{
+				Tools: []context.ToolCallResponse{
+					{Tool: "agents_expense_tools__submit", Result: map[string]interface{}{"success": true}},
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "tool called - any of multiple",
+			tc: &Case{
+				Assert: map[string]interface{}{
+					"type":  "tool_called",
+					"value": []interface{}{"setup", "init"},
+				},
+			},
+			response: &context.Response{
+				Tools: []context.ToolCallResponse{
+					{Tool: "agents_expense_tools__init", Result: map[string]interface{}{"success": true}},
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "tool called - with arguments (map)",
+			tc: &Case{
+				Assert: map[string]interface{}{
+					"type": "tool_called",
+					"value": map[string]interface{}{
+						"name": "setup",
+						"arguments": map[string]interface{}{
+							"action": "init",
+						},
+					},
+				},
+			},
+			response: &context.Response{
+				Tools: []context.ToolCallResponse{
+					{
+						Tool:      "agents_expense_tools__setup",
+						Arguments: map[string]interface{}{"action": "init", "config": map[string]interface{}{}},
+						Result:    map[string]interface{}{"success": true},
+					},
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "tool called - with arguments (JSON string)",
+			tc: &Case{
+				Assert: map[string]interface{}{
+					"type": "tool_called",
+					"value": map[string]interface{}{
+						"name": "setup",
+						"arguments": map[string]interface{}{
+							"action": "init",
+						},
+					},
+				},
+			},
+			response: &context.Response{
+				Tools: []context.ToolCallResponse{
+					{
+						Tool:      "agents_expense_tools__setup",
+						Arguments: `{"action":"init","config":{"default_currency":"USD"}}`,
+						Result:    map[string]interface{}{"success": true},
+					},
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "tool called - wrong arguments",
+			tc: &Case{
+				Assert: map[string]interface{}{
+					"type": "tool_called",
+					"value": map[string]interface{}{
+						"name": "setup",
+						"arguments": map[string]interface{}{
+							"action": "update",
+						},
+					},
+				},
+			},
+			response: &context.Response{
+				Tools: []context.ToolCallResponse{
+					{
+						Tool:      "agents_expense_tools__setup",
+						Arguments: map[string]interface{}{"action": "init"},
+						Result:    map[string]interface{}{"success": true},
+					},
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "no response",
+			tc: &Case{
+				Assert: map[string]interface{}{
+					"type":  "tool_called",
+					"value": "setup",
+				},
+			},
+			response: nil,
+			expected: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			asserter := NewAsserter().WithResponse(tt.response)
+			passed, errMsg := asserter.Validate(tt.tc, nil)
+			if passed != tt.expected {
+				t.Errorf("Expected passed=%v, got passed=%v, error: %s", tt.expected, passed, errMsg)
+			}
+		})
+	}
+}
+
+func TestAsserter_ToolResult(t *testing.T) {
+	tests := []struct {
+		name     string
+		tc       *Case
+		response *context.Response
+		expected bool
+	}{
+		{
+			name: "tool result - success check",
+			tc: &Case{
+				Assert: map[string]interface{}{
+					"type": "tool_result",
+					"value": map[string]interface{}{
+						"tool": "setup",
+						"result": map[string]interface{}{
+							"success": true,
+						},
+					},
+				},
+			},
+			response: &context.Response{
+				Tools: []context.ToolCallResponse{
+					{
+						Tool:   "agents_expense_tools__setup",
+						Result: map[string]interface{}{"success": true, "message": "Setup complete"},
+					},
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "tool result - message check with regex",
+			tc: &Case{
+				Assert: map[string]interface{}{
+					"type": "tool_result",
+					"value": map[string]interface{}{
+						"tool": "setup",
+						"result": map[string]interface{}{
+							"message": "regex:(?i)setup.*complete",
+						},
+					},
+				},
+			},
+			response: &context.Response{
+				Tools: []context.ToolCallResponse{
+					{
+						Tool:   "agents_expense_tools__setup",
+						Result: map[string]interface{}{"success": true, "message": "Setup complete!"},
+					},
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "tool result - no expected result (just check no error)",
+			tc: &Case{
+				Assert: map[string]interface{}{
+					"type": "tool_result",
+					"value": map[string]interface{}{
+						"tool": "setup",
+					},
+				},
+			},
+			response: &context.Response{
+				Tools: []context.ToolCallResponse{
+					{
+						Tool:   "agents_expense_tools__setup",
+						Result: map[string]interface{}{"success": true},
+					},
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "tool result - tool has error",
+			tc: &Case{
+				Assert: map[string]interface{}{
+					"type": "tool_result",
+					"value": map[string]interface{}{
+						"tool": "setup",
+					},
+				},
+			},
+			response: &context.Response{
+				Tools: []context.ToolCallResponse{
+					{
+						Tool:  "agents_expense_tools__setup",
+						Error: "permission denied",
+					},
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "tool result - result mismatch",
+			tc: &Case{
+				Assert: map[string]interface{}{
+					"type": "tool_result",
+					"value": map[string]interface{}{
+						"tool": "setup",
+						"result": map[string]interface{}{
+							"success": true,
+						},
+					},
+				},
+			},
+			response: &context.Response{
+				Tools: []context.ToolCallResponse{
+					{
+						Tool:   "agents_expense_tools__setup",
+						Result: map[string]interface{}{"success": false},
+					},
+				},
+			},
+			expected: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			asserter := NewAsserter().WithResponse(tt.response)
+			passed, errMsg := asserter.Validate(tt.tc, nil)
+			if passed != tt.expected {
+				t.Errorf("Expected passed=%v, got passed=%v, error: %s", tt.expected, passed, errMsg)
+			}
+		})
+	}
+}
+
+func TestAsserter_MultipleToolAssertions(t *testing.T) {
+	// This tests the exact scenario from setup-006: tool_called + tool_result
+	response := &context.Response{
+		Tools: []context.ToolCallResponse{
+			{
+				Tool:      "agents_expense_tools__setup",
+				Arguments: `{"action":"init","config":{"default_currency":"USD","categories":[{"id":"meals","name":"Meals","daily_limit":100}]}}`,
+				Result: map[string]interface{}{
+					"success": true,
+					"action":  "init",
+					"config": map[string]interface{}{
+						"default_currency": "USD",
+					},
+					"message": "Setup complete!",
+				},
+			},
+		},
+	}
+
+	asserter := NewAsserter().WithResponse(response)
+
+	tc := &Case{
+		Assert: []interface{}{
+			map[string]interface{}{
+				"type": "tool_called",
+				"value": map[string]interface{}{
+					"name": "setup",
+					"arguments": map[string]interface{}{
+						"action": "init",
+					},
+				},
+			},
+			map[string]interface{}{
+				"type": "tool_result",
+				"value": map[string]interface{}{
+					"tool": "setup",
+					"result": map[string]interface{}{
+						"success": true,
+					},
+				},
+			},
+		},
+	}
+
+	result := asserter.ValidateWithDetails(tc, nil)
+	if !result.Passed {
+		t.Errorf("Expected multiple tool assertions to pass, got: %s", result.Message)
+	}
+}
+
+func TestAsserter_SharedAsserterWithResponse(t *testing.T) {
+	// Test that a shared asserter correctly uses WithResponse
+	asserter := NewAsserter()
+
+	// First call without response - should fail
+	tc := &Case{
+		Assert: map[string]interface{}{
+			"type":  "tool_called",
+			"value": "setup",
+		},
+	}
+	result := asserter.ValidateWithDetails(tc, nil)
+	if result.Passed {
+		t.Error("Expected tool_called to fail without response")
+	}
+	if result.Message != "no response available for tool_called assertion" {
+		t.Errorf("Unexpected message: %s", result.Message)
+	}
+
+	// Now set response
+	response := &context.Response{
+		Tools: []context.ToolCallResponse{
+			{
+				Tool:   "agents_expense_tools__setup",
+				Result: map[string]interface{}{"success": true},
+			},
+		},
+	}
+	asserter.WithResponse(response)
+
+	// Should pass now
+	result = asserter.ValidateWithDetails(tc, nil)
+	if !result.Passed {
+		t.Errorf("Expected tool_called to pass with response, got: %s", result.Message)
+	}
+}
+
+func TestAsserter_Setup006Scenario(t *testing.T) {
+	// Exact reproduction of setup-006 scenario
+	// Turn 2: tool was called with action: init, result has success: true
+	response := &context.Response{
+		Tools: []context.ToolCallResponse{
+			{
+				Tool:      "agents_expense_tools__setup",
+				Arguments: `{"action":"init","config":{"default_currency":"USD","categories":[{"id":"meals","name":"Meals","daily_limit":100},{"id":"travel","name":"Travel","daily_limit":500}]}}`,
+				Result: map[string]interface{}{
+					"config": map[string]interface{}{
+						"categories": []interface{}{
+							map[string]interface{}{"daily_limit": float64(100), "id": "meals", "name": "Meals"},
+							map[string]interface{}{"daily_limit": float64(500), "id": "travel", "name": "Travel"},
+						},
+						"default_currency": "USD",
+					},
+					"message": "Setup complete! The expense system has been initialized successfully with the configured settings. You can now start submitting expenses.",
+					"success": true,
+					"action":  "init",
+				},
+			},
+		},
+	}
+
+	// This is the exact assert from setup-006's quick_complete checkpoint
+	assertDef := []interface{}{
+		map[string]interface{}{
+			"type": "tool_called",
+			"value": map[string]interface{}{
+				"name": "setup",
+				"arguments": map[string]interface{}{
+					"action": "init",
+				},
+			},
+		},
+		map[string]interface{}{
+			"type": "tool_result",
+			"value": map[string]interface{}{
+				"tool": "setup",
+				"result": map[string]interface{}{
+					"success": true,
+				},
+			},
+		},
+	}
+
+	asserter := NewAsserter().WithResponse(response)
+	tc := &Case{Assert: assertDef}
+
+	result := asserter.ValidateWithDetails(tc, nil)
+	if !result.Passed {
+		t.Errorf("Expected setup-006 scenario to pass, got: %s", result.Message)
+	}
+
+	// Also test individual assertions
+	t.Run("tool_called only", func(t *testing.T) {
+		tc2 := &Case{Assert: assertDef[0]}
+		result2 := asserter.ValidateWithDetails(tc2, nil)
+		if !result2.Passed {
+			t.Errorf("Expected tool_called to pass, got: %s", result2.Message)
+		}
+	})
+
+	t.Run("tool_result only", func(t *testing.T) {
+		tc3 := &Case{Assert: assertDef[1]}
+		result3 := asserter.ValidateWithDetails(tc3, nil)
+		if !result3.Passed {
+			t.Errorf("Expected tool_result to pass, got: %s", result3.Message)
+		}
+	})
+}
+
+func TestAsserter_Setup003Scenario(t *testing.T) {
+	// Exact reproduction of setup-003 scenario
+	// Turn 3: tool was called with action: update
+	response := &context.Response{
+		Tools: []context.ToolCallResponse{
+			{
+				Tool:      "agents_expense_tools__setup",
+				Arguments: `{"action":"update","config":{"categories":[{"daily_limit":500,"id":"meals","name":"Business Meals"}]}}`,
+				Result: map[string]interface{}{
+					"action": "update",
+					"config": map[string]interface{}{
+						"categories": []interface{}{
+							map[string]interface{}{"daily_limit": float64(500), "id": "meals", "name": "Business Meals"},
+						},
+					},
+					"message": "Configuration updated successfully! Your changes have been saved.",
+					"success": true,
+				},
+			},
+		},
+	}
+
+	// This is the exact assert from setup-003's update_complete checkpoint
+	assertDef := []interface{}{
+		map[string]interface{}{
+			"type": "tool_called",
+			"value": map[string]interface{}{
+				"name": "setup",
+				"arguments": map[string]interface{}{
+					"action": "update",
+				},
+			},
+		},
+		map[string]interface{}{
+			"type": "tool_result",
+			"value": map[string]interface{}{
+				"tool": "setup",
+				"result": map[string]interface{}{
+					"success": true,
+				},
+			},
+		},
+	}
+
+	asserter := NewAsserter().WithResponse(response)
+	tc := &Case{Assert: assertDef}
+
+	result := asserter.ValidateWithDetails(tc, nil)
+	if !result.Passed {
+		t.Errorf("Expected setup-003 scenario to pass, got: %s", result.Message)
+	}
+
+	// Test individual assertions
+	t.Run("tool_called with action:update", func(t *testing.T) {
+		tc2 := &Case{Assert: assertDef[0]}
+		result2 := asserter.ValidateWithDetails(tc2, nil)
+		if !result2.Passed {
+			t.Errorf("Expected tool_called to pass, got: %s", result2.Message)
+		}
+	})
+}
+
+func TestMatchToolName(t *testing.T) {
+	tests := []struct {
+		actual   string
+		expected string
+		match    bool
+	}{
+		{"agents_expense_tools__setup", "agents_expense_tools__setup", true},
+		{"agents_expense_tools__setup", "setup", true},
+		{"agents.expense.tools.setup", "setup", true},
+		{"agents_expense_tools__setup", "init", false},
+		{"setup", "setup", true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.actual+"_"+tt.expected, func(t *testing.T) {
+			if matchToolName(tt.actual, tt.expected) != tt.match {
+				t.Errorf("matchToolName(%q, %q) = %v, want %v", tt.actual, tt.expected, !tt.match, tt.match)
+			}
+		})
+	}
+}

agent/test/dynamic_runner.go

@@ -77,7 +77,14 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s
 
 	// Output dynamic test start
 	if r.opts.Verbose {
-		r.output.Info("Dynamic test: %s (max %d turns)", tc.ID, maxTurns)
+		r.output.Verbose("Dynamic test: %s (max %d turns)", tc.ID, maxTurns)
+	}
+
+	// Use consistent chatID across all turns to preserve session state (ctx.memory.chat)
+	// Priority: context config > generated ID
+	chatID := fmt.Sprintf("dynamic-%s", tc.ID)
+	if r.opts.ContextData != nil && r.opts.ContextData.ChatID != "" {
+		chatID = r.opts.ContextData.ChatID
 	}
 
 	// Conversation loop
@@ -111,7 +118,7 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s
 			// Check if goal achieved
 			if simOutput.GoalAchieved {
 				if r.opts.Verbose {
-					r.output.Info("  Turn %d: Simulator signaled goal achieved", turn)
+					r.output.Verbose("Turn %d: Simulator signaled goal achieved", turn)
 				}
 
 				// Check if all required checkpoints reached
@@ -135,7 +142,7 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s
 			turnResult.Input = simOutput.Message
 
 			if r.opts.Verbose {
-				r.output.Info("  Turn %d: User: %s", turn, truncateOutput(simOutput.Message, 50))
+				r.output.Verbose("Turn %d: User: %s", turn, truncateOutput(simOutput.Message, 50))
 			}
 		} else {
 			// Use initial input for first turn
@@ -143,14 +150,15 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s
 				lastMsg := messages[len(messages)-1]
 				turnResult.Input = lastMsg.Content
 				if r.opts.Verbose {
-					r.output.Info("  Turn %d: User: %s", turn, truncateOutput(lastMsg.Content, 50))
+					r.output.Verbose("Turn %d: User: %s", turn, truncateOutput(lastMsg.Content, 50))
 				}
 			}
 		}
 
 		// Call target agent
+		// Use consistent chatID across all turns to preserve session state (ctx.memory.chat)
 		ctx := NewTestContextFromOptions(
-			fmt.Sprintf("dynamic-%s-%d", tc.ID, turn),
+			chatID,
 			agentID,
 			r.opts,
 			tc,
@@ -171,28 +179,28 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s
 			return result
 		}
 
-		// Extract output
+		// Extract output (summary for display and conversation history)
 		output := extractOutput(response)
 		turnResult.Output = output
 		turnResult.DurationMs = time.Since(turnStart).Milliseconds()
 
+		// Store full response for reporting
+		turnResult.Response = buildTurnResponse(response)
+
 		if r.opts.Verbose {
-			r.output.Info("  Turn %d: Agent: %s", turn, truncateOutput(output, 50))
+			r.output.Verbose("Turn %d: Agent: %s", turn, truncateOutput(output, 50))
 		}
 
-		// Add assistant response to messages
+		// Add assistant response to messages, including tool calls if any
-		messages = append(messages, context.Message{
+		messages = appendAssistantMessages(messages, response)
-			Role:    context.RoleAssistant,
-			Content: output,
-		})
 
-		// Check checkpoints against this response
+		// Check checkpoints against this response (including tool results)
-		reachedIDs := r.checkCheckpoints(tc.Checkpoints, output, result)
+		reachedIDs := r.checkCheckpoints(tc.Checkpoints, response, result)
 		turnResult.CheckpointsReached = reachedIDs
 
 		if r.opts.Verbose && len(reachedIDs) > 0 {
 			for _, id := range reachedIDs {
-				r.output.Info("    ✓ checkpoint: %s", id)
+				r.output.Verbose("  ✓ checkpoint: %s", id)
 			}
 		}
 
@@ -360,10 +368,18 @@ func (r *DynamicRunner) parseSimulatorResponse(response *context.Response) (*Sim
 	return output, nil
 }
 
-// checkCheckpoints validates checkpoints against current output
+// checkCheckpoints validates checkpoints against current response
-func (r *DynamicRunner) checkCheckpoints(checkpoints []*Checkpoint, output interface{}, result *DynamicResult) []string {
+// It checks both the completion content and tool results for comprehensive validation
+func (r *DynamicRunner) checkCheckpoints(checkpoints []*Checkpoint, response *context.Response, result *DynamicResult) []string {
 	reachedIDs := make([]string, 0)
 
+	// Build combined output for checkpoint validation
+	// This includes both content and tool result messages
+	combinedOutput := buildCombinedOutput(response)
+
+	// Set response on asserter for tool-related assertions
+	r.asserter.WithResponse(response)
+
 	for _, cp := range checkpoints {
 		cpResult := result.Checkpoints[cp.ID]
 		if cpResult.Reached {
@@ -386,22 +402,166 @@ func (r *DynamicRunner) checkCheckpoints(checkpoints []*Checkpoint, output inter
 			}
 		}
 
-		// Validate using asserter
+		// Validate using asserter against combined output with full details
 		tempCase := &Case{Assert: cp.Assert}
-		passed, msg := r.asserter.Validate(tempCase, output)
+		assertResult := r.asserter.ValidateWithDetails(tempCase, combinedOutput)
 
-		if passed {
+		if assertResult.Passed {
 			cpResult.Reached = true
 			cpResult.Passed = true
 			cpResult.ReachedAtTurn = len(result.Turns) + 1
-			cpResult.Message = msg
+			cpResult.Message = assertResult.Message
 			reachedIDs = append(reachedIDs, cp.ID)
+		} else {
+			// Store failure message for debugging (but don't mark as failed yet - it might pass in a later turn)
+			if cpResult.Message == "" {
+				cpResult.Message = assertResult.Message
+			}
+		}
+
+		// Store agent validation details if this is an agent assertion
+		if isAgentAssertion(cp.Assert) {
+			// Extract criteria from assertion value
+			var criteria string
+			if assertMap, ok := cp.Assert.(map[string]interface{}); ok {
+				if c, ok := assertMap["value"].(string); ok {
+					criteria = c
+				}
+			}
+
+			cpResult.AgentValidation = &AgentValidationResult{
+				Passed:   assertResult.Passed,
+				Criteria: criteria,
+				Input:    combinedOutput, // Content sent to validator for checking
+			}
+
+			// Extract reason and store full response from validator
+			if assertResult.Expected != nil {
+				if validatorResponse, ok := assertResult.Expected.(map[string]interface{}); ok {
+					if reason, ok := validatorResponse["reason"].(string); ok {
+						cpResult.AgentValidation.Reason = reason
+					}
+					// Store the full validator response
+					cpResult.AgentValidation.Response = validatorResponse
+				}
+			}
 		}
 	}
 
 	return reachedIDs
 }
 
+// isAgentAssertion checks if the assertion is an agent-based assertion
+func isAgentAssertion(assert interface{}) bool {
+	if assertMap, ok := assert.(map[string]interface{}); ok {
+		if assertType, ok := assertMap["type"].(string); ok {
+			return assertType == "agent"
+		}
+	}
+	return false
+}
+
+// truncateForReport truncates content for report output
+func truncateForReport(content interface{}, maxLen int) interface{} {
+	if content == nil {
+		return nil
+	}
+
+	str, ok := content.(string)
+	if !ok {
+		return content
+	}
+
+	if len(str) <= maxLen {
+		return str
+	}
+
+	return str[:maxLen] + "... (truncated)"
+}
+
+// buildCombinedOutput builds a combined output string from response
+// that includes both completion content and tool result messages
+func buildCombinedOutput(response *context.Response) string {
+	if response == nil {
+		return ""
+	}
+
+	var parts []string
+
+	// Add completion content
+	if response.Completion != nil && response.Completion.Content != nil {
+		if content := extractContentString(response.Completion.Content); content != "" {
+			parts = append(parts, content)
+		}
+	}
+
+	// Add tool result messages
+	if len(response.Tools) > 0 {
+		for _, tool := range response.Tools {
+			if tool.Result != nil {
+				// Try to extract message from result
+				if resultMap, ok := tool.Result.(map[string]interface{}); ok {
+					if msg, exists := resultMap["message"]; exists && msg != nil {
+						if msgStr, ok := msg.(string); ok && msgStr != "" {
+							parts = append(parts, msgStr)
+						}
+					}
+				}
+			}
+		}
+	}
+
+	// Join all parts with newline for comprehensive matching
+	return joinNonEmpty(parts, "\n")
+}
+
+// extractContentString extracts string content from various types
+func extractContentString(content interface{}) string {
+	if content == nil {
+		return ""
+	}
+
+	switch v := content.(type) {
+	case string:
+		return v
+	case []interface{}:
+		// Handle array content (e.g., multimodal content)
+		var texts []string
+		for _, item := range v {
+			if m, ok := item.(map[string]interface{}); ok {
+				if text, ok := m["text"].(string); ok {
+					texts = append(texts, text)
+				}
+			}
+		}
+		return joinNonEmpty(texts, "\n")
+	default:
+		// Try to marshal to string
+		if data, err := jsoniter.MarshalToString(content); err == nil {
+			return data
+		}
+		return ""
+	}
+}
+
+// joinNonEmpty joins non-empty strings with separator
+func joinNonEmpty(parts []string, sep string) string {
+	var nonEmpty []string
+	for _, p := range parts {
+		if p != "" {
+			nonEmpty = append(nonEmpty, p)
+		}
+	}
+	if len(nonEmpty) == 0 {
+		return ""
+	}
+	result := nonEmpty[0]
+	for i := 1; i < len(nonEmpty); i++ {
+		result += sep + nonEmpty[i]
+	}
+	return result
+}
+
 // allRequiredCheckpointsReached checks if all required checkpoints are reached
 func (r *DynamicRunner) allRequiredCheckpointsReached(result *DynamicResult) bool {
 	for _, cp := range result.Checkpoints {
@@ -411,3 +571,114 @@ func (r *DynamicRunner) allRequiredCheckpointsReached(result *DynamicResult) boo
 	}
 	return true
 }
+
+// buildTurnResponse builds a TurnResponse from the agent response
+func buildTurnResponse(response *context.Response) *TurnResponse {
+	if response == nil {
+		return nil
+	}
+
+	tr := &TurnResponse{}
+
+	// Extract completion content
+	if response.Completion != nil {
+		tr.Content = response.Completion.Content
+
+		// Extract tool calls from completion
+		if len(response.Completion.ToolCalls) > 0 {
+			for _, tc := range response.Completion.ToolCalls {
+				tr.ToolCalls = append(tr.ToolCalls, ToolCallInfo{
+					Tool:      tc.Function.Name,
+					Arguments: tc.Function.Arguments,
+				})
+			}
+		}
+	}
+
+	// Add tool results
+	if len(response.Tools) > 0 {
+		// If we already have tool calls from completion, match results
+		if len(tr.ToolCalls) > 0 {
+			for i, toolResult := range response.Tools {
+				if i < len(tr.ToolCalls) {
+					tr.ToolCalls[i].Result = toolResult.Result
+				}
+			}
+		} else {
+			// Create tool call entries from results
+			for _, toolResult := range response.Tools {
+				tr.ToolCalls = append(tr.ToolCalls, ToolCallInfo{
+					Tool:      toolResult.Tool,
+					Arguments: toolResult.Arguments,
+					Result:    toolResult.Result,
+				})
+			}
+		}
+	}
+
+	// Extract Next hook data
+	if response.Next != nil && !isEmptyValue(response.Next) {
+		tr.Next = response.Next
+	}
+
+	return tr
+}
+
+// appendAssistantMessages appends assistant messages to the conversation history
+// including tool calls and tool results if present
+func appendAssistantMessages(messages []context.Message, response *context.Response) []context.Message {
+	if response == nil {
+		return messages
+	}
+
+	// Check if there are tool calls in the completion
+	hasToolCalls := response.Completion != nil && len(response.Completion.ToolCalls) > 0
+
+	if hasToolCalls {
+		// Add assistant message with tool calls
+		assistantMsg := context.Message{
+			Role:      context.RoleAssistant,
+			ToolCalls: response.Completion.ToolCalls,
+		}
+		// Include content if present
+		if response.Completion.Content != nil && !isEmptyValue(response.Completion.Content) {
+			assistantMsg.Content = response.Completion.Content
+		}
+		messages = append(messages, assistantMsg)
+
+		// Add tool result messages for each tool call
+		for i, tc := range response.Completion.ToolCalls {
+			toolCallID := tc.ID
+			var resultContent string
+
+			// Get result from response.Tools if available
+			if i < len(response.Tools) {
+				resultJSON, err := jsoniter.MarshalToString(response.Tools[i].Result)
+				if err == nil {
+					resultContent = resultJSON
+				} else {
+					resultContent = fmt.Sprintf("%v", response.Tools[i].Result)
+				}
+			} else {
+				resultContent = "{}"
+			}
+
+			messages = append(messages, context.Message{
+				Role:       context.RoleTool,
+				ToolCallID: &toolCallID,
+				Content:    resultContent,
+			})
+		}
+	} else {
+		// No tool calls, just add content if present
+		content := extractOutput(response)
+		if content != nil && !isEmptyValue(content) {
+			messages = append(messages, context.Message{
+				Role:    context.RoleAssistant,
+				Content: content,
+			})
+		}
+	}
+
+	return messages
+}

agent/test/dynamic_types.go

@@ -34,9 +34,12 @@ type TurnResult struct {
 	// Input is the user message (from simulator or initial input)
 	Input interface{} `json:"input"`
 
-	// Output is the agent's response
+	// Output is the agent's response (summary for display and conversation history)
 	Output interface{} `json:"output,omitempty"`
 
+	// Response is the full agent response including completion and tool results
+	Response *TurnResponse `json:"response,omitempty"`
+
 	// CheckpointsReached lists checkpoint IDs reached in this turn
 	CheckpointsReached []string `json:"checkpoints_reached,omitempty"`
 
@@ -47,6 +50,30 @@ type TurnResult struct {
 	Error string `json:"error,omitempty"`
 }
 
+// TurnResponse contains the full agent response for a turn
+type TurnResponse struct {
+	// Content is the text content from LLM completion
+	Content interface{} `json:"content,omitempty"`
+
+	// ToolCalls contains the tool calls made by the agent
+	ToolCalls []ToolCallInfo `json:"tool_calls,omitempty"`
+
+	// Next is the data returned from Next hook
+	Next interface{} `json:"next,omitempty"`
+}
+
+// ToolCallInfo contains information about a tool call
+type ToolCallInfo struct {
+	// Tool is the tool name
+	Tool string `json:"tool"`
+
+	// Arguments are the tool call arguments
+	Arguments interface{} `json:"arguments,omitempty"`
+
+	// Result is the tool execution result
+	Result interface{} `json:"result,omitempty"`
+}
+
 // CheckpointResult represents the result of a checkpoint validation
 type CheckpointResult struct {
 	// ID is the checkpoint identifier
@@ -66,6 +93,27 @@ type CheckpointResult struct {
 
 	// Message contains assertion result message
 	Message string `json:"message,omitempty"`
+
+	// AgentValidation contains the agent validator's response (for agent assertions)
+	AgentValidation *AgentValidationResult `json:"agent_validation,omitempty"`
+}
+
+// AgentValidationResult contains the result from an agent-based assertion
+type AgentValidationResult struct {
+	// Passed indicates if the agent validator determined the assertion passed
+	Passed bool `json:"passed"`
+
+	// Reason is the explanation from the agent validator
+	Reason string `json:"reason,omitempty"`
+
+	// Criteria is the validation criteria that was checked
+	Criteria string `json:"criteria,omitempty"`
+
+	// Input is the content that was sent to validator for checking
+	Input interface{} `json:"input,omitempty"`
+
+	// Response is the raw response from the validator agent
+	Response interface{} `json:"response,omitempty"`
 }
 
 // SimulatorInput is the input sent to the simulator agent

agent/test/loader.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"fmt"
 	"os"
+	"regexp"
 	"strings"
 	"time"
 
@@ -145,6 +146,13 @@ func FilterByIDs(cases []*Case, ids []string) []*Case {
 	})
 }
 
+// FilterByPattern returns test cases whose ID matches the given regex pattern
+func FilterByPattern(cases []*Case, pattern *regexp.Regexp) []*Case {
+	return FilterTestCases(cases, func(tc *Case) bool {
+		return pattern.MatchString(tc.ID)
+	})
+}
+
 // LoadFromAgent generates test cases using a generator agent
 func (l *JSONLLoader) LoadFromAgent(agentID string, targetInfo *TargetAgentInfo, params map[string]interface{}) ([]*Case, error) {
 	return GenerateTestCases(agentID, targetInfo, params)

agent/test/runner.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -232,6 +233,32 @@ func (r *Executor) RunTests() (*Report, error) {
 		r.output.Warning("Skipped: %d test cases", skippedCount)
 	}
 
+	// Filter by --run pattern if specified
+	if r.opts.Run != "" {
+		runPattern, err := regexp.Compile(r.opts.Run)
+		if err != nil {
+			return nil, fmt.Errorf("invalid --run pattern %q: %w", r.opts.Run, err)
+		}
+		activeTests = FilterByPattern(activeTests, runPattern)
+		if len(activeTests) == 0 {
+			return nil, fmt.Errorf("no test cases match pattern %q", r.opts.Run)
+		}
+		r.output.Info("Filter: %q (%d test cases match)", r.opts.Run, len(activeTests))
+	}
+
+	// Load context config if specified
+	if r.opts.ContextFile != "" {
+		ctxConfig, err := LoadContextConfig(r.opts.ContextFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to load context file: %w", err)
+		}
+		r.opts.ContextData = ctxConfig
+		r.output.Info("Context: %s", r.opts.ContextFile)
+	}
+
+	// Set options on hook executor (for context data access in hooks)
+	r.hookExecutor.SetOptions(r.opts)
+
 	// Print test info
 	if r.opts.Runs > 1 {
 		r.output.Info("Runs: %d per test case (stability analysis)", r.opts.Runs)
@@ -526,14 +553,12 @@ func (r *Executor) runDynamicTest(ast *assistant.Assistant, tc *Case, agentID st
 	// Convert to standard result
 	result := dynamicResult.ToResult()
 
-	// Execute after script if specified
+	// Execute after script if specified (before outputting result)
-	defer func() {
+	if tc.After != "" && (tc.Before == "" || beforeData != nil || result.Status != StatusError || !isBeforeError(result.Error)) {
-		if tc.After != "" && (tc.Before == "" || beforeData != nil || result.Status != StatusError || !isBeforeError(result.Error)) {
+		if err := r.hookExecutor.ExecuteAfter(tc.After, tc, result, beforeData, r.agentPath); err != nil {
-			if err := r.hookExecutor.ExecuteAfter(tc.After, tc, result, beforeData, r.agentPath); err != nil {
+			r.output.Warning("after script failed: %s", err.Error())
-				r.output.Warning("after script failed: %s", err.Error())
-			}
 		}
-	}()
+	}
 
 	// Output result
 	duration := time.Duration(result.DurationMs) * time.Millisecond
@@ -718,7 +743,7 @@ func buildContextOptions(tc *Case, runnerOpts *Options) *context.Options {
 }
 
 // extractOutput extracts the output from the agent response
-// Priority: Next hook data (if non-empty) > Completion content > nil
+// Priority: Next hook data > Completion content > Tool results message > nil
 func extractOutput(response *context.Response) interface{} {
 	if response == nil {
 		return nil
@@ -729,11 +754,76 @@ func extractOutput(response *context.Response) interface{} {
 	if response.Next != nil && !isEmptyValue(response.Next) {
 		return response.Next
 	}
-	// Fall back to raw completion content
+
+	// Fall back to completion response
 	if response.Completion != nil {
-		return response.Completion.Content
+		// If content is non-empty, return it
+		if response.Completion.Content != nil && !isEmptyValue(response.Completion.Content) {
+			return response.Completion.Content
+		}
 	}
 
+	// If no content but tools were executed, extract message from tool results
+	// This handles the case where LLM calls tools but doesn't generate text
+	if len(response.Tools) > 0 {
+		return extractToolResultMessage(response.Tools)
+	}
+
+	return nil
+}
+
+// extractToolResultMessage extracts the message field from tool results
+// Returns the first non-empty message found, or a summary of tool calls
+func extractToolResultMessage(tools []context.ToolCallResponse) interface{} {
+	if len(tools) == 0 {
+		return nil
+	}
+
+	// Try to extract "message" field from tool results first
+	for _, tool := range tools {
+		if tool.Result != nil {
+			// Try to get message from result map
+			if resultMap, ok := tool.Result.(map[string]interface{}); ok {
+				if msg, exists := resultMap["message"]; exists && msg != nil {
+					if msgStr, ok := msg.(string); ok && msgStr != "" {
+						return msgStr
+					}
+				}
+			}
+		}
+	}
+
+	// No message found, generate a summary of tool calls
+	var summaries []string
+	for _, tool := range tools {
+		toolName := tool.Tool
+		if toolName == "" {
+			toolName = "unknown"
+		}
+		// Extract key info from result if possible
+		if tool.Result != nil {
+			if resultMap, ok := tool.Result.(map[string]interface{}); ok {
+				// Try common result fields
+				if action, ok := resultMap["action"].(string); ok {
+					summaries = append(summaries, fmt.Sprintf("[%s: %s]", toolName, action))
+					continue
+				}
+				if success, ok := resultMap["success"].(bool); ok {
+					status := "failed"
+					if success {
+						status = "success"
+					}
+					summaries = append(summaries, fmt.Sprintf("[%s: %s]", toolName, status))
+					continue
+				}
+			}
+		}
+		summaries = append(summaries, fmt.Sprintf("[%s]", toolName))
+	}
+
+	if len(summaries) > 0 {
+		return strings.Join(summaries, " ")
+	}
 	return nil
 }
 

agent/test/script_hooks.go

@@ -19,6 +19,7 @@ type HookExecutor struct {
 	output       *OutputWriter
 	loadedDirs   map[string]bool // Track which directories have been loaded
 	agentContext *context.Context
+	opts         *Options // Test options (includes ContextData from --ctx)
 }
 
 // NewHookExecutor creates a new hook executor
@@ -35,6 +36,11 @@ func (h *HookExecutor) SetAgentContext(ctx *context.Context) {
 	h.agentContext = ctx
 }
 
+// SetOptions sets the test options for hook execution
+func (h *HookExecutor) SetOptions(opts *Options) {
+	h.opts = opts
+}
+
 // HookRef represents a parsed hook reference
 // Format: "src/env_test.ts:Before" or just "Before" (uses default test file)
 type HookRef struct {
@@ -86,13 +92,23 @@ func ParseHookRef(ref string) (*HookRef, error) {
 func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) {
 	srcDir := filepath.Join(agentPath, "src")
 
-	// Check if already loaded
+	// Convert to relative path for application.App
+	// application.App expects paths relative to YAO_ROOT
+	relSrcDir := srcDir
+	if application.App != nil {
+		if rel, err := filepath.Rel(application.App.Root(), srcDir); err == nil {
+			relSrcDir = rel
+		}
+	}
+
+	// Check if already loaded (use absolute path as key)
+	// No logging for already loaded - this is normal and happens frequently
 	if h.loadedDirs[srcDir] {
 		return nil, nil
 	}
 
 	// Check if src directory exists
-	exists, err := application.App.Exists(srcDir)
+	exists, err := application.App.Exists(relSrcDir)
 	if err != nil {
 		return nil, err
 	}
@@ -103,7 +119,7 @@ func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) {
 	var loadedScripts []string
 	exts := []string{"*_test.ts", "*_test.js"}
 
-	err = application.App.Walk(srcDir, func(root, file string, isdir bool) error {
+	err = application.App.Walk(relSrcDir, func(root, file string, isdir bool) error {
 		if isdir {
 			return nil
 		}
@@ -114,10 +130,10 @@ func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) {
 			return nil
 		}
 
-		// Generate script ID
+		// Generate script ID (use relative path for consistency)
-		scriptID := generateHookScriptID(file, srcDir)
+		scriptID := generateHookScriptID(file, relSrcDir)
 
-		// Load the script
+		// Load the script (file path from Walk is relative to App root)
 		_, err := v8.Load(file, scriptID)
 		if err != nil {
 			if h.verbose {
@@ -127,10 +143,6 @@ func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) {
 		}
 
 		loadedScripts = append(loadedScripts, scriptID)
-		if h.verbose {
-			h.output.Verbose("Loaded hook script: %s (id: %s)", base, scriptID)
-		}
-
 		return nil
 	}, exts...)
 
@@ -139,6 +151,12 @@ func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) {
 	}
 
 	h.loadedDirs[srcDir] = true
+
+	// Log summary only once when scripts are first loaded
+	if h.verbose && len(loadedScripts) > 0 {
+		h.output.Verbose("Loaded %d hook scripts from %s", len(loadedScripts), relSrcDir)
+	}
+
 	return loadedScripts, nil
 }
 
@@ -392,13 +410,19 @@ func (h *HookExecutor) executeHookFunctionWithCases(script *v8.Script, funcName
 		return nil, fmt.Errorf("failed to convert to function: %w", err)
 	}
 
+	// Build ctx argument
+	ctxJS, err := h.buildCtxArg(v8ctx)
+	if err != nil {
+		return nil, err
+	}
+
 	// Convert test cases to JS array
 	casesJS, err := h.testCasesToJS(v8ctx, testCases)
 	if err != nil {
 		return nil, err
 	}
 
-	jsResult, err := fn.Call(global, casesJS)
+	jsResult, err := fn.Call(global, ctxJS, casesJS)
 	if err != nil {
 		return nil, fmt.Errorf("hook function %s failed: %w", funcName, err)
 	}
@@ -454,6 +478,12 @@ func (h *HookExecutor) executeHookFunctionWithResults(script *v8.Script, funcNam
 		return nil, fmt.Errorf("failed to convert to function: %w", err)
 	}
 
+	// Build ctx argument
+	ctxJS, err := h.buildCtxArg(v8ctx)
+	if err != nil {
+		return nil, err
+	}
+
 	// Convert results to JS array
 	resultsJS, err := h.resultsToJS(v8ctx, results)
 	if err != nil {
@@ -466,7 +496,7 @@ func (h *HookExecutor) executeHookFunctionWithResults(script *v8.Script, funcNam
 		return nil, fmt.Errorf("failed to convert beforeData: %w", err)
 	}
 
-	jsResult, err := fn.Call(global, resultsJS, beforeDataJS)
+	jsResult, err := fn.Call(global, ctxJS, resultsJS, beforeDataJS)
 	if err != nil {
 		return nil, fmt.Errorf("hook function %s failed: %w", funcName, err)
 	}
@@ -498,11 +528,105 @@ func (h *HookExecutor) setShareData(v8ctx *v8go.Context) error {
 	})
 }
 
+// buildCtxArg builds the context argument for hook functions
+func (h *HookExecutor) buildCtxArg(v8ctx *v8go.Context) (*v8go.Value, error) {
+	ctxMap := map[string]interface{}{
+		"locale": "en",
+	}
+
+	// Use ContextData from --ctx flag if available
+	if h.opts != nil && h.opts.ContextData != nil {
+		cfg := h.opts.ContextData
+		if cfg.Locale != "" {
+			ctxMap["locale"] = cfg.Locale
+		}
+		if cfg.Authorized != nil {
+			authorized := map[string]interface{}{}
+			if cfg.Authorized.UserID != "" {
+				authorized["user_id"] = cfg.Authorized.UserID
+			}
+			if cfg.Authorized.TeamID != "" {
+				authorized["team_id"] = cfg.Authorized.TeamID
+			}
+			if cfg.Authorized.TenantID != "" {
+				authorized["tenant_id"] = cfg.Authorized.TenantID
+			}
+			if cfg.Authorized.Sub != "" {
+				authorized["sub"] = cfg.Authorized.Sub
+			}
+			ctxMap["authorized"] = authorized
+		}
+		if cfg.Metadata != nil {
+			ctxMap["metadata"] = cfg.Metadata
+		}
+	}
+
+	return bridge.JsValue(v8ctx, ctxMap)
+}
+
 // buildHookArgs builds the arguments for a hook function call
+// Arguments order: ctx, testCase, result (for After), beforeData (for After)
 func (h *HookExecutor) buildHookArgs(v8ctx *v8go.Context, testCase *Case, result *Result, beforeData interface{}) ([]*v8go.Value, error) {
 	var args []*v8go.Value
 
-	// Arg 1: testCase
+	// Arg 1: ctx (context) - build from opts.ContextData if available
+	ctxMap := map[string]interface{}{
+		"locale": "en",
+	}
+
+	// Use ContextData from --ctx flag if available
+	if h.opts != nil && h.opts.ContextData != nil {
+		cfg := h.opts.ContextData
+		if cfg.Locale != "" {
+			ctxMap["locale"] = cfg.Locale
+		}
+		if cfg.Authorized != nil {
+			authorized := map[string]interface{}{}
+			if cfg.Authorized.UserID != "" {
+				authorized["user_id"] = cfg.Authorized.UserID
+			}
+			if cfg.Authorized.TeamID != "" {
+				authorized["team_id"] = cfg.Authorized.TeamID
+			}
+			if cfg.Authorized.TenantID != "" {
+				authorized["tenant_id"] = cfg.Authorized.TenantID
+			}
+			if cfg.Authorized.Sub != "" {
+				authorized["sub"] = cfg.Authorized.Sub
+			}
+			ctxMap["authorized"] = authorized
+		}
+		if cfg.Metadata != nil {
+			ctxMap["metadata"] = cfg.Metadata
+		}
+	} else if testCase != nil {
+		// Fallback to test case fields
+		if testCase.UserID != "" {
+			ctxMap["user_id"] = testCase.UserID
+		}
+		if testCase.TeamID != "" {
+			ctxMap["team_id"] = testCase.TeamID
+		}
+		// Build authorized info
+		authorized := map[string]interface{}{}
+		if testCase.UserID != "" {
+			authorized["user_id"] = testCase.UserID
+		}
+		if testCase.TeamID != "" {
+			authorized["team_id"] = testCase.TeamID
+		}
+		if len(authorized) > 0 {
+			ctxMap["authorized"] = authorized
+		}
+	}
+
+	ctxJS, err := bridge.JsValue(v8ctx, ctxMap)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert ctx: %w", err)
+	}
+	args = append(args, ctxJS)
+
+	// Arg 2: testCase
 	if testCase != nil {
 		tcMap := map[string]interface{}{
 			"id":    testCase.ID,
@@ -514,12 +638,20 @@ func (h *HookExecutor) buildHookArgs(v8ctx *v8go.Context, testCase *Case, result
 		if testCase.Assert != nil {
 			tcMap["assert"] = testCase.Assert
 		}
+		// Include simulator options for dynamic tests
+		if testCase.Simulator != nil {
+			tcMap["simulator"] = testCase.Simulator
+		}
 
 		tcJS, err := bridge.JsValue(v8ctx, tcMap)
 		if err != nil {
 			return nil, fmt.Errorf("failed to convert testCase: %w", err)
 		}
 		args = append(args, tcJS)
+	} else {
+		// Pass empty object if no testCase
+		emptyJS, _ := bridge.JsValue(v8ctx, map[string]interface{}{})
+		args = append(args, emptyJS)
 	}
 
 	// Arg 2: result (for After)

agent/test/types.go

@@ -165,6 +165,10 @@ type Options struct {
 // ContextConfig represents custom context configuration from JSON file
 // This allows full customization of the test context including authorized info
 type ContextConfig struct {
+	// ChatID is the chat session identifier
+	// Used to maintain session state across turns in dynamic tests
+	ChatID string `json:"chat_id,omitempty"`
+
 	// Authorized contains custom authorization data
 	Authorized *AuthorizedConfig `json:"authorized,omitempty"`
 
@@ -554,9 +558,15 @@ type AssertionResult struct {
 }
 
 // GetEnvironment returns the effective test environment for this test case
-// Priority: command line flags > test case fields > defaults
+// Priority: command line flags > context config > test case fields > defaults
 func (tc *Case) GetEnvironment(opts *Options) *Environment {
-	env := NewEnvironment("", "")
+	// Start with context config if available, otherwise use defaults
+	var env *Environment
+	if opts != nil && opts.ContextData != nil {
+		env = NewEnvironmentWithContext("", "", opts.ContextData)
+	} else {
+		env = NewEnvironment("", "")
+	}
 
 	// Apply test case specific values
 	if tc.UserID != "" {