Implement Agent-Driven Assertions in Test Framework

- Added support for agent-driven assertions in the Asserter, allowing validation of responses using specified agents.
- Introduced the `Use` and `Options` fields in the Assertion struct to facilitate agent configuration.
- Enhanced the `evaluateAssertion` method to handle assertions of type "agent".
- Implemented the `assertAgent` method to manage agent interactions and validation logic.
- Updated `script_assert.go` to include the `assertAgentMethod` for JavaScript API integration.
- Revised documentation in DESIGN_V2.md and TODO_V2.md to reflect the new agent-driven assertion capabilities and implementation status.

agent/test/DESIGN_V2.md

@@ -991,12 +991,13 @@ Existing single-turn tests work unchanged:
 ## Current Implementation Status
 
 | Feature                 | Status     | Notes                                              |
-| ----------------------- | ---------- | ---------------------------------------- |
+| ----------------------- | ---------- | -------------------------------------------------- |
 | Simple text input       | ✅ Done    | `input: "Hello"`                                   |
 | Message history         | ✅ Done    | `input: [{role, content}, ...]`                    |
 | File attachments        | ✅ Done    | `file://` protocol in content parts                |
 | Static assertions       | ✅ Done    | contains, equals, regex, json_path, etc.           |
-| Agent-driven assertions | 🔲 Planned | `type: "agent"` with validator agent     |
+| Before/After hooks      | ✅ Done    | `before/after` in JSONL, `--before/--after` in CLI |
+| Agent-driven assertions | ✅ Done    | `type: "agent"` + `t.assert.Agent()` JSAPI         |
 | Dynamic mode            | 🔲 Planned | Simulator + Checkpoints                            |
 | Agent-driven input      | 🔲 Planned | `-i agents:xxx` for test generation                |
 

agent/test/TODO_V2.md

@@ -29,15 +29,17 @@
 - [x] 创建示例脚本 `assistants/tests/hooks-test/src/env_test.ts`
 - [x] 创建单元测试 `script_hooks_test.go` (黑盒测试)
 
-## Phase 2: Agent-Driven Assertions
+## Phase 2: Agent-Driven Assertions ✅
 
 **修改文件**: `assert.go`, `script_assert.go`
 
-- [ ] `types.go`: 添加 `Use`, `Options` 字段到 `Assertion`
+- [x] `types.go`: 添加 `Use`, `Options` 字段到 `Assertion`
-- [ ] `assert.go`: 实现 `assertAgent` 方法
+- [x] `assert.go`: 实现 `assertAgent` 方法
-- [ ] `assert.go`: 在 `evaluateAssertion` 添加 `agent` 类型
+- [x] `assert.go`: 在 `evaluateAssertion` 添加 `agent` 类型
-- [ ] `script_assert.go`: 添加 `AssertAgent` 方法到 `TestingT`
+- [x] `assert.go`: 使用 `goutext.ExtractJSON` 容错解析 LLM 响应
-- [ ] 创建示例 validator agent
+- [x] `script_assert.go`: 添加 `assertAgentMethod` 到 `newAssertObject`
+- [x] 创建示例 validator agent (`assistants/tests/validator-agent`)
+- [x] 创建单元测试 `assert_agent_test.go` (JSONL 断言 + JSAPI 断言)
 
 ## Phase 3: Dynamic Mode (Simulator + Checkpoints)
 
@@ -82,6 +84,7 @@
 - [x] `-v` verbose mode
 - [x] Script testing (`*_test.ts`)
 - [x] Before/After hooks (Phase 1)
+- [x] Agent-driven assertions (Phase 2)
 
 ## Open Questions
 

agent/test/assert.go

@@ -9,6 +9,9 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/yaoapp/gou/process"
+	goutext "github.com/yaoapp/gou/text"
+	"github.com/yaoapp/yao/agent/assistant"
+	"github.com/yaoapp/yao/agent/context"
 )
 
 // Asserter handles test assertions
@@ -115,6 +118,9 @@ func (a *Asserter) mapToAssertion(m map[string]interface{}) *Assertion {
 	if s, ok := m["script"].(string); ok {
 		assertion.Script = s
 	}
+	if u, ok := m["use"].(string); ok {
+		assertion.Use = u
+	}
 	if msg, ok := m["message"].(string); ok {
 		assertion.Message = msg
 	}
@@ -122,6 +128,17 @@ func (a *Asserter) mapToAssertion(m map[string]interface{}) *Assertion {
 		assertion.Negate = n
 	}
 
+	// Parse options for agent assertions
+	if opts, ok := m["options"].(map[string]interface{}); ok {
+		assertion.Options = &AssertionOptions{}
+		if c, ok := opts["connector"].(string); ok {
+			assertion.Options.Connector = c
+		}
+		if meta, ok := opts["metadata"].(map[string]interface{}); ok {
+			assertion.Options.Metadata = meta
+		}
+	}
+
 	return assertion
 }
 
@@ -147,6 +164,8 @@ func (a *Asserter) evaluateAssertion(assertion *Assertion, output, input interfa
 		result = a.assertType(assertion, output)
 	case "script":
 		result = a.assertScript(assertion, output, input)
+	case "agent":
+		result = a.assertAgent(assertion, output, input)
 	default:
 		result.Passed = false
 		result.Message = fmt.Sprintf("unknown assertion type: %s", assertion.Type)
@@ -229,18 +248,15 @@ func (a *Asserter) assertJSONPath(assertion *Assertion, output interface{}) *Ass
 	var jsonData interface{}
 	switch v := output.(type) {
 	case string:
-		// Try to parse as JSON
+		// Use gou/text to extract JSON (handles markdown, auto-repair, etc.)
-		if err := jsoniter.Unmarshal([]byte(v), &jsonData); err != nil {
+		extracted := goutext.ExtractJSON(v)
-			// Try to extract JSON from markdown code blocks
-			extracted := extractJSONFromText(v)
 		if extracted != nil {
 			jsonData = extracted
 		} else {
 			result.Passed = false
-				result.Message = fmt.Sprintf("output is not valid JSON: %s", err.Error())
+			result.Message = fmt.Sprintf("output is not valid JSON: %s", v)
 			return result
 		}
-		}
 	case map[string]interface{}, []interface{}:
 		jsonData = v
 	default:
@@ -478,6 +494,158 @@ func (a *Asserter) getType(v interface{}) string {
 	}
 }
 
+// assertAgent uses an agent to validate the output
+func (a *Asserter) assertAgent(assertion *Assertion, output, input interface{}) *AssertionResult {
+	result := &AssertionResult{
+		Assertion: assertion,
+		Actual:    output,
+	}
+
+	// Parse use field: "agents:tests.validator-agent"
+	if !strings.HasPrefix(assertion.Use, "agents:") {
+		result.Passed = false
+		result.Message = "agent assertion requires 'use' field with 'agents:' prefix"
+		return result
+	}
+
+	agentID := strings.TrimPrefix(assertion.Use, "agents:")
+
+	// Get assistant
+	ast, err := assistant.Get(agentID)
+	if err != nil {
+		result.Passed = false
+		result.Message = fmt.Sprintf("failed to get validator agent: %s", err.Error())
+		return result
+	}
+
+	// Build validation request
+	validationInput := map[string]interface{}{
+		"output": output,
+		"input":  input,
+	}
+
+	// Add criteria from Value field
+	if assertion.Value != nil {
+		validationInput["criteria"] = assertion.Value
+	}
+
+	// Add metadata from options
+	if assertion.Options != nil && assertion.Options.Metadata != nil {
+		for k, v := range assertion.Options.Metadata {
+			validationInput[k] = v
+		}
+	}
+
+	// Build context options - skip history and trace for validator
+	opts := &context.Options{
+		Skip: &context.Skip{
+			History: true,
+			Trace:   true,
+			Output:  true,
+		},
+		Metadata: map[string]interface{}{
+			"test_mode": "validator",
+		},
+	}
+	if assertion.Options != nil && assertion.Options.Connector != "" {
+		opts.Connector = assertion.Options.Connector
+	}
+
+	// Create context and call agent
+	env := NewEnvironment("", "")
+	ctx := NewTestContext("validator", agentID, env)
+	defer ctx.Release()
+
+	// Convert validation input to JSON string for the message
+	inputJSON, err := json.Marshal(validationInput)
+	if err != nil {
+		result.Passed = false
+		result.Message = fmt.Sprintf("failed to marshal validation input: %s", err.Error())
+		return result
+	}
+
+	messages := []context.Message{{
+		Role:    context.RoleUser,
+		Content: string(inputJSON),
+	}}
+
+	response, err := ast.Stream(ctx, messages, opts)
+	if err != nil {
+		result.Passed = false
+		result.Message = fmt.Sprintf("validator agent error: %s", err.Error())
+		return result
+	}
+
+	// Parse response
+	return a.parseValidatorResponse(response, result)
+}
+
+// parseValidatorResponse parses the validator agent's response
+func (a *Asserter) parseValidatorResponse(response *context.Response, result *AssertionResult) *AssertionResult {
+	output := extractValidatorOutput(response)
+
+	// Expected format: { "passed": bool, "reason": string, "score": float, "suggestions": [] }
+	if outputMap, ok := output.(map[string]interface{}); ok {
+		if passed, ok := outputMap["passed"].(bool); ok {
+			result.Passed = passed
+		} else {
+			result.Passed = false
+			result.Message = "validator response missing 'passed' field"
+			return result
+		}
+		if reason, ok := outputMap["reason"].(string); ok {
+			result.Message = reason
+		}
+		// Store score and suggestions in expected field for reference
+		result.Expected = outputMap
+	} else {
+		result.Passed = false
+		result.Message = "validator agent returned invalid response format"
+	}
+
+	return result
+}
+
+// extractValidatorOutput extracts the output from a validator response
+func extractValidatorOutput(response *context.Response) interface{} {
+	if response == nil || response.Completion == nil {
+		return nil
+	}
+
+	// Get content from completion
+	content := response.Completion.Content
+	if content == nil {
+		return nil
+	}
+
+	// Try to get text content
+	var text string
+	switch v := content.(type) {
+	case string:
+		text = v
+	default:
+		// Try to marshal and use as-is
+		data, err := json.Marshal(content)
+		if err != nil {
+			return nil
+		}
+		text = string(data)
+	}
+
+	if text == "" {
+		return nil
+	}
+
+	// Use gou/text to extract JSON (handles markdown code blocks, auto-repair, etc.)
+	result := goutext.ExtractJSON(text)
+	if result != nil {
+		return result
+	}
+
+	// Return raw text if extraction fails
+	return text
+}
+
 // assertScript runs a custom assertion script
 func (a *Asserter) assertScript(assertion *Assertion, output, input interface{}) *AssertionResult {
 	result := &AssertionResult{
@@ -559,30 +727,3 @@ func (a *Asserter) toString(v interface{}) string {
 		return string(b)
 	}
 }
-
-// extractJSONFromText tries to extract JSON from text (e.g., markdown code blocks)
-func extractJSONFromText(text string) interface{} {
-	// Try to find JSON in code blocks
-	patterns := []string{
-		"```json\n",
-		"```\n",
-	}
-
-	for _, start := range patterns {
-		if idx := strings.Index(text, start); idx >= 0 {
-			text = text[idx+len(start):]
-			if endIdx := strings.Index(text, "```"); endIdx >= 0 {
-				text = text[:endIdx]
-			}
-			break
-		}
-	}
-
-	// Try to parse
-	var result interface{}
-	if err := jsoniter.Unmarshal([]byte(strings.TrimSpace(text)), &result); err == nil {
-		return result
-	}
-
-	return nil
-}

agent/test/assert_agent_test.go

@@ -0,0 +1,275 @@
+package test_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	v8 "github.com/yaoapp/gou/runtime/v8"
+	"github.com/yaoapp/yao/agent"
+	agenttest "github.com/yaoapp/yao/agent/test"
+	"github.com/yaoapp/yao/config"
+	"github.com/yaoapp/yao/test"
+	"rogchap.com/v8go"
+)
+
+func TestAsserter_AgentAssertion(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	// Load agent (includes assistants)
+	err := agent.Load(config.Conf)
+	if err != nil {
+		t.Fatalf("Failed to load agent: %v", err)
+	}
+
+	asserter := agenttest.NewAsserter()
+
+	tests := []struct {
+		name     string
+		tc       *agenttest.Case
+		output   interface{}
+		expected bool
+		skipMsg  string
+	}{
+		{
+			name: "agent assertion - pass",
+			tc: &agenttest.Case{
+				Assert: map[string]interface{}{
+					"type":  "agent",
+					"use":   "agents:tests.validator-agent",
+					"value": "Response should be a greeting",
+				},
+			},
+			output:   "Hello! How can I help you today?",
+			expected: true,
+		},
+		{
+			name: "agent assertion - fail",
+			tc: &agenttest.Case{
+				Assert: map[string]interface{}{
+					"type":  "agent",
+					"use":   "agents:tests.validator-agent",
+					"value": "Response should provide a detailed technical answer",
+				},
+			},
+			output:   "I don't know.",
+			expected: false,
+		},
+		{
+			name: "agent assertion - missing prefix",
+			tc: &agenttest.Case{
+				Assert: map[string]interface{}{
+					"type":  "agent",
+					"use":   "tests.validator-agent", // Missing agents: prefix
+					"value": "Should pass",
+				},
+			},
+			output:   "Hello",
+			expected: false, // Should fail due to missing prefix
+		},
+		{
+			name: "agent assertion - with metadata",
+			tc: &agenttest.Case{
+				Assert: map[string]interface{}{
+					"type":  "agent",
+					"use":   "agents:tests.validator-agent",
+					"value": "Response is helpful",
+					"options": map[string]interface{}{
+						"metadata": map[string]interface{}{
+							"context": "customer support",
+						},
+					},
+				},
+			},
+			output:   "I'd be happy to help you with your order. Let me look that up for you.",
+			expected: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.skipMsg != "" {
+				t.Skip(tt.skipMsg)
+			}
+
+			passed, errMsg := asserter.Validate(tt.tc, tt.output)
+			if passed != tt.expected {
+				t.Errorf("Expected passed=%v, got passed=%v, error: %s", tt.expected, passed, errMsg)
+			}
+		})
+	}
+}
+
+func TestAsserter_AgentAssertion_InvalidAgent(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	// Load agent (includes assistants)
+	err := agent.Load(config.Conf)
+	if err != nil {
+		t.Fatalf("Failed to load agent: %v", err)
+	}
+
+	asserter := agenttest.NewAsserter()
+
+	tc := &agenttest.Case{
+		Assert: map[string]interface{}{
+			"type":  "agent",
+			"use":   "agents:nonexistent.agent",
+			"value": "Should fail",
+		},
+	}
+
+	passed, errMsg := asserter.Validate(tc, "Hello")
+	assert.False(t, passed, "Should fail for nonexistent agent")
+	assert.Contains(t, errMsg, "failed to get validator agent", "Error should mention agent loading failure")
+}
+
+func TestAsserter_MapToAssertion_WithUseAndOptions(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	// Load agent (includes assistants)
+	err := agent.Load(config.Conf)
+	if err != nil {
+		t.Fatalf("Failed to load agent: %v", err)
+	}
+
+	asserter := agenttest.NewAsserter()
+
+	// Test that mapToAssertion correctly parses use and options fields
+	tc := &agenttest.Case{
+		Assert: map[string]interface{}{
+			"type":  "agent",
+			"use":   "agents:tests.validator-agent",
+			"value": "criteria here",
+			"options": map[string]interface{}{
+				"connector": "gpt-4o",
+				"metadata": map[string]interface{}{
+					"key": "value",
+				},
+			},
+		},
+	}
+
+	// Validate triggers parseAssertions internally
+	// We just verify it doesn't panic and processes correctly
+	_, _ = asserter.Validate(tc, "test output")
+	// If we get here without panic, the parsing worked
+}
+
+// TestTestingT_AssertAgent tests the JSAPI t.assert.Agent() method
+func TestTestingT_AssertAgent(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	// Load agent (includes assistants)
+	err := agent.Load(config.Conf)
+	if err != nil {
+		t.Fatalf("Failed to load agent: %v", err)
+	}
+
+	tests := []struct {
+		name       string
+		script     string
+		shouldFail bool
+	}{
+		{
+			name: "JSAPI agent assertion - pass",
+			script: `
+				function test(t) {
+					var response = "Hello! How can I help you today?";
+					t.assert.Agent(response, "tests.validator-agent", {
+						criteria: "Response should be a friendly greeting"
+					});
+				}
+				test(__test_t);
+			`,
+			shouldFail: false,
+		},
+		{
+			name: "JSAPI agent assertion - JSON response",
+			script: `
+				function test(t) {
+					var response = {
+						status: "success",
+						data: { user: "john", email: "john@example.com" },
+						message: "User created successfully"
+					};
+					t.assert.Agent(response, "tests.validator-agent", {
+						criteria: "Response should be a successful API response with user data"
+					});
+				}
+				test(__test_t);
+			`,
+			shouldFail: false,
+		},
+		{
+			name: "JSAPI agent assertion - with metadata",
+			script: `
+				function test(t) {
+					var response = "I'd be happy to help you with your order.";
+					t.assert.Agent(response, "tests.validator-agent", {
+						criteria: "Response is helpful and professional",
+						metadata: { context: "customer support" }
+					});
+				}
+				test(__test_t);
+			`,
+			shouldFail: false,
+		},
+		{
+			name: "JSAPI agent assertion - fail case",
+			script: `
+				function test(t) {
+					var response = "I don't know.";
+					t.assert.Agent(response, "tests.validator-agent", {
+						criteria: "Response should provide a detailed technical explanation"
+					});
+				}
+				test(__test_t);
+			`,
+			shouldFail: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Create TestingT
+			testingT := agenttest.NewTestingT(tt.name)
+
+			// Create V8 isolate and context
+			iso := v8go.NewIsolate()
+			defer iso.Dispose()
+
+			v8ctx := v8go.NewContext(iso)
+			defer v8ctx.Close()
+
+			// Create testing object
+			testObj, err := agenttest.NewTestingTObject(v8ctx, testingT)
+			if err != nil {
+				t.Fatalf("Failed to create testing object: %v", err)
+			}
+
+			// Set testing object as global
+			global := v8ctx.Global()
+			global.Set("__test_t", testObj)
+
+			// Run the test script
+			_, err = v8ctx.RunScript(tt.script, "test.js")
+
+			// Check results
+			if tt.shouldFail {
+				assert.True(t, testingT.Failed(), "Test should have failed")
+			} else {
+				if err != nil {
+					t.Errorf("Script execution error: %v", err)
+				}
+				assert.False(t, testingT.Failed(), "Test should have passed, errors: %v", testingT.Errors())
+			}
+		})
+	}
+}
+
+// Ensure v8 is used (for script loading)
+var _ = v8.Scripts

agent/test/script_assert.go

@@ -261,6 +261,9 @@ func newAssertObject(v8ctx *v8go.Context, t *TestingT) (*v8go.Value, error) {
 	// JSON path assertion
 	assertObj.Set("JSONPath", assertJSONPathMethod(iso, t))
 
+	// Agent-driven assertion
+	assertObj.Set("Agent", assertAgentMethod(iso, t))
+
 	// Create instance
 	instance, err := assertObj.NewInstance(v8ctx)
 	if err != nil {
@@ -924,6 +927,70 @@ func assertJSONPathMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate
 	})
 }
 
+// assertAgentMethod implements assert.Agent(response, agentID, options?)
+// Uses a validator agent to check the response
+// agentID is the direct agent ID (no "agents:" prefix needed)
+func assertAgentMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate {
+	return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value {
+		v8ctx := info.Context()
+		args := info.Args()
+		if len(args) < 2 {
+			t.fail("Agent requires response and agentID arguments", &ScriptAssertionInfo{Type: "Agent"})
+			return v8go.Undefined(iso)
+		}
+
+		response, _ := bridge.GoValue(args[0], v8ctx)
+		agentID := args[1].String()
+
+		// Get options if provided
+		var options map[string]interface{}
+		if len(args) > 2 && args[2].IsObject() {
+			optVal, _ := bridge.GoValue(args[2], v8ctx)
+			options, _ = optVal.(map[string]interface{})
+		}
+
+		// Build assertion with agents: prefix
+		assertion := &Assertion{
+			Type: "agent",
+			Use:  "agents:" + agentID,
+		}
+
+		// Extract criteria and metadata from options
+		if options != nil {
+			if criteria, ok := options["criteria"]; ok {
+				assertion.Value = criteria
+			}
+			if metadata, ok := options["metadata"].(map[string]interface{}); ok {
+				assertion.Options = &AssertionOptions{Metadata: metadata}
+			}
+			if connector, ok := options["connector"].(string); ok {
+				if assertion.Options == nil {
+					assertion.Options = &AssertionOptions{}
+				}
+				assertion.Options.Connector = connector
+			}
+		}
+
+		// Use the asserter to validate
+		asserter := &Asserter{}
+		result := asserter.assertAgent(assertion, response, nil)
+
+		if !result.Passed {
+			msg := result.Message
+			if msg == "" {
+				msg = "agent assertion failed"
+			}
+			t.fail(msg, &ScriptAssertionInfo{
+				Type:    "Agent",
+				Actual:  response,
+				Message: msg,
+			})
+		}
+
+		return v8go.Undefined(iso)
+	})
+}
+
 // Helper functions
 
 // deepEqual performs deep equality comparison

agent/test/types.go

@@ -436,6 +436,7 @@ type Assertion struct {
 	// - "script": run a custom assertion script
 	// - "type": check output type (string, object, array, number, boolean)
 	// - "schema": validate against JSON schema
+	// - "agent": use an agent to validate the response
 	Type string `json:"type"`
 
 	// Value is the expected value or pattern (depends on type)
@@ -448,6 +449,14 @@ type Assertion struct {
 	// The script receives (output, input, expected) and returns {pass: bool, message: string}
 	Script string `json:"script,omitempty"`
 
+	// Use specifies the agent/script for validation
+	// For agent assertions: "agents:tests.validator-agent" (with prefix)
+	// For script assertions: "scripts:tests.validate" (with prefix)
+	Use string `json:"use,omitempty"`
+
+	// Options for agent-driven assertions (aligned with context.Options)
+	Options *AssertionOptions `json:"options,omitempty"`
+
 	// Message is a custom failure message
 	Message string `json:"message,omitempty"`
 
@@ -455,6 +464,15 @@ type Assertion struct {
 	Negate bool `json:"negate,omitempty"`
 }
 
+// AssertionOptions for agent-driven assertions
+type AssertionOptions struct {
+	// Connector overrides the agent's default connector
+	Connector string `json:"connector,omitempty"`
+
+	// Metadata contains custom data passed to the validator agent
+	Metadata map[string]interface{} `json:"metadata,omitempty"`
+}
+
 // AssertionResult represents the result of an assertion
 type AssertionResult struct {
 	// Passed indicates whether the assertion passed