diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md index 9b956550..08ed791a 100644 --- a/agent/test/DESIGN_V2.md +++ b/agent/test/DESIGN_V2.md @@ -998,8 +998,9 @@ Existing single-turn tests work unchanged: | Static assertions | ✅ Done | contains, equals, regex, json_path, etc. | | Before/After hooks | ✅ Done | `before/after` in JSONL, `--before/--after` in CLI | | Agent-driven assertions | ✅ Done | `type: "agent"` + `t.assert.Agent()` JSAPI | +| Agent-driven input | ✅ Done | `-i agents:xxx` for test generation | +| Dry-run mode | ✅ Done | `--dry-run` to preview generated tests | | Dynamic mode | 🔲 Planned | Simulator + Checkpoints | -| Agent-driven input | 🔲 Planned | `-i agents:xxx` for test generation | ## Open Questions diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md index 8179ffc4..7ba60499 100644 --- a/agent/test/TODO_V2.md +++ b/agent/test/TODO_V2.md @@ -41,29 +41,53 @@ - [x] 创建示例 validator agent (`assistants/tests/validator-agent`) - [x] 创建单元测试 `assert_agent_test.go` (JSONL 断言 + JSAPI 断言) -## Phase 3: Dynamic Mode (Simulator + Checkpoints) +## Phase 3: Agent-Driven Input ✅ + +**新增文件**: `input_source.go` + +> 用 Agent 生成测试用例，生成后使用标准模式执行。相对简单。 + +**准备工作**: + +- [x] 创建 generator agent (`yao-dev-app/assistants/tests/generator-agent`) +- [x] 编写 generator agent 的 prompts.yml + +**实现**: + +- [x] `input_source.go`: 实现 `ParseInputSource` +- [x] `input_source.go`: 实现 `GenerateTestCases` +- [x] `loader.go`: 添加 `LoadFromAgent` 方法 +- [x] `loader.go`: 添加 `LoadFromScript` 方法 +- [x] `runner.go`: 在 `RunTests` 支持不同输入源 +- [x] `cmd/agent/test.go`: 添加 `--dry-run` flag + +**测试**: + +- [x] 创建单元测试 `input_source_test.go` + +## Phase 4: Dynamic Mode (Simulator + Checkpoints) **新增文件**: `dynamic_runner.go`, `dynamic_types.go` +> 运行时使用 Simulator Agent 动态生成对话，需要多轮循环和 checkpoint 匹配。依赖 Phase 3 的 Agent 调用经验。 + +**准备工作**: + +- [ ] 创建 simulator agent (`yao-dev-app/assistants/tests/simulator-agent`) +- [ ] 编写 simulator agent 的 prompts.yml (模拟用户行为) + +**实现**: + - [ ] `types.go`: 添加 `Simulator`, `Checkpoints` 字段到 `Case` - [ ] `dynamic_types.go`: 定义 `Checkpoint`, `DynamicResult` 等类型 - [ ] `dynamic_runner.go`: 实现 `DynamicRunner` - [ ] `dynamic_runner.go`: 实现 checkpoint 匹配逻辑 - [ ] `dynamic_runner.go`: 实现终止条件判断 - [ ] `runner.go`: 在 `runSingleTest` 判断并调用动态模式 -- [ ] 创建示例 simulator agent -## Phase 4: Agent-Driven Input +**测试**: -**新增文件**: `input_source.go` - -- [ ] `input_source.go`: 实现 `ParseInputSource` -- [ ] `input_source.go`: 实现 `GenerateTestCases` -- [ ] `loader.go`: 添加 `LoadFromAgent` 方法 -- [ ] `loader.go`: 添加 `LoadFromScript` 方法 -- [ ] `runner.go`: 在 `RunTests` 支持不同输入源 -- [ ] `cmd/agent/agent.go`: 添加 `--dry-run` flag -- [ ] 创建示例 generator agent +- [ ] 创建单元测试 `dynamic_runner_test.go` ## Phase 5: Console Output Optimization @@ -85,6 +109,8 @@ - [x] Script testing (`*_test.ts`) - [x] Before/After hooks (Phase 1) - [x] Agent-driven assertions (Phase 2) +- [x] Agent-driven input (Phase 3) +- [x] `--dry-run` flag ## Open Questions diff --git a/agent/test/input_source.go b/agent/test/input_source.go new file mode 100644 index 00000000..0b905f80 --- /dev/null +++ b/agent/test/input_source.go @@ -0,0 +1,392 @@ +package test + +import ( + "fmt" + "net/url" + "strconv" + "strings" + + jsoniter "github.com/json-iterator/go" + goutext "github.com/yaoapp/gou/text" + "github.com/yaoapp/yao/agent/assistant" + "github.com/yaoapp/yao/agent/context" +) + +// InputSourceType represents the type of input source +type InputSourceType string + +const ( + // InputSourceFile indicates input from a JSONL file + InputSourceFile InputSourceType = "file" + // InputSourceMessage indicates input from a direct message string + InputSourceMessage InputSourceType = "message" + // InputSourceScript indicates script test mode + InputSourceScript InputSourceType = "script" + // InputSourceAgent indicates input generated by an agent + InputSourceAgent InputSourceType = "agent" +) + +// InputSource represents a parsed input source +type InputSource struct { + Type InputSourceType // file, message, script, agent + Value string // path, message, script ref, or agent ID + Params map[string]interface{} // query parameters (for agent source) +} + +// ParseInputSource parses the -i flag value into an InputSource +// Supported formats: +// - "agents:workers.test.generator" - Agent-generated test cases +// - "agents:workers.test.generator?count=10&focus=edge-cases" - With parameters +// - "scripts.tests.gen" - Script-generated test cases +// - "./tests/inputs.jsonl" - JSONL file +// - "Hello, how are you?" - Direct message +func ParseInputSource(input string) *InputSource { + // Check for agents: prefix + if strings.HasPrefix(input, "agents:") { + return parseAgentSource(strings.TrimPrefix(input, "agents:")) + } + + // Check for scripts: prefix (for generator scripts) + if strings.HasPrefix(input, "scripts:") { + return &InputSource{ + Type: InputSourceScript, + Value: strings.TrimPrefix(input, "scripts:"), + } + } + + // Check for script test mode (scripts.xxx format without prefix) + if strings.HasPrefix(input, "scripts.") { + return &InputSource{ + Type: InputSourceScript, + Value: input, + } + } + + // Check for file extension + if strings.HasSuffix(input, ".jsonl") || strings.HasSuffix(input, ".json") { + return &InputSource{ + Type: InputSourceFile, + Value: input, + } + } + + // Check if it looks like a file path + if strings.Contains(input, "/") || strings.Contains(input, "\\") { + return &InputSource{ + Type: InputSourceFile, + Value: input, + } + } + + // Default to message + return &InputSource{ + Type: InputSourceMessage, + Value: input, + } +} + +// parseAgentSource parses an agent source string with optional query parameters +// Format: "agent.id" or "agent.id?count=10&focus=edge-cases" +func parseAgentSource(input string) *InputSource { + source := &InputSource{ + Type: InputSourceAgent, + Params: make(map[string]interface{}), + } + + // Check for query parameters + if idx := strings.Index(input, "?"); idx >= 0 { + source.Value = input[:idx] + queryStr := input[idx+1:] + + // Parse query parameters + values, err := url.ParseQuery(queryStr) + if err == nil { + for key, vals := range values { + if len(vals) > 0 { + // Try to parse as number + if num, err := strconv.Atoi(vals[0]); err == nil { + source.Params[key] = num + } else if num, err := strconv.ParseFloat(vals[0], 64); err == nil { + source.Params[key] = num + } else if vals[0] == "true" { + source.Params[key] = true + } else if vals[0] == "false" { + source.Params[key] = false + } else { + source.Params[key] = vals[0] + } + } + } + } + } else { + source.Value = input + } + + return source +} + +// GeneratorInput represents the input sent to a generator agent +type GeneratorInput struct { + TargetAgent *TargetAgentInfo `json:"target_agent"` + Count int `json:"count,omitempty"` + Focus string `json:"focus,omitempty"` + Extra map[string]interface{} `json:"extra,omitempty"` +} + +// TargetAgentInfo contains information about the agent being tested +type TargetAgentInfo struct { + ID string `json:"id"` + Description string `json:"description,omitempty"` + Tools []map[string]interface{} `json:"tools,omitempty"` +} + +// GenerateTestCases generates test cases using a generator agent +func GenerateTestCases(agentID string, targetInfo *TargetAgentInfo, params map[string]interface{}) ([]*Case, error) { + // Get generator assistant + ast, err := assistant.Get(agentID) + if err != nil { + return nil, fmt.Errorf("failed to get generator agent %s: %w", agentID, err) + } + + // Build generation request + genInput := &GeneratorInput{ + TargetAgent: targetInfo, + Count: 5, // Default count + } + + // Apply parameters + if params != nil { + if count, ok := params["count"].(int); ok { + genInput.Count = count + } + if focus, ok := params["focus"].(string); ok { + genInput.Focus = focus + } + // Store extra parameters + genInput.Extra = make(map[string]interface{}) + for k, v := range params { + if k != "count" && k != "focus" { + genInput.Extra[k] = v + } + } + } + + // Create context + env := NewEnvironment("", "") + ctx := NewTestContext("generator", agentID, env) + defer ctx.Release() + + // Build options - skip history and trace for efficiency + opts := &context.Options{ + Skip: &context.Skip{ + History: true, + Trace: true, + Output: true, + }, + Metadata: map[string]interface{}{ + "test_mode": "generator", + }, + } + + // Build message + inputJSON, err := jsoniter.Marshal(genInput) + if err != nil { + return nil, fmt.Errorf("failed to marshal generator input: %w", err) + } + + messages := []context.Message{{ + Role: context.RoleUser, + Content: string(inputJSON), + }} + + // Call generator agent + response, err := ast.Stream(ctx, messages, opts) + if err != nil { + return nil, fmt.Errorf("generator agent error: %w", err) + } + + // Extract and parse response + return parseGeneratedCases(response) +} + +// parseGeneratedCases parses the generator agent's response into test cases +func parseGeneratedCases(response *context.Response) ([]*Case, error) { + if response == nil || response.Completion == nil { + return nil, fmt.Errorf("empty response from generator agent") + } + + // Extract content + content := response.Completion.Content + if content == nil { + return nil, fmt.Errorf("no content in generator response") + } + + // Convert content to string + var text string + switch v := content.(type) { + case string: + text = v + default: + data, err := jsoniter.Marshal(content) + if err != nil { + return nil, fmt.Errorf("failed to marshal content: %w", err) + } + text = string(data) + } + + // Use goutext.ExtractJSON for fault-tolerant parsing + parsed := goutext.ExtractJSON(text) + if parsed == nil { + return nil, fmt.Errorf("failed to parse generator response as JSON: %s", truncateOutput(text, 200)) + } + + // Convert to []*Case + return convertToCases(parsed) +} + +// convertToCases converts parsed JSON to test cases +func convertToCases(parsed interface{}) ([]*Case, error) { + // Handle array of cases + arr, ok := parsed.([]interface{}) + if !ok { + // Maybe it's a single case wrapped in an object + if obj, ok := parsed.(map[string]interface{}); ok { + if cases, ok := obj["cases"].([]interface{}); ok { + arr = cases + } else if testCases, ok := obj["test_cases"].([]interface{}); ok { + arr = testCases + } else { + // Single case + arr = []interface{}{obj} + } + } else { + return nil, fmt.Errorf("expected array of test cases, got %T", parsed) + } + } + + cases := make([]*Case, 0, len(arr)) + for i, item := range arr { + caseMap, ok := item.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("test case %d is not an object", i) + } + + tc, err := mapToCase(caseMap) + if err != nil { + return nil, fmt.Errorf("failed to parse test case %d: %w", i, err) + } + + cases = append(cases, tc) + } + + return cases, nil +} + +// mapToCase converts a map to a Case struct +func mapToCase(m map[string]interface{}) (*Case, error) { + tc := &Case{} + + // Required: id + if id, ok := m["id"].(string); ok { + tc.ID = id + } else { + return nil, fmt.Errorf("missing required field 'id'") + } + + // Required: input + if input, ok := m["input"]; ok { + tc.Input = input + } else { + return nil, fmt.Errorf("missing required field 'input'") + } + + // Optional: assertions/assert + if assertions, ok := m["assertions"]; ok { + tc.Assert = assertions + } else if assert, ok := m["assert"]; ok { + tc.Assert = assert + } + + // Optional: options - convert map to CaseOptions + if options, ok := m["options"].(map[string]interface{}); ok { + tc.Options = mapToCaseOptions(options) + } + + // Optional: before/after + if before, ok := m["before"].(string); ok { + tc.Before = before + } + if after, ok := m["after"].(string); ok { + tc.After = after + } + + // Optional: timeout + if timeout, ok := m["timeout"].(string); ok { + tc.Timeout = timeout + } + + return tc, nil +} + +// ToInputMode converts InputSourceType to InputMode for backward compatibility +func (s *InputSource) ToInputMode() InputMode { + switch s.Type { + case InputSourceFile: + return InputModeFile + case InputSourceMessage: + return InputModeMessage + case InputSourceScript: + return InputModeScript + case InputSourceAgent: + // Agent source generates cases, then runs in file mode + return InputModeFile + default: + return InputModeMessage + } +} + +// mapToCaseOptions converts a map to CaseOptions +func mapToCaseOptions(m map[string]interface{}) *CaseOptions { + opts := &CaseOptions{} + + if connector, ok := m["connector"].(string); ok { + opts.Connector = connector + } + + if mode, ok := m["mode"].(string); ok { + opts.Mode = mode + } + + if disableGlobalPrompts, ok := m["disable_global_prompts"].(bool); ok { + opts.DisableGlobalPrompts = disableGlobalPrompts + } + + if search, ok := m["search"].(bool); ok { + opts.Search = &search + } + + if metadata, ok := m["metadata"].(map[string]interface{}); ok { + opts.Metadata = metadata + } + + if skip, ok := m["skip"].(map[string]interface{}); ok { + opts.Skip = &CaseSkipOptions{} + if history, ok := skip["history"].(bool); ok { + opts.Skip.History = history + } + if trace, ok := skip["trace"].(bool); ok { + opts.Skip.Trace = trace + } + if output, ok := skip["output"].(bool); ok { + opts.Skip.Output = output + } + if keyword, ok := skip["keyword"].(bool); ok { + opts.Skip.Keyword = keyword + } + if searchSkip, ok := skip["search"].(bool); ok { + opts.Skip.Search = searchSkip + } + } + + return opts +} diff --git a/agent/test/input_source_test.go b/agent/test/input_source_test.go new file mode 100644 index 00000000..7e89a9ae --- /dev/null +++ b/agent/test/input_source_test.go @@ -0,0 +1,181 @@ +package test_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/yao/agent" + agenttest "github.com/yaoapp/yao/agent/test" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +func TestParseInputSource(t *testing.T) { + tests := []struct { + name string + input string + wantType agenttest.InputSourceType + wantValue string + wantParams map[string]interface{} + }{ + { + name: "JSONL file", + input: "./tests/inputs.jsonl", + wantType: agenttest.InputSourceFile, + wantValue: "./tests/inputs.jsonl", + }, + { + name: "JSON file", + input: "./tests/inputs.json", + wantType: agenttest.InputSourceFile, + wantValue: "./tests/inputs.json", + }, + { + name: "direct message", + input: "Hello, how are you?", + wantType: agenttest.InputSourceMessage, + wantValue: "Hello, how are you?", + }, + { + name: "agent source simple", + input: "agents:tests.generator-agent", + wantType: agenttest.InputSourceAgent, + wantValue: "tests.generator-agent", + }, + { + name: "agent source with params", + input: "agents:tests.generator-agent?count=10&focus=edge-cases", + wantType: agenttest.InputSourceAgent, + wantValue: "tests.generator-agent", + wantParams: map[string]interface{}{ + "count": 10, + "focus": "edge-cases", + }, + }, + { + name: "agent source with boolean param", + input: "agents:tests.generator-agent?verbose=true", + wantType: agenttest.InputSourceAgent, + wantValue: "tests.generator-agent", + wantParams: map[string]interface{}{ + "verbose": true, + }, + }, + { + name: "script source with prefix", + input: "scripts:tests.gen.Generate", + wantType: agenttest.InputSourceScript, + wantValue: "tests.gen.Generate", + }, + { + name: "script test mode", + input: "scripts.tests.gen", + wantType: agenttest.InputSourceScript, + wantValue: "scripts.tests.gen", + }, + { + name: "path with separator", + input: "/path/to/inputs.jsonl", + wantType: agenttest.InputSourceFile, + wantValue: "/path/to/inputs.jsonl", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := agenttest.ParseInputSource(tt.input) + + assert.Equal(t, tt.wantType, source.Type, "Type mismatch") + assert.Equal(t, tt.wantValue, source.Value, "Value mismatch") + + if tt.wantParams != nil { + for k, v := range tt.wantParams { + assert.Equal(t, v, source.Params[k], "Param %s mismatch", k) + } + } + }) + } +} + +func TestInputSource_ToInputMode(t *testing.T) { + tests := []struct { + name string + source *agenttest.InputSource + wantMode agenttest.InputMode + }{ + { + name: "file source", + source: &agenttest.InputSource{Type: agenttest.InputSourceFile}, + wantMode: agenttest.InputModeFile, + }, + { + name: "message source", + source: &agenttest.InputSource{Type: agenttest.InputSourceMessage}, + wantMode: agenttest.InputModeMessage, + }, + { + name: "script source", + source: &agenttest.InputSource{Type: agenttest.InputSourceScript}, + wantMode: agenttest.InputModeScript, + }, + { + name: "agent source", + source: &agenttest.InputSource{Type: agenttest.InputSourceAgent}, + wantMode: agenttest.InputModeFile, // Agent generates cases, then runs in file mode + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mode := tt.source.ToInputMode() + assert.Equal(t, tt.wantMode, mode) + }) + } +} + +func TestGenerateTestCases(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent (includes assistants) + err := agent.Load(config.Conf) + if err != nil { + t.Fatalf("Failed to load agent: %v", err) + } + + // Test generating test cases from the generator agent + targetInfo := &agenttest.TargetAgentInfo{ + ID: "tests.next", + Description: "A simple test agent for greeting", + } + + params := map[string]interface{}{ + "count": 3, + "focus": "happy-path", + } + + cases, err := agenttest.GenerateTestCases("tests.generator-agent", targetInfo, params) + if err != nil { + t.Fatalf("Failed to generate test cases: %v", err) + } + + // Verify we got some test cases + assert.NotEmpty(t, cases, "Should generate at least one test case") + + // Verify each case has required fields + for _, tc := range cases { + assert.NotEmpty(t, tc.ID, "Test case should have ID") + assert.NotNil(t, tc.Input, "Test case should have Input") + } + + t.Logf("Generated %d test cases", len(cases)) + for _, tc := range cases { + t.Logf(" - %s", tc.ID) + } +} + +func TestMapToCaseOptions(t *testing.T) { + // Test that options map is correctly converted + source := agenttest.ParseInputSource("agents:test?count=5") + assert.Equal(t, 5, source.Params["count"]) +} diff --git a/agent/test/interfaces.go b/agent/test/interfaces.go index a5996010..d0083d2e 100644 --- a/agent/test/interfaces.go +++ b/agent/test/interfaces.go @@ -43,6 +43,12 @@ type Loader interface { // LoadFile loads test cases from a JSONL file LoadFile(path string) ([]*Case, error) + + // LoadFromAgent generates test cases using a generator agent + LoadFromAgent(agentID string, targetInfo *TargetAgentInfo, params map[string]interface{}) ([]*Case, error) + + // LoadFromScript generates test cases using a script + LoadFromScript(scriptRef string, targetInfo *TargetAgentInfo) ([]*Case, error) } // Resolver is the interface for resolving agent information diff --git a/agent/test/loader.go b/agent/test/loader.go index 49de1a77..52092b68 100644 --- a/agent/test/loader.go +++ b/agent/test/loader.go @@ -8,6 +8,7 @@ import ( "time" jsoniter "github.com/json-iterator/go" + "github.com/yaoapp/gou/process" ) // JSONLLoader loads test cases from JSONL files @@ -143,3 +144,35 @@ func FilterByIDs(cases []*Case, ids []string) []*Case { return idSet[tc.ID] }) } + +// LoadFromAgent generates test cases using a generator agent +func (l *JSONLLoader) LoadFromAgent(agentID string, targetInfo *TargetAgentInfo, params map[string]interface{}) ([]*Case, error) { + return GenerateTestCases(agentID, targetInfo, params) +} + +// LoadFromScript generates test cases using a script +// scriptRef format: "module.FunctionName" (e.g., "tests.gen.Generate") +func (l *JSONLLoader) LoadFromScript(scriptRef string, targetInfo *TargetAgentInfo) ([]*Case, error) { + // Parse script reference + parts := strings.Split(scriptRef, ".") + if len(parts) < 2 { + return nil, fmt.Errorf("invalid script reference format: %s (expected 'module.Function')", scriptRef) + } + + // Build process name: scripts.module.Function + processName := "scripts." + scriptRef + + // Execute via process + p, err := process.Of(processName, targetInfo) + if err != nil { + return nil, fmt.Errorf("failed to create process %s: %w", processName, err) + } + + result, err := p.Exec() + if err != nil { + return nil, fmt.Errorf("script execution failed: %w", err) + } + + // Parse result as test cases + return convertToCases(result) +} diff --git a/agent/test/runner.go b/agent/test/runner.go index be0f2d04..7afbee70 100644 --- a/agent/test/runner.go +++ b/agent/test/runner.go @@ -5,6 +5,8 @@ import ( "fmt" "os" "path/filepath" + "reflect" + "strings" "sync" "time" @@ -170,15 +172,58 @@ func (r *Executor) RunTests() (*Report, error) { r.output.Info("Connector: %s", agentInfo.Connector) } - // Load test cases + // Load test cases based on input source var testCases []*Case + inputSource := ParseInputSource(r.opts.Input) - // File mode - load from JSONL - testCases, err = r.loader.LoadFile(r.opts.Input) - if err != nil { - return nil, fmt.Errorf("failed to load test cases: %w", err) + switch inputSource.Type { + case InputSourceAgent: + // Generate test cases using agent + r.output.Info("Generating test cases from agent: %s", inputSource.Value) + targetInfo := &TargetAgentInfo{ + ID: agentInfo.ID, + Description: agentInfo.Description, + } + testCases, err = r.loader.LoadFromAgent(inputSource.Value, targetInfo, inputSource.Params) + if err != nil { + return nil, fmt.Errorf("failed to generate test cases: %w", err) + } + r.output.Info("Generated: %d test cases", len(testCases)) + + case InputSourceScript: + // Generate test cases using script (if it's a generator script, not test script) + // Note: scripts. prefix without "scripts:" is handled by RunScriptTests + if strings.HasPrefix(r.opts.Input, "scripts:") { + scriptRef := strings.TrimPrefix(r.opts.Input, "scripts:") + r.output.Info("Generating test cases from script: %s", scriptRef) + targetInfo := &TargetAgentInfo{ + ID: agentInfo.ID, + Description: agentInfo.Description, + } + testCases, err = r.loader.LoadFromScript(scriptRef, targetInfo) + if err != nil { + return nil, fmt.Errorf("failed to generate test cases from script: %w", err) + } + r.output.Info("Generated: %d test cases", len(testCases)) + } else { + // This is a test script (scripts.xxx format), handled by RunScriptTests + return nil, fmt.Errorf("script test mode should be handled by RunScriptTests") + } + + default: + // File mode - load from JSONL + testCases, err = r.loader.LoadFile(r.opts.Input) + if err != nil { + return nil, fmt.Errorf("failed to load test cases: %w", err) + } + r.output.Info("Input: %s (%d test cases)", r.opts.Input, len(testCases)) + } + + // Handle dry-run mode - just output the generated test cases + if r.opts.DryRun { + r.output.Info("Dry-run mode: outputting generated test cases") + return r.outputDryRun(testCases, agentInfo) } - r.output.Info("Input: %s (%d test cases)", r.opts.Input, len(testCases)) // Filter skipped tests activeTests := FilterSkipped(testCases) @@ -641,6 +686,12 @@ func isEmptyValue(v interface{}) bool { return true } + // Use reflection to check for typed nil (e.g., *NextHookResponse(nil)) + rv := reflect.ValueOf(v) + if rv.Kind() == reflect.Ptr && rv.IsNil() { + return true + } + switch val := v.(type) { case string: return val == "" @@ -648,6 +699,12 @@ func isEmptyValue(v interface{}) bool { return len(val) == 0 case []interface{}: return len(val) == 0 + case *context.NextHookResponse: + // Check if NextHookResponse is effectively empty + if val == nil { + return true + } + return val.Data == nil && val.Delegate == nil } return false @@ -681,3 +738,51 @@ func (r *Executor) getInputOptions() *InputOptions { return opts } + +// outputDryRun outputs generated test cases without running them +func (r *Executor) outputDryRun(testCases []*Case, agentInfo *AgentInfo) (*Report, error) { + r.output.Info("Generated Test Cases:") + + // Output each test case as JSONL + for _, tc := range testCases { + data, err := jsoniter.Marshal(tc) + if err != nil { + r.output.Warning("Failed to marshal test case %s: %s", tc.ID, err.Error()) + continue + } + fmt.Println(string(data)) + } + + // Write to output file if specified + if r.opts.OutputFile != "" { + file, err := os.Create(r.opts.OutputFile) + if err != nil { + return nil, fmt.Errorf("failed to create output file: %w", err) + } + defer file.Close() + + for _, tc := range testCases { + data, err := jsoniter.Marshal(tc) + if err != nil { + continue + } + file.WriteString(string(data) + "\n") + } + + r.output.Info("Output written to: %s", r.opts.OutputFile) + } + + // Return a minimal report + connector := r.opts.Connector + if connector == "" { + connector = agentInfo.Connector + } + + return &Report{ + Summary: &Summary{ + Total: len(testCases), + AgentID: agentInfo.ID, + Connector: connector, + }, + }, nil +} diff --git a/agent/test/runner_integration_test.go b/agent/test/runner_integration_test.go new file mode 100644 index 00000000..aae1c398 --- /dev/null +++ b/agent/test/runner_integration_test.go @@ -0,0 +1,280 @@ +package test_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/yaoapp/yao/agent" + agenttest "github.com/yaoapp/yao/agent/test" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +// TestRunner_AgentDrivenInput tests the complete flow: +// 1. Use generator-agent to generate test cases +// 2. Run the generated tests against simple-greeting agent +func TestRunner_AgentDrivenInput(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Test with agent-driven input + opts := &agenttest.Options{ + Input: "agents:tests.generator-agent?count=3", + AgentID: "tests.simple-greeting", + Verbose: true, + InputMode: agenttest.InputModeFile, // Will be overridden by ParseInputSource + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + require.NotNil(t, report.Summary, "Summary should not be nil") + + // Verify report + assert.Greater(t, report.Summary.Total, 0, "Should have at least one test case") + t.Logf("Total: %d, Passed: %d, Failed: %d", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) +} + +// TestRunner_AgentDrivenInput_DryRun tests dry-run mode +func TestRunner_AgentDrivenInput_DryRun(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Test with dry-run mode + opts := &agenttest.Options{ + Input: "agents:tests.generator-agent?count=2", + AgentID: "tests.simple-greeting", + DryRun: true, + Verbose: true, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Dry-run should not return error") + require.NotNil(t, report, "Report should not be nil") + + // In dry-run mode, tests are generated but not executed + // So Passed and Failed should both be 0, but Total should have the count + assert.Greater(t, report.Summary.Total, 0, "Should have generated test cases") + + t.Logf("Generated %d test cases in dry-run mode", report.Summary.Total) +} + +// TestRunner_FileInput tests loading test cases from JSONL file +func TestRunner_FileInput(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Create a temporary JSONL file with test cases + // Use case-insensitive contains for robustness + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "inputs.jsonl") + + testCases := `{"id": "greeting-hello", "input": "Hello", "assert": {"type": "regex", "value": "(?i)hello"}} +{"id": "greeting-hi", "input": "Hi there", "assert": {"type": "regex", "value": "(?i)(hi|hello)"}} +{"id": "greeting-morning", "input": "Good morning", "assert": {"type": "regex", "value": "(?i)(hello|morning|good)"}}` + + err = os.WriteFile(inputFile, []byte(testCases), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run tests from file + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.simple-greeting", + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + // Verify report + assert.Equal(t, 3, report.Summary.Total, "Should have 3 test cases") + t.Logf("Total: %d, Passed: %d, Failed: %d", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) + + // Check results for debugging + if report.Results != nil { + for _, r := range report.Results { + t.Logf(" [%s] Status: %s, Output: %v", r.ID, r.Status, r.Output) + } + } +} + +// TestRunner_DirectMessage tests direct message mode +func TestRunner_DirectMessage(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Test with direct message + opts := &agenttest.Options{ + Input: "Hello, how are you?", + AgentID: "tests.simple-greeting", + Verbose: true, + InputMode: agenttest.InputModeMessage, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + // Direct message mode returns a minimal report + assert.Equal(t, 1, report.Summary.Total, "Should have 1 test case") + assert.Equal(t, 1, report.Summary.Passed, "Direct message should pass") +} + +// TestRunner_WithBeforeAfter tests before/after hooks +func TestRunner_WithBeforeAfter(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Create a temporary JSONL file with test cases that use hooks + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "inputs.jsonl") + + // Note: hooks-test agent has env_test.ts with Before/After functions + testCases := `{"id": "hook-test-1", "input": "Hello", "assert": {"type": "contains", "value": "hello"}, "before": "env_test.Before", "after": "env_test.After"}` + + err = os.WriteFile(inputFile, []byte(testCases), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run tests with hooks (using hooks-test agent which has the hook scripts) + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.hooks-test", + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + t.Logf("Total: %d, Passed: %d, Failed: %d", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) +} + +// TestRunner_Parallel tests parallel execution +func TestRunner_Parallel(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Create a temporary JSONL file with multiple test cases + // Use regex for case-insensitive matching + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "inputs.jsonl") + + testCases := `{"id": "parallel-1", "input": "Hello", "assert": {"type": "regex", "value": "(?i)(hello|hi)"}} +{"id": "parallel-2", "input": "Hi", "assert": {"type": "regex", "value": "(?i)(hello|hi)"}} +{"id": "parallel-3", "input": "Hey", "assert": {"type": "regex", "value": "(?i)(hello|hi|hey)"}} +{"id": "parallel-4", "input": "Good day", "assert": {"type": "regex", "value": "(?i)(hello|good|day)"}}` + + err = os.WriteFile(inputFile, []byte(testCases), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run tests in parallel + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.simple-greeting", + Parallel: 2, // Run 2 tests in parallel + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + assert.Equal(t, 4, report.Summary.Total, "Should have 4 test cases") + t.Logf("Total: %d, Passed: %d, Failed: %d (parallel: 2)", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) +} + +// TestRunner_FailFast tests fail-fast behavior +func TestRunner_FailFast(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Create a temporary JSONL file with a failing test first + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "inputs.jsonl") + + // First test will fail (expects "impossible" which won't be in response) + testCases := `{"id": "fail-first", "input": "Hello", "assert": {"type": "contains", "value": "IMPOSSIBLE_STRING_12345"}} +{"id": "should-skip", "input": "Hi", "assert": {"type": "contains", "value": "hi"}}` + + err = os.WriteFile(inputFile, []byte(testCases), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run tests with fail-fast + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.simple-greeting", + FailFast: true, + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error (fail-fast is not an error)") + require.NotNil(t, report, "Report should not be nil") + + // With fail-fast, only the first test should run + assert.Equal(t, 1, report.Summary.Failed, "First test should fail") + // The second test might not run due to fail-fast + t.Logf("Total: %d, Passed: %d, Failed: %d (fail-fast enabled)", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) +} diff --git a/agent/test/types.go b/agent/test/types.go index 75387879..4d78561d 100644 --- a/agent/test/types.go +++ b/agent/test/types.go @@ -152,6 +152,10 @@ type Options struct { // AfterAll is the global after script (e.g., "scripts:tests.env.AfterAll") // Called once after all test cases AfterAll string `json:"after_all,omitempty"` + + // DryRun generates test cases without running them + // Useful for previewing agent-generated test cases + DryRun bool `json:"dry_run,omitempty"` } // ContextConfig represents custom context configuration from JSON file diff --git a/cmd/agent/test.go b/cmd/agent/test.go index f48384f0..7447c463 100644 --- a/cmd/agent/test.go +++ b/cmd/agent/test.go @@ -35,6 +35,7 @@ var ( testFailFast bool testBefore string // --before flag for global BeforeAll hook testAfter string // --after flag for global AfterAll hook + testDryRun bool // --dry-run flag for generating tests without running ) // TestCmd is the agent test command @@ -161,6 +162,7 @@ var TestCmd = &cobra.Command{ FailFast: testFailFast, BeforeAll: testBefore, AfterAll: testAfter, + DryRun: testDryRun, } // Merge with defaults @@ -250,6 +252,7 @@ func init() { TestCmd.Flags().BoolVar(&testFailFast, "fail-fast", false, L("Stop on first failure")) TestCmd.Flags().StringVar(&testBefore, "before", "", L("Global BeforeAll hook (e.g., env_test.BeforeAll)")) TestCmd.Flags().StringVar(&testAfter, "after", "", L("Global AfterAll hook (e.g., env_test.AfterAll)")) + TestCmd.Flags().BoolVar(&testDryRun, "dry-run", false, L("Generate test cases without running them")) // Mark input as required TestCmd.MarkFlagRequired("input")