6ea982f027
- Added support for generating test cases using agents and scripts through the new `LoadFromAgent` and `LoadFromScript` methods in the loader. - Enhanced the `RunTests` method to handle different input sources, including agent-driven and script-based test case generation. - Introduced a `--dry-run` flag to allow users to preview generated test cases without executing them. - Updated relevant documentation in DESIGN_V2.md and TODO_V2.md to reflect the new features and implementation status.
280 lines
9.2 KiB
Go
280 lines
9.2 KiB
Go
package test_test
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"github.com/yaoapp/yao/agent"
|
|
agenttest "github.com/yaoapp/yao/agent/test"
|
|
"github.com/yaoapp/yao/config"
|
|
"github.com/yaoapp/yao/test"
|
|
)
|
|
|
|
// TestRunner_AgentDrivenInput tests the complete flow:
|
|
// 1. Use generator-agent to generate test cases
|
|
// 2. Run the generated tests against simple-greeting agent
|
|
func TestRunner_AgentDrivenInput(t *testing.T) {
|
|
test.Prepare(t, config.Conf)
|
|
defer test.Clean()
|
|
|
|
// Load agents
|
|
err := agent.Load(config.Conf)
|
|
require.NoError(t, err, "Failed to load agents")
|
|
|
|
// Test with agent-driven input
|
|
opts := &agenttest.Options{
|
|
Input: "agents:tests.generator-agent?count=3",
|
|
AgentID: "tests.simple-greeting",
|
|
Verbose: true,
|
|
InputMode: agenttest.InputModeFile, // Will be overridden by ParseInputSource
|
|
}
|
|
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
|
|
|
|
runner := agenttest.NewRunner(opts)
|
|
report, err := runner.Run()
|
|
|
|
require.NoError(t, err, "Runner should not return error")
|
|
require.NotNil(t, report, "Report should not be nil")
|
|
require.NotNil(t, report.Summary, "Summary should not be nil")
|
|
|
|
// Verify report
|
|
assert.Greater(t, report.Summary.Total, 0, "Should have at least one test case")
|
|
t.Logf("Total: %d, Passed: %d, Failed: %d",
|
|
report.Summary.Total, report.Summary.Passed, report.Summary.Failed)
|
|
}
|
|
|
|
// TestRunner_AgentDrivenInput_DryRun tests dry-run mode
|
|
func TestRunner_AgentDrivenInput_DryRun(t *testing.T) {
|
|
test.Prepare(t, config.Conf)
|
|
defer test.Clean()
|
|
|
|
// Load agents
|
|
err := agent.Load(config.Conf)
|
|
require.NoError(t, err, "Failed to load agents")
|
|
|
|
// Test with dry-run mode
|
|
opts := &agenttest.Options{
|
|
Input: "agents:tests.generator-agent?count=2",
|
|
AgentID: "tests.simple-greeting",
|
|
DryRun: true,
|
|
Verbose: true,
|
|
}
|
|
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
|
|
|
|
runner := agenttest.NewRunner(opts)
|
|
report, err := runner.Run()
|
|
|
|
require.NoError(t, err, "Dry-run should not return error")
|
|
require.NotNil(t, report, "Report should not be nil")
|
|
|
|
// In dry-run mode, tests are generated but not executed
|
|
// So Passed and Failed should both be 0, but Total should have the count
|
|
assert.Greater(t, report.Summary.Total, 0, "Should have generated test cases")
|
|
|
|
t.Logf("Generated %d test cases in dry-run mode", report.Summary.Total)
|
|
}
|
|
|
|
// TestRunner_FileInput tests loading test cases from JSONL file
|
|
func TestRunner_FileInput(t *testing.T) {
|
|
test.Prepare(t, config.Conf)
|
|
defer test.Clean()
|
|
|
|
// Load agents
|
|
err := agent.Load(config.Conf)
|
|
require.NoError(t, err, "Failed to load agents")
|
|
|
|
// Create a temporary JSONL file with test cases
|
|
// Use case-insensitive contains for robustness
|
|
tmpDir := t.TempDir()
|
|
inputFile := filepath.Join(tmpDir, "inputs.jsonl")
|
|
|
|
testCases := `{"id": "greeting-hello", "input": "Hello", "assert": {"type": "regex", "value": "(?i)hello"}}
|
|
{"id": "greeting-hi", "input": "Hi there", "assert": {"type": "regex", "value": "(?i)(hi|hello)"}}
|
|
{"id": "greeting-morning", "input": "Good morning", "assert": {"type": "regex", "value": "(?i)(hello|morning|good)"}}`
|
|
|
|
err = os.WriteFile(inputFile, []byte(testCases), 0644)
|
|
require.NoError(t, err, "Failed to write test file")
|
|
|
|
// Run tests from file
|
|
opts := &agenttest.Options{
|
|
Input: inputFile,
|
|
AgentID: "tests.simple-greeting",
|
|
Verbose: true,
|
|
InputMode: agenttest.InputModeFile,
|
|
}
|
|
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
|
|
|
|
runner := agenttest.NewRunner(opts)
|
|
report, err := runner.Run()
|
|
|
|
require.NoError(t, err, "Runner should not return error")
|
|
require.NotNil(t, report, "Report should not be nil")
|
|
|
|
// Verify report
|
|
assert.Equal(t, 3, report.Summary.Total, "Should have 3 test cases")
|
|
t.Logf("Total: %d, Passed: %d, Failed: %d",
|
|
report.Summary.Total, report.Summary.Passed, report.Summary.Failed)
|
|
|
|
// Check results for debugging
|
|
if report.Results != nil {
|
|
for _, r := range report.Results {
|
|
t.Logf(" [%s] Status: %s, Output: %v", r.ID, r.Status, r.Output)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestRunner_DirectMessage tests direct message mode
|
|
func TestRunner_DirectMessage(t *testing.T) {
|
|
test.Prepare(t, config.Conf)
|
|
defer test.Clean()
|
|
|
|
// Load agents
|
|
err := agent.Load(config.Conf)
|
|
require.NoError(t, err, "Failed to load agents")
|
|
|
|
// Test with direct message
|
|
opts := &agenttest.Options{
|
|
Input: "Hello, how are you?",
|
|
AgentID: "tests.simple-greeting",
|
|
Verbose: true,
|
|
InputMode: agenttest.InputModeMessage,
|
|
}
|
|
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
|
|
|
|
runner := agenttest.NewRunner(opts)
|
|
report, err := runner.Run()
|
|
|
|
require.NoError(t, err, "Runner should not return error")
|
|
require.NotNil(t, report, "Report should not be nil")
|
|
|
|
// Direct message mode returns a minimal report
|
|
assert.Equal(t, 1, report.Summary.Total, "Should have 1 test case")
|
|
assert.Equal(t, 1, report.Summary.Passed, "Direct message should pass")
|
|
}
|
|
|
|
// TestRunner_WithBeforeAfter tests before/after hooks
|
|
func TestRunner_WithBeforeAfter(t *testing.T) {
|
|
test.Prepare(t, config.Conf)
|
|
defer test.Clean()
|
|
|
|
// Load agents
|
|
err := agent.Load(config.Conf)
|
|
require.NoError(t, err, "Failed to load agents")
|
|
|
|
// Create a temporary JSONL file with test cases that use hooks
|
|
tmpDir := t.TempDir()
|
|
inputFile := filepath.Join(tmpDir, "inputs.jsonl")
|
|
|
|
// Note: hooks-test agent has env_test.ts with Before/After functions
|
|
testCases := `{"id": "hook-test-1", "input": "Hello", "assert": {"type": "contains", "value": "hello"}, "before": "env_test.Before", "after": "env_test.After"}`
|
|
|
|
err = os.WriteFile(inputFile, []byte(testCases), 0644)
|
|
require.NoError(t, err, "Failed to write test file")
|
|
|
|
// Run tests with hooks (using hooks-test agent which has the hook scripts)
|
|
opts := &agenttest.Options{
|
|
Input: inputFile,
|
|
AgentID: "tests.hooks-test",
|
|
Verbose: true,
|
|
InputMode: agenttest.InputModeFile,
|
|
}
|
|
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
|
|
|
|
runner := agenttest.NewRunner(opts)
|
|
report, err := runner.Run()
|
|
|
|
require.NoError(t, err, "Runner should not return error")
|
|
require.NotNil(t, report, "Report should not be nil")
|
|
|
|
t.Logf("Total: %d, Passed: %d, Failed: %d",
|
|
report.Summary.Total, report.Summary.Passed, report.Summary.Failed)
|
|
}
|
|
|
|
// TestRunner_Parallel tests parallel execution
|
|
func TestRunner_Parallel(t *testing.T) {
|
|
test.Prepare(t, config.Conf)
|
|
defer test.Clean()
|
|
|
|
// Load agents
|
|
err := agent.Load(config.Conf)
|
|
require.NoError(t, err, "Failed to load agents")
|
|
|
|
// Create a temporary JSONL file with multiple test cases
|
|
// Use regex for case-insensitive matching
|
|
tmpDir := t.TempDir()
|
|
inputFile := filepath.Join(tmpDir, "inputs.jsonl")
|
|
|
|
testCases := `{"id": "parallel-1", "input": "Hello", "assert": {"type": "regex", "value": "(?i)(hello|hi)"}}
|
|
{"id": "parallel-2", "input": "Hi", "assert": {"type": "regex", "value": "(?i)(hello|hi)"}}
|
|
{"id": "parallel-3", "input": "Hey", "assert": {"type": "regex", "value": "(?i)(hello|hi|hey)"}}
|
|
{"id": "parallel-4", "input": "Good day", "assert": {"type": "regex", "value": "(?i)(hello|good|day)"}}`
|
|
|
|
err = os.WriteFile(inputFile, []byte(testCases), 0644)
|
|
require.NoError(t, err, "Failed to write test file")
|
|
|
|
// Run tests in parallel
|
|
opts := &agenttest.Options{
|
|
Input: inputFile,
|
|
AgentID: "tests.simple-greeting",
|
|
Parallel: 2, // Run 2 tests in parallel
|
|
Verbose: true,
|
|
InputMode: agenttest.InputModeFile,
|
|
}
|
|
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
|
|
|
|
runner := agenttest.NewRunner(opts)
|
|
report, err := runner.Run()
|
|
|
|
require.NoError(t, err, "Runner should not return error")
|
|
require.NotNil(t, report, "Report should not be nil")
|
|
|
|
assert.Equal(t, 4, report.Summary.Total, "Should have 4 test cases")
|
|
t.Logf("Total: %d, Passed: %d, Failed: %d (parallel: 2)",
|
|
report.Summary.Total, report.Summary.Passed, report.Summary.Failed)
|
|
}
|
|
|
|
// TestRunner_FailFast tests fail-fast behavior
|
|
func TestRunner_FailFast(t *testing.T) {
|
|
test.Prepare(t, config.Conf)
|
|
defer test.Clean()
|
|
|
|
// Load agents
|
|
err := agent.Load(config.Conf)
|
|
require.NoError(t, err, "Failed to load agents")
|
|
|
|
// Create a temporary JSONL file with a failing test first
|
|
tmpDir := t.TempDir()
|
|
inputFile := filepath.Join(tmpDir, "inputs.jsonl")
|
|
|
|
// First test will fail (expects "impossible" which won't be in response)
|
|
testCases := `{"id": "fail-first", "input": "Hello", "assert": {"type": "contains", "value": "IMPOSSIBLE_STRING_12345"}}
|
|
{"id": "should-skip", "input": "Hi", "assert": {"type": "contains", "value": "hi"}}`
|
|
|
|
err = os.WriteFile(inputFile, []byte(testCases), 0644)
|
|
require.NoError(t, err, "Failed to write test file")
|
|
|
|
// Run tests with fail-fast
|
|
opts := &agenttest.Options{
|
|
Input: inputFile,
|
|
AgentID: "tests.simple-greeting",
|
|
FailFast: true,
|
|
Verbose: true,
|
|
InputMode: agenttest.InputModeFile,
|
|
}
|
|
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
|
|
|
|
runner := agenttest.NewRunner(opts)
|
|
report, err := runner.Run()
|
|
|
|
require.NoError(t, err, "Runner should not return error (fail-fast is not an error)")
|
|
require.NotNil(t, report, "Report should not be nil")
|
|
|
|
// With fail-fast, only the first test should run
|
|
assert.Equal(t, 1, report.Summary.Failed, "First test should fail")
|
|
// The second test might not run due to fail-fast
|
|
t.Logf("Total: %d, Passed: %d, Failed: %d (fail-fast enabled)",
|
|
report.Summary.Total, report.Summary.Passed, report.Summary.Failed)
|
|
}
|