From 5e64c78435c4a6f90dffacacd5735dc3cdf884e2 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 21 Dec 2025 10:44:59 +0800 Subject: [PATCH] Implement Script Testing Framework and Enhance Test Context Management - Introduced a new script testing mode to allow testing of agent handler scripts (hooks, tools, etc.) using a Go-like interface, enabling better unit testing of TypeScript/JavaScript code. - Enhanced the `LoadScripts` function to skip test files during script loading, ensuring only relevant scripts are processed. - Refactored the test context creation to support custom context configurations via a JSON file, allowing for flexible authorization and metadata management during tests. - Updated the test runner to handle script tests, including the ability to filter tests using regex patterns and manage custom context data. - Improved documentation to include details on script testing usage, input formats, and available assertions, enhancing developer experience and clarity. --- agent/assistant/scripts.go | 5 + agent/test/DESIGN.md | 736 +++++++++++++++++++++++++- agent/test/README.md | 223 ++++++-- agent/test/assert.go | 26 +- agent/test/context.go | 63 ++- agent/test/output.go | 46 ++ agent/test/resolver.go | 16 +- agent/test/runner.go | 32 ++ agent/test/script.go | 482 +++++++++++++++++ agent/test/script_assert.go | 1000 +++++++++++++++++++++++++++++++++++ agent/test/script_types.go | 170 ++++++ agent/test/types.go | 157 ++++++ cmd/README.md | 177 +++++-- cmd/agent/agent.go | 2 + cmd/agent/test.go | 32 +- 15 files changed, 3048 insertions(+), 119 deletions(-) create mode 100644 agent/test/script.go create mode 100644 agent/test/script_assert.go create mode 100644 agent/test/script_types.go diff --git a/agent/assistant/scripts.go b/agent/assistant/scripts.go index 574bd90c..1dde2bd3 100644 --- a/agent/assistant/scripts.go +++ b/agent/assistant/scripts.go @@ -75,6 +75,11 @@ func LoadScripts(srcDir string) (*hook.Script, map[string]*Script, error) { // Get relative path for determining if it's index relPath := strings.TrimPrefix(file, root+"/") + // Skip test files (*_test.ts, *_test.js) + if strings.HasSuffix(relPath, "_test.ts") || strings.HasSuffix(relPath, "_test.js") { + return nil + } + // Check if it's the root index.ts/js (hook script) // Only src/index.ts is the hook script, not src/foo/index.ts isRootIndex := relPath == "index.ts" || relPath == "index.js" diff --git a/agent/test/DESIGN.md b/agent/test/DESIGN.md index 5af29a12..36b14f6a 100644 --- a/agent/test/DESIGN.md +++ b/agent/test/DESIGN.md @@ -5,6 +5,8 @@ Agent Test Package provides a framework for testing AI agents with structured test cases. It supports batch testing, report generation, stability analysis, and CI integration. +Additionally, it supports **Script Testing** for testing Agent handler scripts (hooks, tools, etc.) with a Go-like testing interface. + ### Quick Start ```bash @@ -23,6 +25,12 @@ yao agent test -i assistants/keyword/tests/inputs.jsonl --runs 5 # Generate HTML report yao agent test -i assistants/keyword/tests/inputs.jsonl -r report.html -o report.html + +# Run script tests (test agent handler scripts) +yao agent test -i scripts.expense.setup -v + +# Run script tests with specific user/team context +yao agent test -i scripts.expense.tools -u admin -t ops-team -v ``` ## Usage @@ -77,7 +85,7 @@ yao agent test -i tests/inputs.jsonl \ ### Input Modes -The `-i` flag supports two input modes: +The `-i` flag supports three input modes: **1. JSONL File Mode** - Load test cases from a file: @@ -104,6 +112,33 @@ When using direct message mode: - Output is printed to stdout (or saved to `-o` if specified) - Useful for quick testing and debugging +**3. Script Test Mode** - Test agent handler scripts: + +```bash +# Run all tests in a script module +yao agent test -i scripts.expense.setup -v + +# Run with specific user/team context +yao agent test -i scripts.expense.tools -u admin -t ops-team + +# Run with timeout +yao agent test -i scripts.expense.setup --timeout 30s + +# Run specific tests by pattern (like go test -run) +yao agent test -i scripts.expense.setup -run TestSystemReady + +# Run tests matching a regex pattern +yao agent test -i scripts.expense.setup -run "TestSystem.*" +``` + +When using script test mode: + +- Input starts with `scripts.` prefix to indicate script testing +- Maps to the script file (e.g., `scripts.expense.setup` → `expense/src/setup_test.ts`) +- Automatically discovers and runs all `Test*` functions in the script +- Uses Go-like testing interface with assertions +- See [Script Testing](#script-testing) section for details + ### Default Output Path When `-o` is not specified and using JSONL file mode, the output file is automatically generated in the same directory as the input file: @@ -131,8 +166,10 @@ When using direct message mode without `-o`, output is printed to stdout. | `-c` | `--connector` | Override connector | agent default | `-c openai.gpt4` | | `-u` | `--user` | Test user ID (global override) | "test-user" | `-u admin` | | `-t` | `--team` | Test team ID (global override) | "test-team" | `-t ops-team` | +| | `--ctx` | Path to context JSON file | - | `--ctx tests/context.json` | | `-r` | `--reporter` | Custom reporter agent ID | - (use built-in) | `-r report.beautiful` | | | `--runs` | Number of runs for stability analysis | 1 | `--runs 5` | +| | `--run` | Regex pattern to filter tests | - | `--run "TestSystem.*"` | | | `--timeout` | Default timeout per test case | 5m | `--timeout 10m` | | | `--parallel` | Number of parallel test cases | 1 | `--parallel 4` | | `-v` | `--verbose` | Verbose output | false | `-v` | @@ -330,6 +367,437 @@ Create a custom reporter agent at `assistants/reporters/my-reporter/`: - Make it printable ``` +## Script Testing + +Script Testing allows you to test Agent handler scripts (hooks, tools, setup functions, etc.) using a Go-like testing interface. This is useful for unit testing individual functions in your agent's TypeScript/JavaScript code. + +### Quick Start + +```bash +# Run script tests +yao agent test -i scripts.expense.setup -v + +# With user/team context +yao agent test -i scripts.expense.setup -u admin -t ops-team -v + +# With timeout +yao agent test -i scripts.expense.setup --timeout 30s -v +``` + +### Script Resolution + +The `scripts.` prefix indicates script test mode. The script is resolved as follows: + +| Input | Script Path | Test File | +| ----------------------- | ---------------------- | --------------------------- | +| `scripts.expense.setup` | `expense/src/setup.ts` | `expense/src/setup_test.ts` | +| `scripts.expense.tools` | `expense/src/tools.ts` | `expense/src/tools_test.ts` | +| `scripts.keyword.index` | `keyword/src/index.ts` | `keyword/src/index_test.ts` | + +The test file naming convention is `{module}_test.ts` (similar to Go's `_test.go` convention). + +### Test Function Signature + +Test functions must follow this signature: + +```typescript +function TestFunctionName(t: testing.T, ctx: agent.Context) { + // Test logic here +} +``` + +**Requirements:** + +- Function name must start with `Test` (case-sensitive) +- First parameter `t` is the testing object with assertions +- Second parameter `ctx` is the agent context (same as used in hooks/tools) +- Functions not starting with `Test` are ignored (can be used as helpers) + +### Example Test File + +```typescript +// setup_test.ts +// @ts-nocheck + +// Test the SystemReady function +function TestSystemReady(t: testing.T, ctx: agent.Context) { + const { assert } = t; + + // Call the function being tested + const result = SystemReady(ctx); + + // Assert the result + assert.True(result, "SystemReady should return true"); +} + +// Test error case +function TestSystemReadyWithInvalidContext(t: testing.T, ctx: agent.Context) { + const { assert } = t; + + // Modify context to simulate error condition + ctx.User = null; + + const result = SystemReady(ctx); + assert.False(result, "SystemReady should return false when user is null"); +} + +// Helper function (not a test - doesn't start with "Test") +function createMockData() { + return { id: 1, name: "test" }; +} + +// Test with helper +function TestSetupWithMockData(t: testing.T, ctx: agent.Context) { + const { assert } = t; + const mockData = createMockData(); + + const result = Setup(ctx, mockData); + assert.NotNil(result, "Setup should return a result"); + assert.Equal(result.id, 1, "Result ID should match"); +} +``` + +### Testing Object (`t`) + +The `t` parameter provides the testing interface: + +```typescript +interface testing.T { + // Assertions object + assert: testing.Assert; + + // Test metadata + name: string; // Current test function name + failed: boolean; // Whether the test has failed + + // Logging (output appears in test report) + log(...args: any[]): void; // Log info message + error(...args: any[]): void; // Log error message + + // Control flow + skip(reason?: string): void; // Skip this test + fail(reason?: string): void; // Mark test as failed + fatal(reason?: string): void; // Mark as failed and stop execution +} +``` + +### Assertions (`t.assert`) + +The `assert` object provides assertion methods: + +| Method | Description | +| ----------------------------------- | ---------------------------------- | +| `True(value, message?)` | Assert value is true | +| `False(value, message?)` | Assert value is false | +| `Equal(actual, expected, message?)` | Assert deep equality | +| `NotEqual(actual, expected, msg?)` | Assert not equal | +| `Nil(value, message?)` | Assert value is null/undefined | +| `NotNil(value, message?)` | Assert value is not null/undefined | +| `Contains(str, substr, message?)` | Assert string contains substring | +| `NotContains(str, substr, msg?)` | Assert string does not contain | +| `Len(value, length, message?)` | Assert array/string length | +| `Greater(a, b, message?)` | Assert a > b | +| `GreaterOrEqual(a, b, message?)` | Assert a >= b | +| `Less(a, b, message?)` | Assert a < b | +| `LessOrEqual(a, b, message?)` | Assert a <= b | +| `Error(err, message?)` | Assert err is an error | +| `NoError(err, message?)` | Assert err is null/undefined | +| `Panic(fn, message?)` | Assert function throws | +| `NoPanic(fn, message?)` | Assert function does not throw | +| `Match(value, pattern, message?)` | Assert value matches regex | +| `NotMatch(value, pattern, msg?)` | Assert value does not match regex | +| `JSONPath(obj, path, expected, m?)` | Assert JSON path value | +| `Type(value, typeName, message?)` | Assert value type | + +### Agent Context (`ctx`) + +The `ctx` parameter is the same `agent.Context` used in agent hooks and tools: + +```typescript +interface agent.Context { + // User information (from -u flag or default) + User: { + ID: string; + Name?: string; + }; + + // Team information (from -t flag or default) + Team: { + ID: string; + Name?: string; + }; + + // Locale (default: "en-us") + Locale: string; + + // Client information + Client: { + Type: string; // "test" + IP: string; // "127.0.0.1" + }; + + // Metadata (can be set via test case) + Metadata: Record; + + // Chat/Session ID + ChatID: string; + + // Assistant ID (resolved from script path) + AssistantID: string; +} +``` + +### Script Test Output + +Script test results are reported in the same format as agent tests: + +``` +═══════════════════════════════════════════════════════════════════════════════ + Script Test: scripts.expense.setup +═══════════════════════════════════════════════════════════════════════════════ + Script: expense/src/setup_test.ts + Tests: 3 functions + User: test-user + Team: test-team +─────────────────────────────────────────────────────────────────────────────── + Running Tests +─────────────────────────────────────────────────────────────────────────────── +► [TestSystemReady] ... + ✓ PASSED (12ms) + +► [TestSystemReadyWithInvalidContext] ... + ✓ PASSED (8ms) + +► [TestSetupWithMockData] ... + ✗ FAILED (15ms) + └─ assertion failed: Result ID should match + expected: 1 + actual: 2 + +═══════════════════════════════════════════════════════════════════════════════ + Summary: 2 passed, 1 failed, 0 skipped (35ms) +═══════════════════════════════════════════════════════════════════════════════ +``` + +### Script Test Options + +Script tests support the following command line options: + +| Flag | Description | Default | Example | +| ------------- | -------------------------------- | ----------- | -------------------- | +| `-u` | User ID for context | "test-user" | `-u admin` | +| `-t` | Team ID for context | "test-team" | `-t ops-team` | +| `--ctx` | Path to context JSON file | - | `--ctx context.json` | +| `-v` | Verbose output | false | `-v` | +| `--run` | Regex to filter tests | - | `--run "TestSystem"` | +| `--timeout` | Timeout per test function | 30s | `--timeout 1m` | +| `--fail-fast` | Stop on first failure | false | `--fail-fast` | +| `-o` | Output file for report | stdout | `-o report.json` | +| `-r` | Reporter agent for custom report | - | `-r report.html` | + +The `--run` flag accepts a Go-style regex pattern to filter which tests to run: + +```bash +# Run only TestSystemReady +yao agent test -i scripts.expense.setup --run TestSystemReady + +# Run all tests starting with "TestSystem" +yao agent test -i scripts.expense.setup --run "TestSystem.*" + +# Run tests containing "Error" +yao agent test -i scripts.expense.setup --run ".*Error.*" +``` + +### Custom Context Configuration + +The `--ctx` flag allows you to provide a JSON file with custom context configuration, giving full control over authorization data, metadata, and client information: + +```bash +# Use custom context file +yao agent test -i scripts.expense.setup --ctx tests/context.json -v +``` + +**Context JSON Format:** + +```json +{ + "authorized": { + "sub": "user-12345", + "client_id": "my-app", + "scope": "read write", + "session_id": "sess-abc123", + "user_id": "admin", + "team_id": "team-001", + "tenant_id": "acme-corp", + "remember_me": false, + "constraints": { + "owner_only": false, + "creator_only": false, + "editor_only": false, + "team_only": true, + "extra": { + "department": "engineering", + "region": "us-west" + } + } + }, + "metadata": { + "request_id": "req-123", + "trace_id": "trace-456", + "custom_field": "custom_value" + }, + "client": { + "type": "web", + "user_agent": "Mozilla/5.0", + "ip": "192.168.1.100" + }, + "locale": "zh-cn", + "referer": "https://example.com/dashboard" +} +``` + +**Field Descriptions:** + +| Field | Description | +| -------------------------- | --------------------------------------------------- | +| `authorized.sub` | Subject identifier (JWT sub claim) | +| `authorized.client_id` | OAuth client ID | +| `authorized.scope` | Access scope | +| `authorized.session_id` | Session identifier | +| `authorized.user_id` | User identifier (overrides -u flag) | +| `authorized.team_id` | Team identifier (overrides -t flag) | +| `authorized.tenant_id` | Tenant identifier | +| `authorized.remember_me` | Remember me flag | +| `authorized.constraints` | Data access constraints (set by ACL enforcement) | +| `constraints.owner_only` | Only access owner's data | +| `constraints.creator_only` | Only access creator's data | +| `constraints.editor_only` | Only access editor's data | +| `constraints.team_only` | Only access team's data (filter by team_id) | +| `constraints.extra` | User-defined constraints (department, region, etc.) | +| `metadata` | Custom metadata passed to context | +| `client.type` | Client type (web, mobile, test, etc.) | +| `client.user_agent` | Client user agent string | +| `client.ip` | Client IP address | +| `locale` | Locale setting (e.g., "en-us", "zh-cn") | +| `referer` | Request referer URL | + +**Priority:** When both `-u`/`-t` flags and `--ctx` file are provided, the context file values take precedence. + +### Script Test Report Format + +When using `-o` to save results: + +```json +{ + "type": "script_test", + "script": "scripts.expense.setup", + "script_path": "expense/src/setup_test.ts", + "summary": { + "total": 3, + "passed": 2, + "failed": 1, + "skipped": 0, + "duration_ms": 35 + }, + "environment": { + "user_id": "test-user", + "team_id": "test-team", + "locale": "en-us" + }, + "results": [ + { + "name": "TestSystemReady", + "status": "passed", + "duration_ms": 12, + "logs": [] + }, + { + "name": "TestSystemReadyWithInvalidContext", + "status": "passed", + "duration_ms": 8, + "logs": [] + }, + { + "name": "TestSetupWithMockData", + "status": "failed", + "duration_ms": 15, + "error": "assertion failed: Result ID should match", + "assertion": { + "type": "Equal", + "expected": 1, + "actual": 2, + "message": "Result ID should match" + }, + "logs": [] + } + ], + "metadata": { + "started_at": "2024-12-17T10:00:00Z", + "completed_at": "2024-12-17T10:00:00Z", + "version": "0.10.5" + } +} +``` + +### Best Practices + +1. **Naming Convention**: Use descriptive test names that explain what's being tested + + - Good: `TestSystemReadyWithValidUser`, `TestSetupReturnsErrorOnMissingConfig` + - Bad: `Test1`, `TestIt` + +2. **One Assertion Per Concept**: Each test should verify one behavior + + ```typescript + // Good: Focused tests + function TestSetupCreatesDatabase(t, ctx) { ... } + function TestSetupInitializesCache(t, ctx) { ... } + + // Bad: Testing too many things + function TestSetup(t, ctx) { + // tests database, cache, config, etc. + } + ``` + +3. **Use Helper Functions**: Extract common setup logic + + ```typescript + function setupTestContext(ctx) { + ctx.Metadata.testMode = true; + return ctx; + } + + function TestFeatureA(t, ctx) { + ctx = setupTestContext(ctx); + // ... + } + ``` + +4. **Test Error Cases**: Don't just test happy paths + + ```typescript + function TestSetupWithMissingConfig(t, ctx) { + const { assert } = t; + ctx.Metadata.config = null; + + const result = Setup(ctx); + assert.Error(result.error, "Should return error for missing config"); + } + ``` + +5. **Clean Up**: If your test modifies global state, clean up after + ```typescript + function TestWithGlobalState(t, ctx) { + const originalValue = GlobalConfig.value; + try { + GlobalConfig.value = "test"; + // ... test logic + } finally { + GlobalConfig.value = originalValue; + } + } + ``` + ## Input Format (JSONL) Each line in the input file is a JSON object with the following structure: @@ -632,8 +1100,13 @@ agent/test/ ├── runner.go # Test runner implementation ├── loader.go # Test case loader ├── resolver.go # Agent resolver -├── environment.go # Test environment setup -├── stability.go # Stability analysis +├── context.go # Test context creation +├── assert.go # Assertion implementation +├── input.go # Input parsing +├── output.go # Output formatting +├── script.go # Script test runner (NEW) +├── script_types.go # Script test types (NEW) +├── script_assert.go # Script assertion bindings (NEW) └── reporter/ ├── json.go # JSON reporter ├── html.go # HTML reporter @@ -665,7 +1138,74 @@ Executes test cases against an agent: - Executes each test case (optionally multiple runs) - Collects results and stability metrics -### 5. Reporter +### 5. ScriptRunner (NEW) + +Executes script tests for agent handler scripts: + +- Resolves script path from `scripts.` prefix +- Discovers `Test*` functions in the script +- Creates test context with environment +- Executes each test function with testing object and context +- Collects results and generates report + +### 6. ScriptTestCase (NEW) + +Represents a single script test function: + +```go +type ScriptTestCase struct { + Name string // Function name (e.g., "TestSystemReady") + Function string // Full function reference +} +``` + +### 7. ScriptTestResult (NEW) + +Represents the result of running a script test function: + +```go +type ScriptTestResult struct { + Name string `json:"name"` + Status Status `json:"status"` + DurationMs int64 `json:"duration_ms"` + Error string `json:"error,omitempty"` + Assertion *AssertionInfo `json:"assertion,omitempty"` + Logs []string `json:"logs,omitempty"` +} + +type AssertionInfo struct { + Type string `json:"type"` + Expected interface{} `json:"expected,omitempty"` + Actual interface{} `json:"actual,omitempty"` + Message string `json:"message,omitempty"` +} +``` + +### 8. ScriptTestReport (NEW) + +Represents the complete script test report: + +```go +type ScriptTestReport struct { + Type string `json:"type"` // "script_test" + Script string `json:"script"` + ScriptPath string `json:"script_path"` + Summary *ScriptTestSummary `json:"summary"` + Environment *Environment `json:"environment"` + Results []*ScriptTestResult `json:"results"` + Metadata *ReportMetadata `json:"metadata"` +} + +type ScriptTestSummary struct { + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Skipped int `json:"skipped"` + DurationMs int64 `json:"duration_ms"` +} +``` + +### 9. Reporter Generates reports in various formats. The format is determined by the `-o` file extension: @@ -713,7 +1253,8 @@ This allows for fully customizable report generation using AI agents ```go type Options struct { // Input/Output - InputFile string // Path to inputs.jsonl + Input string // Input source: file path, message, or scripts.xxx + InputMode InputMode // Auto-detected: file, message, or script OutputFile string // Path to output report // Agent Selection @@ -737,6 +1278,186 @@ type Options struct { Verbose bool // Verbose output FailFast bool // Stop on first failure } + +// InputMode represents the input mode for test cases +type InputMode string + +const ( + InputModeFile InputMode = "file" // JSONL file input + InputModeMessage InputMode = "message" // Direct message input + InputModeScript InputMode = "script" // Script test mode (NEW) +) +``` + +### Input Mode Detection + +The input mode is automatically detected based on the input value: + +| Input Pattern | Mode | Description | +| ----------------- | --------- | -------------------------- | +| `scripts.xxx.yyy` | `script` | Script test mode | +| `*.jsonl` | `file` | JSONL file mode | +| `path/to/file` | `file` | File path (if file exists) | +| `"any text"` | `message` | Direct message mode | + +```go +func DetectInputMode(input string) InputMode { + // Check for script test prefix + if strings.HasPrefix(input, "scripts.") { + return InputModeScript + } + + // Check if it's a file path + if strings.HasSuffix(input, ".jsonl") || fileExists(input) { + return InputModeFile + } + + // Default to message mode + return InputModeMessage +} +``` + +## Script Testing Implementation + +### Script Resolution + +```go +// ResolveScript resolves the script path from scripts.xxx.yyy format +func ResolveScript(input string) (*ScriptInfo, error) { + // Remove "scripts." prefix + path := strings.TrimPrefix(input, "scripts.") + + // Split into parts: "expense.setup" -> ["expense", "setup"] + parts := strings.Split(path, ".") + if len(parts) < 2 { + return nil, fmt.Errorf("invalid script path: %s", input) + } + + // Build paths + // assistantDir: expense + // moduleName: setup + // scriptPath: expense/src/setup.ts + // testPath: expense/src/setup_test.ts + assistantDir := parts[0] + moduleName := parts[1] + + return &ScriptInfo{ + ID: input, + Assistant: assistantDir, + Module: moduleName, + ScriptPath: filepath.Join(assistantDir, "src", moduleName+".ts"), + TestPath: filepath.Join(assistantDir, "src", moduleName+"_test.ts"), + }, nil +} +``` + +### Test Function Discovery + +Test functions are discovered by scanning the script for functions starting with `Test`: + +```go +// DiscoverTests finds all Test* functions in the script +func DiscoverTests(scriptPath string) ([]*ScriptTestCase, error) { + // Use the JavaScript runtime to list exported functions + // Filter for functions starting with "Test" + // Return list of test cases +} +``` + +### Testing Object Binding + +The `testing.T` object is provided to test functions via JavaScript runtime binding: + +```go +// TestingT represents the testing object passed to test functions +type TestingT struct { + name string + failed bool + skipped bool + logs []string + assert *AssertObject +} + +// AssertObject provides assertion methods +type AssertObject struct { + t *TestingT +} + +func (a *AssertObject) True(value bool, message ...string) { + if !value { + a.t.fail(formatMessage("expected true, got false", message)) + } +} + +func (a *AssertObject) Equal(actual, expected interface{}, message ...string) { + if !reflect.DeepEqual(actual, expected) { + a.t.fail(formatMessage( + fmt.Sprintf("expected %v, got %v", expected, actual), + message, + )) + } +} + +// ... other assertion methods +``` + +### Script Execution Flow + +``` +1. Parse input: "scripts.expense.setup" +2. Resolve script info: + - TestPath: expense/src/setup_test.ts + - ScriptPath: expense/src/setup.ts +3. Discover test functions: [TestSystemReady, TestSetupWithMockData, ...] +4. For each test function: + a. Create testing.T object + b. Create agent.Context with environment + c. Execute: TestFunction(t, ctx) + d. Collect result (passed/failed/skipped) +5. Generate report +``` + +### Integration with Existing Runner + +```go +func (r *Executor) Run() (*Report, error) { + switch r.opts.InputMode { + case InputModeScript: + return r.RunScriptTests() + case InputModeMessage: + return r.RunDirect() + default: + return r.RunTests() + } +} + +func (r *Executor) RunScriptTests() (*Report, error) { + // 1. Resolve script + scriptInfo, err := ResolveScript(r.opts.Input) + if err != nil { + return nil, err + } + + // 2. Discover tests + tests, err := DiscoverTests(scriptInfo.TestPath) + if err != nil { + return nil, err + } + + // 3. Run each test + results := make([]*ScriptTestResult, 0, len(tests)) + for _, tc := range tests { + result := r.runScriptTest(tc, scriptInfo) + results = append(results, result) + + if r.opts.FailFast && result.Status == StatusFailed { + break + } + } + + // 4. Generate report + return r.buildScriptReport(scriptInfo, results), nil +} ``` ## Exit Codes @@ -785,3 +1506,8 @@ The command exits with code 1 if any tests fail, making it easy to integrate wit 5. **Diff Reports**: Compare results between runs 6. **Flaky Test Detection**: Automatic identification of unstable tests 7. **Test Prioritization**: Run most important/failing tests first +8. **Script Test Enhancements**: + - Parallel script test execution + - Setup/Teardown hooks (`TestMain`, `BeforeEach`, `AfterEach`) + - Mocking utilities for external dependencies + - Code coverage for TypeScript/JavaScript scripts diff --git a/agent/test/README.md b/agent/test/README.md index d4706a7c..d23d7efc 100644 --- a/agent/test/README.md +++ b/agent/test/README.md @@ -4,6 +4,8 @@ A testing framework for Yao AI agents with support for assertions, stability ana ## Quick Start +### Agent Tests + ```bash # Test with direct message (auto-detect agent from current directory) cd assistants/keyword @@ -22,9 +24,22 @@ yao agent test -i tests/inputs.jsonl -o report.html yao agent test -i tests/inputs.jsonl --runs 5 ``` +### Script Tests + +```bash +# Test agent handler scripts (hooks, tools, setup functions) +yao agent test -i scripts.expense.setup -v + +# Run specific tests with regex filter +yao agent test -i scripts.expense.setup --run "TestSystemReady" -v + +# Run with custom context (authorization, metadata) +yao agent test -i scripts.expense.setup --ctx tests/context.json -v +``` + ## Input Modes -The `-i` flag supports two input modes: +The `-i` flag supports three input modes: ### 1. JSONL File Mode @@ -51,22 +66,101 @@ yao agent test -i "Hello" -n workers.system.keyword Output is printed to stdout (or saved to `-o` if specified). +### 3. Script Test Mode + +Test agent handler scripts (hooks, tools, setup functions): + +```bash +# Run all tests in a script module +yao agent test -i scripts.expense.setup -v + +# Run specific tests with filtering +yao agent test -i scripts.expense.setup --run "TestSystemReady" + +# Run with custom context +yao agent test -i scripts.expense.setup --ctx tests/context.json -v +``` + +Script test input format: `scripts..` (e.g., `scripts.expense.setup` → `assistants/expense/src/setup_test.ts`). + +**Writing Test Scripts:** + +Test scripts should be placed alongside the source files with `_test.ts` or `_test.js` suffix: + +``` +assistants/expense/src/ +├── setup.ts # Source file +├── setup_test.ts # Test file +├── tools.ts +└── tools_test.ts +``` + +Test functions must follow the naming convention `Test*` and accept `(t: testing.T, ctx: agent.Context)`: + +```typescript +// assistants/expense/src/setup_test.ts +import { SystemReady } from "./setup"; + +// Test function signature: function Test*(t: testing.T, ctx: agent.Context) +export function TestSystemReady(t: testing.T, ctx: agent.Context) { + const result = SystemReady(ctx); + + // Use t.assert for assertions + t.assert.True(result.success, "SystemReady should succeed"); + t.assert.Equal(result.status, "ready", "Status should be ready"); + t.assert.NotNil(result.data, "Data should not be nil"); +} + +export function TestSystemReadyError(t: testing.T, ctx: agent.Context) { + // Access context properties + console.log("Testing with user:", ctx.authorized.user_id); + + // Test error handling + const result = SystemReady(ctx); + t.assert.False(result.error, "Should not have error"); +} +``` + +**Available Assertions:** + +| Method | Description | +| -------------------------------- | ------------------------------ | +| `t.assert.True(value, msg)` | Assert value is true | +| `t.assert.False(value, msg)` | Assert value is false | +| `t.assert.Equal(a, b, msg)` | Assert a equals b | +| `t.assert.NotEqual(a, b, msg)` | Assert a not equals b | +| `t.assert.Nil(value, msg)` | Assert value is null/undefined | +| `t.assert.NotNil(value, msg)` | Assert value is not nil | +| `t.assert.Contains(s, sub, msg)` | Assert string contains substr | +| `t.assert.Len(arr, n, msg)` | Assert array/string length | + +**Test Control:** + +| Method | Description | +| -------------- | ---------------------------- | +| `t.Log(msg)` | Log a message | +| `t.Error(msg)` | Mark test as failed with msg | +| `t.Fatal(msg)` | Mark failed and stop test | +| `t.Skip(msg)` | Skip this test | + ## Command Line Options -| Flag | Description | Default | -| ------------- | ---------------------------------------- | -------------------------- | -| `-i` | Input: JSONL file path or direct message | (required) | -| `-o` | Output file path | `output-{timestamp}.jsonl` | -| `-n` | Agent ID (optional, auto-detected) | auto-detect | -| `-c` | Override connector | agent default | -| `-u` | Test user ID | `test-user` | -| `-t` | Test team ID | `test-team` | -| `-r` | Reporter agent ID | built-in | -| `--runs` | Runs per test (stability analysis) | 1 | -| `--timeout` | Timeout per test | 5m | -| `--parallel` | Parallel test cases | 1 | -| `-v` | Verbose output | false | -| `--fail-fast` | Stop on first failure | false | +| Flag | Description | Default | +| ------------- | -------------------------------------------------- | -------------------------- | +| `-i` | Input: JSONL file path, message, or script ID | (required) | +| `-o` | Output file path | `output-{timestamp}.jsonl` | +| `-n` | Agent ID (optional, auto-detected) | auto-detect | +| `-c` | Override connector | agent default | +| `-u` | Test user ID | `test-user` | +| `-t` | Test team ID | `test-team` | +| `--ctx` | Path to context JSON file for custom authorization | - | +| `-r` | Reporter agent ID | built-in | +| `--runs` | Runs per test (stability analysis) | 1 | +| `--run` | Regex pattern to filter which tests to run | - | +| `--timeout` | Timeout per test | 5m | +| `--parallel` | Parallel test cases | 1 | +| `-v` | Verbose output | false | +| `--fail-fast` | Stop on first failure | false | ## Agent Resolution @@ -119,24 +213,24 @@ Each line is a JSON object: The `options` field allows per-test-case configuration that maps to `context.Options`: -| Field | Type | Description | -| ------------------------ | ------- | -------------------------------------------------- | -| `connector` | string | Override connector (e.g., `"deepseek.v3"`) | -| `mode` | string | Agent mode (default: `"chat"`) | -| `search` | bool | Enable/disable search mode (default: `true`) | -| `disable_global_prompts` | bool | Temporarily disable global prompts | -| `metadata` | map | Custom data passed to hooks (e.g., scenario) | -| `skip` | object | Skip configuration (see below) | +| Field | Type | Description | +| ------------------------ | ------ | -------------------------------------------- | +| `connector` | string | Override connector (e.g., `"deepseek.v3"`) | +| `mode` | string | Agent mode (default: `"chat"`) | +| `search` | bool | Enable/disable search mode (default: `true`) | +| `disable_global_prompts` | bool | Temporarily disable global prompts | +| `metadata` | map | Custom data passed to hooks (e.g., scenario) | +| `skip` | object | Skip configuration (see below) | #### Options.skip -| Field | Type | Description | -| --------- | ---- | ------------------------ | -| `history` | bool | Skip history loading | -| `trace` | bool | Skip trace logging | -| `output` | bool | Skip output to client | -| `keyword` | bool | Skip keyword extraction | -| `search` | bool | Skip auto search | +| Field | Type | Description | +| --------- | ---- | ----------------------- | +| `history` | bool | Skip history loading | +| `trace` | bool | Skip trace logging | +| `output` | bool | Skip output to client | +| `keyword` | bool | Skip keyword extraction | +| `search` | bool | Skip auto search | **Example with options:** @@ -146,10 +240,18 @@ The `options` field allows per-test-case configuration that maps to `context.Opt "input": "Query users with status active", "options": { "connector": "deepseek.v3", - "metadata": {"scenario": "filter"}, - "skip": {"trace": true} + "metadata": { + "scenario": "filter" + }, + "skip": { + "trace": true + } }, - "assert": {"type": "json_path", "path": "from", "value": "users"} + "assert": { + "type": "json_path", + "path": "from", + "value": "users" + } } ``` @@ -290,7 +392,17 @@ return { pass: true, message: "Validation passed" }; **Multiple expected values (OR logic):** ```jsonl -{"id": "T005", "assert": {"type": "json_path", "path": "error", "value": ["missing_schema", "missing_query"]}} +{ + "id": "T005", + "assert": { + "type": "json_path", + "path": "error", + "value": [ + "missing_schema", + "missing_query" + ] + } +} ``` This passes if `error` equals either `"missing_schema"` or `"missing_query"`. @@ -382,6 +494,9 @@ yao agent test -i tests/inputs.jsonl -o results.jsonl --fail-fast # Parse JSONL results cat results.jsonl | jq 'select(.type == "summary")' + +# Run script tests +yao agent test -i scripts.expense.setup --fail-fast ``` ### GitHub Actions Example @@ -394,6 +509,18 @@ cat results.jsonl | jq 'select(.type == "summary")' --runs 3 \ -o report.json +- name: Run Script Tests + run: | + yao agent test -i scripts.expense.setup -v + yao agent test -i scripts.expense.tools -v + +- name: Run Script Tests with Custom Context + run: | + yao agent test -i scripts.expense.setup \ + --ctx tests/context.json \ + --run "TestSystem.*" \ + -v + - name: Check Stability run: | jq -e '.results | all(.pass_rate >= 80)' report.json @@ -401,6 +528,8 @@ cat results.jsonl | jq 'select(.type == "summary")' ## Examples +### Agent Tests + ```bash # Quick development test (auto-detect agent) cd assistants/keyword @@ -440,6 +569,32 @@ yao agent test -i tests/inputs.jsonl \ -o report.html ``` +### Script Tests + +```bash +# Run all tests in a script module +yao agent test -i scripts.expense.setup -v + +# Run specific tests with regex filter +yao agent test -i scripts.expense.setup --run "TestSystemReady" + +# Run tests matching a pattern +yao agent test -i scripts.expense.setup --run "TestSystem.*" -v + +# Run with custom context (authorization, metadata, etc.) +yao agent test -i scripts.expense.setup --ctx tests/context.json -v + +# Run with specific user/team +yao agent test -i scripts.expense.setup -u admin -t ops-team -v + +# Combine options +yao agent test -i scripts.expense.setup \ + --ctx tests/context.json \ + --run "TestSystem.*" \ + --timeout 30s \ + -v +``` + ## Exit Codes | Code | Description | diff --git a/agent/test/assert.go b/agent/test/assert.go index 1ba82ddf..4ac7ef8d 100644 --- a/agent/test/assert.go +++ b/agent/test/assert.go @@ -255,13 +255,27 @@ func (a *Asserter) assertJSONPath(assertion *Assertion, output interface{}) *Ass result.Actual = actual // Compare expected value with actual value - // First, try direct comparison (handles both primitive values and arrays) - if validateOutput(actual, assertion.Value) { - result.Passed = true - result.Message = fmt.Sprintf("path '%s' equals expected value", assertion.Path) - } else { + // If expected is an array, check if actual matches ANY element (IN semantics) + if expectedArr, ok := assertion.Value.([]interface{}); ok { + // Check if actual is one of the expected values + for _, expectedItem := range expectedArr { + if validateOutput(actual, expectedItem) { + result.Passed = true + result.Message = fmt.Sprintf("path '%s' equals one of expected values", assertion.Path) + return result + } + } result.Passed = false - result.Message = fmt.Sprintf("path '%s': expected %v, got %v", assertion.Path, assertion.Value, actual) + result.Message = fmt.Sprintf("path '%s': expected one of %v, got %v", assertion.Path, assertion.Value, actual) + } else { + // Direct comparison for non-array expected values + if validateOutput(actual, assertion.Value) { + result.Passed = true + result.Message = fmt.Sprintf("path '%s' equals expected value", assertion.Path) + } else { + result.Passed = false + result.Message = fmt.Sprintf("path '%s': expected %v, got %v", assertion.Path, assertion.Value, actual) + } } return result diff --git a/agent/test/context.go b/agent/test/context.go index 244e7bd5..e3184bb5 100644 --- a/agent/test/context.go +++ b/agent/test/context.go @@ -13,11 +13,7 @@ import ( // but configurable via Environment func NewTestContext(chatID, assistantID string, env *Environment) *context.Context { // Build authorized info from environment - authorized := &types.AuthorizedInfo{ - Subject: env.UserID, - UserID: env.UserID, - TenantID: env.TeamID, - } + authorized := buildAuthorizedInfo(env) // Create context with standard initialization ctx := context.New(stdContext.Background(), authorized, chatID) @@ -34,6 +30,13 @@ func NewTestContext(chatID, assistantID string, env *Environment) *context.Conte ctx.IDGenerator = message.NewIDGenerator() ctx.Metadata = make(map[string]interface{}) + // Apply metadata from context config if available + if env.ContextConfig != nil && env.ContextConfig.Metadata != nil { + for k, v := range env.ContextConfig.Metadata { + ctx.Metadata[k] = v + } + } + // Initialize interrupt controller ctx.Interrupt = context.NewInterruptController() @@ -47,6 +50,56 @@ func NewTestContext(chatID, assistantID string, env *Environment) *context.Conte return ctx } +// buildAuthorizedInfo builds AuthorizedInfo from Environment +func buildAuthorizedInfo(env *Environment) *types.AuthorizedInfo { + authorized := &types.AuthorizedInfo{ + Subject: env.UserID, + UserID: env.UserID, + TenantID: env.TeamID, + } + + // Apply custom authorized config if available + if env.ContextConfig != nil && env.ContextConfig.Authorized != nil { + authCfg := env.ContextConfig.Authorized + + if authCfg.Sub != "" { + authorized.Subject = authCfg.Sub + } + if authCfg.ClientID != "" { + authorized.ClientID = authCfg.ClientID + } + if authCfg.Scope != "" { + authorized.Scope = authCfg.Scope + } + if authCfg.SessionID != "" { + authorized.SessionID = authCfg.SessionID + } + if authCfg.UserID != "" { + authorized.UserID = authCfg.UserID + } + if authCfg.TeamID != "" { + authorized.TeamID = authCfg.TeamID + } + if authCfg.TenantID != "" { + authorized.TenantID = authCfg.TenantID + } + authorized.RememberMe = authCfg.RememberMe + + // Apply constraints + if authCfg.Constraints != nil { + authorized.Constraints = types.DataConstraints{ + OwnerOnly: authCfg.Constraints.OwnerOnly, + CreatorOnly: authCfg.Constraints.CreatorOnly, + EditorOnly: authCfg.Constraints.EditorOnly, + TeamOnly: authCfg.Constraints.TeamOnly, + Extra: authCfg.Constraints.Extra, + } + } + } + + return authorized +} + // NewTestContextFromOptions creates a test context from test options and test case func NewTestContextFromOptions(chatID, assistantID string, opts *Options, tc *Case) *context.Context { // Get environment from test case (with options override) diff --git a/agent/test/output.go b/agent/test/output.go index 1b3fc51b..a49f1f6e 100644 --- a/agent/test/output.go +++ b/agent/test/output.go @@ -257,6 +257,52 @@ func (w *OutputWriter) DirectOutput(output interface{}) { } } +// ScriptTestSummary prints the script test summary +func (w *OutputWriter) ScriptTestSummary(summary *ScriptTestSummary, duration time.Duration) { + w.SubHeader("Summary") + + // Results + color.New(color.FgWhite).Printf(" Total: ") + fmt.Printf("%d\n", summary.Total) + + color.New(color.FgWhite).Printf(" Passed: ") + if summary.Passed > 0 { + color.New(color.FgGreen).Printf("%d\n", summary.Passed) + } else { + fmt.Printf("%d\n", summary.Passed) + } + + color.New(color.FgWhite).Printf(" Failed: ") + if summary.Failed > 0 { + color.New(color.FgRed).Printf("%d\n", summary.Failed) + } else { + fmt.Printf("%d\n", summary.Failed) + } + + if summary.Skipped > 0 { + color.New(color.FgWhite).Printf(" Skipped: ") + color.New(color.FgYellow).Printf("%d\n", summary.Skipped) + } + + // Pass rate + passRate := float64(0) + if summary.Total > 0 { + passRate = float64(summary.Passed) / float64(summary.Total) * 100 + } + color.New(color.FgWhite).Printf(" Pass Rate: ") + if passRate == 100 { + color.New(color.FgGreen, color.Bold).Printf("%.1f%%\n", passRate) + } else if passRate >= 80 { + color.New(color.FgYellow).Printf("%.1f%%\n", passRate) + } else { + color.New(color.FgRed).Printf("%.1f%%\n", passRate) + } + + // Duration + color.New(color.FgWhite).Printf(" Duration: ") + fmt.Printf("%s\n", formatDuration(duration)) +} + // StabilityResult prints stability analysis result for a test case func (w *OutputWriter) StabilityResult(sr *StabilityResult) { color.New(color.FgWhite).Printf(" [%s] ", sr.ID) diff --git a/agent/test/resolver.go b/agent/test/resolver.go index 9ccf31ee..4364b9f2 100644 --- a/agent/test/resolver.go +++ b/agent/test/resolver.go @@ -213,8 +213,16 @@ func DefaultOptions() *Options { } // DetectInputMode detects the input mode from the input string -// Returns InputModeFile if input looks like a file path, InputModeMessage otherwise +// Returns: +// - InputModeScript: if input starts with "scripts." +// - InputModeFile: if input ends with ".jsonl" or is an existing file +// - InputModeMessage: otherwise (direct message mode) func DetectInputMode(input string) InputMode { + // Check for script test prefix + if strings.HasPrefix(input, "scripts.") { + return InputModeScript + } + // If input ends with .jsonl or .json, treat as file if strings.HasSuffix(input, ".jsonl") || strings.HasSuffix(input, ".json") { return InputModeFile @@ -269,6 +277,12 @@ func MergeOptions(opts *Options, defaults *Options) *Options { if opts.ReporterID != "" { result.ReporterID = opts.ReporterID } + if opts.ContextFile != "" { + result.ContextFile = opts.ContextFile + } + if opts.Run != "" { + result.Run = opts.Run + } if opts.Verbose { result.Verbose = opts.Verbose } diff --git a/agent/test/runner.go b/agent/test/runner.go index 8065c81c..2394e1a4 100644 --- a/agent/test/runner.go +++ b/agent/test/runner.go @@ -33,6 +33,11 @@ func NewRunner(opts *Options) *Executor { // Run executes all test cases and returns a report func (r *Executor) Run() (*Report, error) { + // For script test mode, use script runner + if r.opts.InputMode == InputModeScript { + return r.RunScriptTests() + } + // For direct message mode, use simplified output (development mode) if r.opts.InputMode == InputModeMessage { return r.RunDirect() @@ -41,6 +46,33 @@ func (r *Executor) Run() (*Report, error) { return r.RunTests() } +// RunScriptTests executes script tests and returns a report +func (r *Executor) RunScriptTests() (*Report, error) { + scriptRunner := NewScriptRunner(r.opts) + scriptReport, err := scriptRunner.Run() + if err != nil { + return nil, err + } + + // Convert to standard report for unified output handling + report := scriptReport.ToReport() + + // Write output if specified + if r.opts.OutputFile != "" { + err = r.writeOutput(report) + if err != nil { + r.output.Error("Failed to write output: %s", err.Error()) + } else { + r.output.OutputFile(r.opts.OutputFile) + } + } + + // Print final result + r.output.FinalResult(!report.HasFailures()) + + return report, nil +} + // RunDirect executes a single direct message and outputs the result directly // This is optimized for development/debugging scenarios func (r *Executor) RunDirect() (*Report, error) { diff --git a/agent/test/script.go b/agent/test/script.go new file mode 100644 index 00000000..5e382340 --- /dev/null +++ b/agent/test/script.go @@ -0,0 +1,482 @@ +package test + +import ( + "fmt" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/yaoapp/gou/application" + v8 "github.com/yaoapp/gou/runtime/v8" + "github.com/yaoapp/gou/runtime/v8/bridge" + "github.com/yaoapp/yao/agent/context" + "rogchap.com/v8go" +) + +// ScriptRunner executes script tests +type ScriptRunner struct { + opts *Options + output *OutputWriter +} + +// NewScriptRunner creates a new script test runner +func NewScriptRunner(opts *Options) *ScriptRunner { + return &ScriptRunner{ + opts: opts, + output: NewOutputWriter(opts.Verbose), + } +} + +// ResolveScript resolves the script path from scripts.xxx.yyy format +func ResolveScript(input string) (*ScriptInfo, error) { + // Remove "scripts." prefix + path := strings.TrimPrefix(input, "scripts.") + + // Split into parts: "expense.setup" -> ["expense", "setup"] + parts := strings.Split(path, ".") + if len(parts) < 2 { + return nil, fmt.Errorf("invalid script path: %s (expected format: scripts.assistant.module)", input) + } + + // Build paths + // assistantDir: expense + // moduleName: setup + // scriptPath: expense/src/setup.ts (or assistants/expense/src/setup.ts) + // testPath: expense/src/setup_test.ts + assistantDir := parts[0] + moduleName := parts[1] + + // Try different path patterns + basePaths := []string{ + filepath.Join("assistants", assistantDir, "src"), + filepath.Join(assistantDir, "src"), + } + + var scriptPath, testPath string + for _, basePath := range basePaths { + // Check for TypeScript files first, then JavaScript + for _, ext := range []string{".ts", ".js"} { + candidateScript := filepath.Join(basePath, moduleName+ext) + candidateTest := filepath.Join(basePath, moduleName+"_test"+ext) + + // Check if test file exists + exists, err := application.App.Exists(candidateTest) + if err == nil && exists { + scriptPath = candidateScript + testPath = candidateTest + break + } + } + if testPath != "" { + break + } + } + + if testPath == "" { + return nil, fmt.Errorf("test file not found for %s (tried: %s)", input, strings.Join(basePaths, ", ")) + } + + return &ScriptInfo{ + ID: input, + Assistant: assistantDir, + Module: moduleName, + ScriptPath: scriptPath, + TestPath: testPath, + }, nil +} + +// DiscoverTests finds all Test* functions in the script +func DiscoverTests(scriptPath string) ([]*ScriptTestCase, error) { + // Read the script file + content, err := application.App.Read(scriptPath) + if err != nil { + return nil, fmt.Errorf("failed to read script: %w", err) + } + + // Parse the script to find Test* functions + // We use a simple regex-like approach to find function declarations + tests := make([]*ScriptTestCase, 0) + lines := strings.Split(string(content), "\n") + + for _, line := range lines { + line = strings.TrimSpace(line) + + // Match function declarations: function TestXxx( or export function TestXxx( + if strings.Contains(line, "function Test") { + // Extract function name + name := extractFunctionName(line) + if name != "" && strings.HasPrefix(name, "Test") { + tests = append(tests, &ScriptTestCase{ + Name: name, + Function: name, + }) + } + } + } + + return tests, nil +} + +// extractFunctionName extracts the function name from a line +func extractFunctionName(line string) string { + // Remove "export" prefix if present + line = strings.TrimPrefix(line, "export ") + line = strings.TrimSpace(line) + + // Match "function Name(" + if !strings.HasPrefix(line, "function ") { + return "" + } + + line = strings.TrimPrefix(line, "function ") + + // Find the opening parenthesis + idx := strings.Index(line, "(") + if idx == -1 { + return "" + } + + return strings.TrimSpace(line[:idx]) +} + +// filterTests filters test cases by a regex pattern (similar to go test -run) +func (r *ScriptRunner) filterTests(tests []*ScriptTestCase, pattern string) ([]*ScriptTestCase, error) { + re, err := regexp.Compile(pattern) + if err != nil { + return nil, err + } + + filtered := make([]*ScriptTestCase, 0) + for _, tc := range tests { + if re.MatchString(tc.Name) { + filtered = append(filtered, tc) + } + } + + return filtered, nil +} + +// Run executes all script tests and returns a report +func (r *ScriptRunner) Run() (*ScriptTestReport, error) { + startTime := time.Now() + + // Resolve script + scriptInfo, err := ResolveScript(r.opts.Input) + if err != nil { + return nil, err + } + + // Print header + r.output.Header("Script Test") + r.output.Info("Script: %s", scriptInfo.TestPath) + + // Discover tests + tests, err := DiscoverTests(scriptInfo.TestPath) + if err != nil { + return nil, err + } + + // Filter tests by -run pattern if specified + if r.opts.Run != "" { + tests, err = r.filterTests(tests, r.opts.Run) + if err != nil { + return nil, fmt.Errorf("invalid -run pattern: %w", err) + } + r.output.Info("Tests: %d functions (filtered by: %s)", len(tests), r.opts.Run) + } else { + r.output.Info("Tests: %d functions", len(tests)) + } + + if len(tests) == 0 { + r.output.Warning("No tests to run") + } + + // Load context config if specified + var ctxConfig *ContextConfig + if r.opts.ContextFile != "" { + var err error + ctxConfig, err = LoadContextConfig(r.opts.ContextFile) + if err != nil { + return nil, fmt.Errorf("failed to load context file: %w", err) + } + r.output.Info("Context: %s", r.opts.ContextFile) + } + + // Create environment with optional context config + var env *Environment + if ctxConfig != nil { + env = NewEnvironmentWithContext(r.opts.UserID, r.opts.TeamID, ctxConfig) + } else { + env = NewEnvironment(r.opts.UserID, r.opts.TeamID) + } + r.output.Info("User: %s", env.UserID) + r.output.Info("Team: %s", env.TeamID) + + // Load all scripts from src directory (including the test file) + // This ensures imports can be resolved properly + srcDir := filepath.Dir(scriptInfo.TestPath) + loadedCount, err := r.loadAllScripts(srcDir) + if err != nil { + return nil, fmt.Errorf("failed to load scripts: %w", err) + } + r.output.Info("Loaded: %d scripts", loadedCount) + + // Create report + report := &ScriptTestReport{ + Type: "script_test", + Script: scriptInfo.ID, + ScriptPath: scriptInfo.TestPath, + Summary: &ScriptTestSummary{Total: len(tests)}, + Environment: env, + Results: make([]*ScriptTestResult, 0, len(tests)), + Metadata: &ScriptTestMetadata{ + StartedAt: startTime, + }, + } + + // Run tests + r.output.SubHeader("Running Tests") + + for _, tc := range tests { + result := r.runScriptTest(tc, scriptInfo, env) + report.Results = append(report.Results, result) + + // Update summary + switch result.Status { + case StatusPassed: + report.Summary.Passed++ + case StatusFailed: + report.Summary.Failed++ + case StatusSkipped: + report.Summary.Skipped++ + } + + // Check fail-fast + if r.opts.FailFast && result.Status == StatusFailed { + r.output.Warning("Stopping due to --fail-fast") + break + } + } + + // Complete report + report.Summary.DurationMs = time.Since(startTime).Milliseconds() + report.Metadata.CompletedAt = time.Now() + + // Print summary + r.output.ScriptTestSummary(report.Summary, time.Since(startTime)) + + return report, nil +} + +// runScriptTest runs a single script test function +func (r *ScriptRunner) runScriptTest(tc *ScriptTestCase, scriptInfo *ScriptInfo, env *Environment) *ScriptTestResult { + r.output.TestStart(tc.Name, "", 1) + startTime := time.Now() + + result := &ScriptTestResult{ + Name: tc.Name, + Status: StatusPassed, + } + + // Create testing.T object + testingT := NewTestingT(tc.Name) + + // Create agent context + chatID := fmt.Sprintf("script-test-%s", tc.Name) + agentCtx := NewTestContext(chatID, scriptInfo.Assistant, env) + defer agentCtx.Release() + + // Execute the test function + err := r.executeTestFunction(tc, scriptInfo, testingT, agentCtx) + + duration := time.Since(startTime) + result.DurationMs = duration.Milliseconds() + result.Logs = testingT.Logs() + + if err != nil { + result.Status = StatusError + result.Error = err.Error() + r.output.TestResult(result.Status, duration) + r.output.TestError(result.Error) + return result + } + + if testingT.Skipped() { + result.Status = StatusSkipped + r.output.TestResult(result.Status, duration) + return result + } + + if testingT.Failed() { + result.Status = StatusFailed + errors := testingT.Errors() + if len(errors) > 0 { + result.Error = errors[0] + } + result.Assertion = testingT.AssertionInfo() + r.output.TestResult(result.Status, duration) + r.output.TestError(result.Error) + return result + } + + r.output.TestResult(result.Status, duration) + return result +} + +// loadAllScripts loads all scripts from the src directory +// This ensures that imports can be resolved properly +func (r *ScriptRunner) loadAllScripts(srcDir string) (int, error) { + count := 0 + + // Check if src directory exists + exists, err := application.App.Exists(srcDir) + if err != nil { + return 0, err + } + if !exists { + return 0, fmt.Errorf("src directory not found: %s", srcDir) + } + + // Walk through src directory to find all script files + exts := []string{"*.ts", "*.js"} + + err = application.App.Walk(srcDir, func(root, file string, isdir bool) error { + if isdir { + return nil + } + + // Get relative path + relPath := strings.TrimPrefix(file, root+"/") + + // Generate script ID from file path + scriptID := generateTestScriptID(file, root) + + // Load the script + _, err := v8.Load(file, scriptID) + if err != nil { + // Log warning but continue loading other scripts + if r.opts.Verbose { + r.output.Warning("Failed to load %s: %v", relPath, err) + } + return nil + } + + count++ + if r.opts.Verbose { + r.output.Verbose("Loaded: %s", relPath) + } + + return nil + }, exts...) + + if err != nil { + return count, fmt.Errorf("failed to walk src directory: %w", err) + } + + return count, nil +} + +// generateTestScriptID generates a script ID from file path for testing +func generateTestScriptID(filePath string, srcDir string) string { + // Normalize path separators + filePath = filepath.ToSlash(filePath) + srcDir = filepath.ToSlash(srcDir) + + // Remove src directory prefix + relPath := strings.TrimPrefix(filePath, srcDir+"/") + relPath = strings.TrimPrefix(relPath, "/") + + // Remove file extension + relPath = strings.TrimSuffix(relPath, filepath.Ext(relPath)) + + // Replace path separators with dots and add test prefix + scriptID := "test." + strings.ReplaceAll(relPath, "/", ".") + + return scriptID +} + +// executeTestFunction executes a single test function using V8 +func (r *ScriptRunner) executeTestFunction(tc *ScriptTestCase, scriptInfo *ScriptInfo, testingT *TestingT, agentCtx *context.Context) error { + // Get the test script (already loaded by loadAllScripts) + testScriptID := generateTestScriptID(scriptInfo.TestPath, filepath.Dir(scriptInfo.TestPath)) + script, ok := v8.Scripts[testScriptID] + if !ok { + return fmt.Errorf("test script not found: %s (id: %s)", scriptInfo.TestPath, testScriptID) + } + + // Create a new script context + scriptCtx, err := script.NewContext("", nil) + if err != nil { + return fmt.Errorf("failed to create script context: %w", err) + } + defer scriptCtx.Close() + + // Get the V8 context + v8ctx := scriptCtx.Context + + // Create testing.T JavaScript object + testingTObj, err := NewTestingTObject(v8ctx, testingT) + if err != nil { + return fmt.Errorf("failed to create testing.T object: %w", err) + } + + // Create agent context JavaScript object + agentCtxObj, err := agentCtx.JsValue(v8ctx) + if err != nil { + return fmt.Errorf("failed to create agent context object: %w", err) + } + + // Get the test function + global := v8ctx.Global() + fnValue, err := global.Get(tc.Function) + if err != nil { + return fmt.Errorf("failed to get test function %s: %w", tc.Function, err) + } + + if !fnValue.IsFunction() { + return fmt.Errorf("test function %s is not a function", tc.Function) + } + + fn, err := fnValue.AsFunction() + if err != nil { + return fmt.Errorf("failed to convert to function: %w", err) + } + + // Call the test function with (t, ctx) + _, err = fn.Call(global, testingTObj, agentCtxObj) + if err != nil { + // Check if this is an assertion failure or a real error + if testingT.Failed() { + // Assertion failure - already recorded + return nil + } + return fmt.Errorf("test function error: %w", err) + } + + return nil +} + +// RegisterTestingGlobals registers testing-related global functions for V8 +// This is called once during initialization +func RegisterTestingGlobals() { + v8.RegisterFunction("__testing_log", testingLogEmbed) +} + +// testingLogEmbed provides a console.log-like function for tests +func testingLogEmbed(iso *v8go.Isolate) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + parts := make([]string, len(args)) + for i, arg := range args { + goVal, err := bridge.GoValue(arg, info.Context()) + if err != nil { + parts[i] = arg.String() + } else { + parts[i] = fmt.Sprintf("%v", goVal) + } + } + fmt.Println(strings.Join(parts, " ")) + return v8go.Undefined(iso) + }) +} diff --git a/agent/test/script_assert.go b/agent/test/script_assert.go new file mode 100644 index 00000000..c0cc0245 --- /dev/null +++ b/agent/test/script_assert.go @@ -0,0 +1,1000 @@ +package test + +import ( + "encoding/json" + "fmt" + "reflect" + "regexp" + "strings" + "sync" + + "github.com/yaoapp/gou/runtime/v8/bridge" + "rogchap.com/v8go" +) + +// TestingT represents the testing object passed to test functions +// It provides assertion methods and test control flow +type TestingT struct { + mu sync.Mutex + name string + failed bool + skipped bool + logs []string + errors []string + + // Assertion failure details (for the first failure) + assertionInfo *ScriptAssertionInfo +} + +// NewTestingT creates a new TestingT instance +func NewTestingT(name string) *TestingT { + return &TestingT{ + name: name, + logs: make([]string, 0), + errors: make([]string, 0), + } +} + +// Name returns the test name +func (t *TestingT) Name() string { + return t.name +} + +// Failed returns whether the test has failed +func (t *TestingT) Failed() bool { + t.mu.Lock() + defer t.mu.Unlock() + return t.failed +} + +// Skipped returns whether the test was skipped +func (t *TestingT) Skipped() bool { + t.mu.Lock() + defer t.mu.Unlock() + return t.skipped +} + +// Logs returns all log messages +func (t *TestingT) Logs() []string { + t.mu.Lock() + defer t.mu.Unlock() + return append([]string{}, t.logs...) +} + +// Errors returns all error messages +func (t *TestingT) Errors() []string { + t.mu.Lock() + defer t.mu.Unlock() + return append([]string{}, t.errors...) +} + +// AssertionInfo returns the first assertion failure info +func (t *TestingT) AssertionInfo() *ScriptAssertionInfo { + t.mu.Lock() + defer t.mu.Unlock() + return t.assertionInfo +} + +// log adds a log message +func (t *TestingT) log(msg string) { + t.mu.Lock() + defer t.mu.Unlock() + t.logs = append(t.logs, msg) +} + +// fail marks the test as failed with an error message +func (t *TestingT) fail(msg string, info *ScriptAssertionInfo) { + t.mu.Lock() + defer t.mu.Unlock() + t.failed = true + t.errors = append(t.errors, msg) + if t.assertionInfo == nil && info != nil { + t.assertionInfo = info + } +} + +// skip marks the test as skipped +func (t *TestingT) skip(reason string) { + t.mu.Lock() + defer t.mu.Unlock() + t.skipped = true + if reason != "" { + t.logs = append(t.logs, "SKIP: "+reason) + } +} + +// NewTestingTObject creates a JavaScript testing.T object for V8 +func NewTestingTObject(v8ctx *v8go.Context, t *TestingT) (*v8go.Value, error) { + iso := v8ctx.Isolate() + + // Create the main testing object + testObj := v8go.NewObjectTemplate(iso) + + // Set name property + testObj.Set("name", t.name) + + // Create assert object + assertObj, err := newAssertObject(v8ctx, t) + if err != nil { + return nil, err + } + + // Set methods + testObj.Set("log", t.logMethod(iso)) + testObj.Set("error", t.errorMethod(iso)) + testObj.Set("skip", t.skipMethod(iso)) + testObj.Set("fail", t.failMethod(iso)) + testObj.Set("fatal", t.fatalMethod(iso)) + + // Create instance + instance, err := testObj.NewInstance(v8ctx) + if err != nil { + return nil, err + } + + obj, err := instance.Value.AsObject() + if err != nil { + return nil, err + } + + // Set assert object + obj.Set("assert", assertObj) + + // Set failed getter (dynamic) + obj.Set("failed", t.failed) + + return instance.Value, nil +} + +// logMethod implements t.log(...args) +func (t *TestingT) logMethod(iso *v8go.Isolate) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + parts := make([]string, len(args)) + for i, arg := range args { + parts[i] = arg.String() + } + t.log(strings.Join(parts, " ")) + return v8go.Undefined(iso) + }) +} + +// errorMethod implements t.error(...args) +func (t *TestingT) errorMethod(iso *v8go.Isolate) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + parts := make([]string, len(args)) + for i, arg := range args { + parts[i] = arg.String() + } + msg := strings.Join(parts, " ") + t.fail(msg, nil) + return v8go.Undefined(iso) + }) +} + +// skipMethod implements t.skip(reason?) +func (t *TestingT) skipMethod(iso *v8go.Isolate) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + reason := "" + if len(info.Args()) > 0 { + reason = info.Args()[0].String() + } + t.skip(reason) + return v8go.Undefined(iso) + }) +} + +// failMethod implements t.fail(reason?) +func (t *TestingT) failMethod(iso *v8go.Isolate) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + reason := "test failed" + if len(info.Args()) > 0 { + reason = info.Args()[0].String() + } + t.fail(reason, nil) + return v8go.Undefined(iso) + }) +} + +// fatalMethod implements t.fatal(reason?) +// Same as fail but intended to stop execution (in JS, this is handled by throwing) +func (t *TestingT) fatalMethod(iso *v8go.Isolate) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + v8ctx := info.Context() + reason := "fatal error" + if len(info.Args()) > 0 { + reason = info.Args()[0].String() + } + t.fail(reason, nil) + // Return exception to stop execution + return bridge.JsException(v8ctx, reason) + }) +} + +// newAssertObject creates the assert object with all assertion methods +func newAssertObject(v8ctx *v8go.Context, t *TestingT) (*v8go.Value, error) { + iso := v8ctx.Isolate() + + assertObj := v8go.NewObjectTemplate(iso) + + // Boolean assertions + assertObj.Set("True", assertTrueMethod(iso, t)) + assertObj.Set("False", assertFalseMethod(iso, t)) + + // Equality assertions + assertObj.Set("Equal", assertEqualMethod(iso, t)) + assertObj.Set("NotEqual", assertNotEqualMethod(iso, t)) + + // Nil assertions + assertObj.Set("Nil", assertNilMethod(iso, t)) + assertObj.Set("NotNil", assertNotNilMethod(iso, t)) + + // String assertions + assertObj.Set("Contains", assertContainsMethod(iso, t)) + assertObj.Set("NotContains", assertNotContainsMethod(iso, t)) + + // Length assertion + assertObj.Set("Len", assertLenMethod(iso, t)) + + // Comparison assertions + assertObj.Set("Greater", assertGreaterMethod(iso, t)) + assertObj.Set("GreaterOrEqual", assertGreaterOrEqualMethod(iso, t)) + assertObj.Set("Less", assertLessMethod(iso, t)) + assertObj.Set("LessOrEqual", assertLessOrEqualMethod(iso, t)) + + // Error assertions + assertObj.Set("Error", assertErrorMethod(iso, t)) + assertObj.Set("NoError", assertNoErrorMethod(iso, t)) + + // Panic assertions + assertObj.Set("Panic", assertPanicMethod(iso, t)) + assertObj.Set("NoPanic", assertNoPanicMethod(iso, t)) + + // Regex assertions + assertObj.Set("Match", assertMatchMethod(iso, t)) + assertObj.Set("NotMatch", assertNotMatchMethod(iso, t)) + + // Type assertion + assertObj.Set("Type", assertTypeMethod(iso, t)) + + // JSON path assertion + assertObj.Set("JSONPath", assertJSONPathMethod(iso, t)) + + // Create instance + instance, err := assertObj.NewInstance(v8ctx) + if err != nil { + return nil, err + } + + return instance.Value, nil +} + +// Helper function to get optional message argument +func getMessage(args []*v8go.Value, startIdx int) string { + if len(args) > startIdx && args[startIdx].IsString() { + return args[startIdx].String() + } + return "" +} + +// assertTrueMethod implements assert.True(value, message?) +func assertTrueMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 1 { + t.fail("True requires a value argument", &ScriptAssertionInfo{Type: "True"}) + return v8go.Undefined(iso) + } + + value := args[0].Boolean() + message := getMessage(args, 1) + + if !value { + msg := "expected true, got false" + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "True", + Expected: true, + Actual: false, + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertFalseMethod implements assert.False(value, message?) +func assertFalseMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 1 { + t.fail("False requires a value argument", &ScriptAssertionInfo{Type: "False"}) + return v8go.Undefined(iso) + } + + value := args[0].Boolean() + message := getMessage(args, 1) + + if value { + msg := "expected false, got true" + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "False", + Expected: false, + Actual: true, + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertEqualMethod implements assert.Equal(actual, expected, message?) +func assertEqualMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + v8ctx := info.Context() + args := info.Args() + if len(args) < 2 { + t.fail("Equal requires actual and expected arguments", &ScriptAssertionInfo{Type: "Equal"}) + return v8go.Undefined(iso) + } + + actual, _ := bridge.GoValue(args[0], v8ctx) + expected, _ := bridge.GoValue(args[1], v8ctx) + message := getMessage(args, 2) + + if !deepEqual(actual, expected) { + msg := fmt.Sprintf("expected %v, got %v", expected, actual) + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "Equal", + Expected: expected, + Actual: actual, + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertNotEqualMethod implements assert.NotEqual(actual, expected, message?) +func assertNotEqualMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + v8ctx := info.Context() + args := info.Args() + if len(args) < 2 { + t.fail("NotEqual requires actual and expected arguments", &ScriptAssertionInfo{Type: "NotEqual"}) + return v8go.Undefined(iso) + } + + actual, _ := bridge.GoValue(args[0], v8ctx) + expected, _ := bridge.GoValue(args[1], v8ctx) + message := getMessage(args, 2) + + if deepEqual(actual, expected) { + msg := fmt.Sprintf("expected values to be different, both are %v", actual) + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "NotEqual", + Expected: expected, + Actual: actual, + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertNilMethod implements assert.Nil(value, message?) +func assertNilMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 1 { + t.fail("Nil requires a value argument", &ScriptAssertionInfo{Type: "Nil"}) + return v8go.Undefined(iso) + } + + isNil := args[0].IsNull() || args[0].IsUndefined() + message := getMessage(args, 1) + + if !isNil { + msg := fmt.Sprintf("expected nil, got %v", args[0].String()) + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "Nil", + Expected: nil, + Actual: args[0].String(), + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertNotNilMethod implements assert.NotNil(value, message?) +func assertNotNilMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 1 { + t.fail("NotNil requires a value argument", &ScriptAssertionInfo{Type: "NotNil"}) + return v8go.Undefined(iso) + } + + isNil := args[0].IsNull() || args[0].IsUndefined() + message := getMessage(args, 1) + + if isNil { + msg := "expected non-nil value, got nil" + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "NotNil", + Actual: nil, + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertContainsMethod implements assert.Contains(str, substr, message?) +func assertContainsMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 2 { + t.fail("Contains requires str and substr arguments", &ScriptAssertionInfo{Type: "Contains"}) + return v8go.Undefined(iso) + } + + str := args[0].String() + substr := args[1].String() + message := getMessage(args, 2) + + if !strings.Contains(str, substr) { + msg := fmt.Sprintf("expected '%s' to contain '%s'", str, substr) + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "Contains", + Expected: substr, + Actual: str, + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertNotContainsMethod implements assert.NotContains(str, substr, message?) +func assertNotContainsMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 2 { + t.fail("NotContains requires str and substr arguments", &ScriptAssertionInfo{Type: "NotContains"}) + return v8go.Undefined(iso) + } + + str := args[0].String() + substr := args[1].String() + message := getMessage(args, 2) + + if strings.Contains(str, substr) { + msg := fmt.Sprintf("expected '%s' to not contain '%s'", str, substr) + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "NotContains", + Expected: substr, + Actual: str, + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertLenMethod implements assert.Len(value, length, message?) +func assertLenMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + v8ctx := info.Context() + args := info.Args() + if len(args) < 2 { + t.fail("Len requires value and length arguments", &ScriptAssertionInfo{Type: "Len"}) + return v8go.Undefined(iso) + } + + value, _ := bridge.GoValue(args[0], v8ctx) + expectedLen := int(args[1].Integer()) + message := getMessage(args, 2) + + actualLen := getLength(value) + + if actualLen != expectedLen { + msg := fmt.Sprintf("expected length %d, got %d", expectedLen, actualLen) + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "Len", + Expected: expectedLen, + Actual: actualLen, + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertGreaterMethod implements assert.Greater(a, b, message?) +func assertGreaterMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 2 { + t.fail("Greater requires two arguments", &ScriptAssertionInfo{Type: "Greater"}) + return v8go.Undefined(iso) + } + + a := args[0].Number() + b := args[1].Number() + message := getMessage(args, 2) + + if !(a > b) { + msg := fmt.Sprintf("expected %v > %v", a, b) + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "Greater", + Expected: fmt.Sprintf("> %v", b), + Actual: a, + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertGreaterOrEqualMethod implements assert.GreaterOrEqual(a, b, message?) +func assertGreaterOrEqualMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 2 { + t.fail("GreaterOrEqual requires two arguments", &ScriptAssertionInfo{Type: "GreaterOrEqual"}) + return v8go.Undefined(iso) + } + + a := args[0].Number() + b := args[1].Number() + message := getMessage(args, 2) + + if !(a >= b) { + msg := fmt.Sprintf("expected %v >= %v", a, b) + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "GreaterOrEqual", + Expected: fmt.Sprintf(">= %v", b), + Actual: a, + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertLessMethod implements assert.Less(a, b, message?) +func assertLessMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 2 { + t.fail("Less requires two arguments", &ScriptAssertionInfo{Type: "Less"}) + return v8go.Undefined(iso) + } + + a := args[0].Number() + b := args[1].Number() + message := getMessage(args, 2) + + if !(a < b) { + msg := fmt.Sprintf("expected %v < %v", a, b) + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "Less", + Expected: fmt.Sprintf("< %v", b), + Actual: a, + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertLessOrEqualMethod implements assert.LessOrEqual(a, b, message?) +func assertLessOrEqualMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 2 { + t.fail("LessOrEqual requires two arguments", &ScriptAssertionInfo{Type: "LessOrEqual"}) + return v8go.Undefined(iso) + } + + a := args[0].Number() + b := args[1].Number() + message := getMessage(args, 2) + + if !(a <= b) { + msg := fmt.Sprintf("expected %v <= %v", a, b) + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "LessOrEqual", + Expected: fmt.Sprintf("<= %v", b), + Actual: a, + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertErrorMethod implements assert.Error(err, message?) +func assertErrorMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 1 { + t.fail("Error requires an argument", &ScriptAssertionInfo{Type: "Error"}) + return v8go.Undefined(iso) + } + + // Check if it's null/undefined (no error) + isError := !args[0].IsNull() && !args[0].IsUndefined() + message := getMessage(args, 1) + + if !isError { + msg := "expected an error, got nil" + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "Error", + Actual: nil, + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertNoErrorMethod implements assert.NoError(err, message?) +func assertNoErrorMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 1 { + t.fail("NoError requires an argument", &ScriptAssertionInfo{Type: "NoError"}) + return v8go.Undefined(iso) + } + + // Check if it's null/undefined (no error) + isError := !args[0].IsNull() && !args[0].IsUndefined() + message := getMessage(args, 1) + + if isError { + msg := fmt.Sprintf("expected no error, got %v", args[0].String()) + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "NoError", + Actual: args[0].String(), + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertPanicMethod implements assert.Panic(fn, message?) +func assertPanicMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + v8ctx := info.Context() + args := info.Args() + if len(args) < 1 { + t.fail("Panic requires a function argument", &ScriptAssertionInfo{Type: "Panic"}) + return v8go.Undefined(iso) + } + + if !args[0].IsFunction() { + t.fail("Panic requires a function argument", &ScriptAssertionInfo{Type: "Panic"}) + return v8go.Undefined(iso) + } + + message := getMessage(args, 1) + + // Try to call the function and check if it throws + fn, _ := args[0].AsFunction() + _, err := fn.Call(v8ctx.Global()) + + if err == nil { + msg := "expected function to panic, but it didn't" + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "Panic", + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertNoPanicMethod implements assert.NoPanic(fn, message?) +func assertNoPanicMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + v8ctx := info.Context() + args := info.Args() + if len(args) < 1 { + t.fail("NoPanic requires a function argument", &ScriptAssertionInfo{Type: "NoPanic"}) + return v8go.Undefined(iso) + } + + if !args[0].IsFunction() { + t.fail("NoPanic requires a function argument", &ScriptAssertionInfo{Type: "NoPanic"}) + return v8go.Undefined(iso) + } + + message := getMessage(args, 1) + + // Try to call the function and check if it throws + fn, _ := args[0].AsFunction() + _, err := fn.Call(v8ctx.Global()) + + if err != nil { + msg := fmt.Sprintf("expected function not to panic, but got: %v", err) + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "NoPanic", + Actual: err.Error(), + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertMatchMethod implements assert.Match(value, pattern, message?) +func assertMatchMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 2 { + t.fail("Match requires value and pattern arguments", &ScriptAssertionInfo{Type: "Match"}) + return v8go.Undefined(iso) + } + + value := args[0].String() + pattern := args[1].String() + message := getMessage(args, 2) + + re, err := regexp.Compile(pattern) + if err != nil { + t.fail(fmt.Sprintf("invalid regex pattern: %v", err), &ScriptAssertionInfo{ + Type: "Match", + Message: message, + }) + return v8go.Undefined(iso) + } + + if !re.MatchString(value) { + msg := fmt.Sprintf("expected '%s' to match pattern '%s'", value, pattern) + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "Match", + Expected: pattern, + Actual: value, + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertNotMatchMethod implements assert.NotMatch(value, pattern, message?) +func assertNotMatchMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 2 { + t.fail("NotMatch requires value and pattern arguments", &ScriptAssertionInfo{Type: "NotMatch"}) + return v8go.Undefined(iso) + } + + value := args[0].String() + pattern := args[1].String() + message := getMessage(args, 2) + + re, err := regexp.Compile(pattern) + if err != nil { + t.fail(fmt.Sprintf("invalid regex pattern: %v", err), &ScriptAssertionInfo{ + Type: "NotMatch", + Message: message, + }) + return v8go.Undefined(iso) + } + + if re.MatchString(value) { + msg := fmt.Sprintf("expected '%s' to not match pattern '%s'", value, pattern) + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "NotMatch", + Expected: pattern, + Actual: value, + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertTypeMethod implements assert.Type(value, typeName, message?) +func assertTypeMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 2 { + t.fail("Type requires value and typeName arguments", &ScriptAssertionInfo{Type: "Type"}) + return v8go.Undefined(iso) + } + + value := args[0] + expectedType := args[1].String() + message := getMessage(args, 2) + + actualType := getJsType(value) + + if actualType != expectedType { + msg := fmt.Sprintf("expected type '%s', got '%s'", expectedType, actualType) + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "Type", + Expected: expectedType, + Actual: actualType, + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// assertJSONPathMethod implements assert.JSONPath(obj, path, expected, message?) +func assertJSONPathMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + v8ctx := info.Context() + args := info.Args() + if len(args) < 3 { + t.fail("JSONPath requires obj, path, and expected arguments", &ScriptAssertionInfo{Type: "JSONPath"}) + return v8go.Undefined(iso) + } + + obj, _ := bridge.GoValue(args[0], v8ctx) + path := args[1].String() + expected, _ := bridge.GoValue(args[2], v8ctx) + message := getMessage(args, 3) + + // Use the existing extractPath function from assert.go + asserter := &Asserter{} + actual := asserter.extractPath(obj, strings.TrimPrefix(path, "$.")) + + if !deepEqual(actual, expected) { + msg := fmt.Sprintf("path '%s': expected %v, got %v", path, expected, actual) + if message != "" { + msg = message + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "JSONPath", + Expected: expected, + Actual: actual, + Message: message, + }) + } + return v8go.Undefined(iso) + }) +} + +// Helper functions + +// deepEqual performs deep equality comparison +func deepEqual(a, b interface{}) bool { + // Handle nil cases + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + + // Try JSON comparison for complex types + aJSON, errA := json.Marshal(a) + bJSON, errB := json.Marshal(b) + if errA == nil && errB == nil { + return string(aJSON) == string(bJSON) + } + + // Fall back to reflect.DeepEqual + return reflect.DeepEqual(a, b) +} + +// getLength returns the length of a value (array, string, map) +func getLength(v interface{}) int { + if v == nil { + return 0 + } + + switch val := v.(type) { + case string: + return len(val) + case []interface{}: + return len(val) + case map[string]interface{}: + return len(val) + default: + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.Slice, reflect.Array, reflect.Map, reflect.String: + return rv.Len() + } + } + return 0 +} + +// getJsType returns the JavaScript type name of a value +func getJsType(v *v8go.Value) string { + if v.IsNull() { + return "null" + } + if v.IsUndefined() { + return "undefined" + } + if v.IsString() { + return "string" + } + if v.IsNumber() { + return "number" + } + if v.IsBoolean() { + return "boolean" + } + if v.IsArray() { + return "array" + } + if v.IsFunction() { + return "function" + } + if v.IsObject() { + return "object" + } + return "unknown" +} diff --git a/agent/test/script_types.go b/agent/test/script_types.go new file mode 100644 index 00000000..27531ebc --- /dev/null +++ b/agent/test/script_types.go @@ -0,0 +1,170 @@ +package test + +import "time" + +// ScriptInfo contains information about the script being tested +type ScriptInfo struct { + // ID is the script identifier (e.g., "scripts.expense.setup") + ID string `json:"id"` + + // Assistant is the assistant directory name (e.g., "expense") + Assistant string `json:"assistant"` + + // Module is the module name (e.g., "setup") + Module string `json:"module"` + + // ScriptPath is the path to the main script file (e.g., "expense/src/setup.ts") + ScriptPath string `json:"script_path"` + + // TestPath is the path to the test script file (e.g., "expense/src/setup_test.ts") + TestPath string `json:"test_path"` +} + +// ScriptTestCase represents a single script test function +type ScriptTestCase struct { + // Name is the test function name (e.g., "TestSystemReady") + Name string `json:"name"` + + // Function is the full function reference + Function string `json:"function"` +} + +// ScriptTestResult represents the result of running a script test function +type ScriptTestResult struct { + // Name is the test function name + Name string `json:"name"` + + // Status is the test execution status + Status Status `json:"status"` + + // DurationMs is the execution duration in milliseconds + DurationMs int64 `json:"duration_ms"` + + // Error contains the error message if the test failed + Error string `json:"error,omitempty"` + + // Assertion contains assertion failure details + Assertion *ScriptAssertionInfo `json:"assertion,omitempty"` + + // Logs contains log messages from the test + Logs []string `json:"logs,omitempty"` +} + +// ScriptAssertionInfo contains details about an assertion failure +type ScriptAssertionInfo struct { + // Type is the assertion type (e.g., "Equal", "True") + Type string `json:"type"` + + // Expected is the expected value + Expected interface{} `json:"expected,omitempty"` + + // Actual is the actual value + Actual interface{} `json:"actual,omitempty"` + + // Message is the custom failure message + Message string `json:"message,omitempty"` +} + +// ScriptTestSummary contains aggregated statistics for script tests +type ScriptTestSummary struct { + // Total number of test functions + Total int `json:"total"` + + // Passed number of test functions that passed + Passed int `json:"passed"` + + // Failed number of test functions that failed + Failed int `json:"failed"` + + // Skipped number of test functions that were skipped + Skipped int `json:"skipped"` + + // DurationMs is the total execution duration in milliseconds + DurationMs int64 `json:"duration_ms"` +} + +// ScriptTestReport represents the complete script test report +type ScriptTestReport struct { + // Type indicates this is a script test report + Type string `json:"type"` // "script_test" + + // Script is the script identifier (e.g., "scripts.expense.setup") + Script string `json:"script"` + + // ScriptPath is the path to the test script file + ScriptPath string `json:"script_path"` + + // Summary contains aggregated statistics + Summary *ScriptTestSummary `json:"summary"` + + // Environment contains the test environment configuration + Environment *Environment `json:"environment"` + + // Results contains individual test results + Results []*ScriptTestResult `json:"results"` + + // Metadata contains additional report metadata + Metadata *ScriptTestMetadata `json:"metadata"` +} + +// ScriptTestMetadata contains metadata about the script test report +type ScriptTestMetadata struct { + // StartedAt is when the test run started + StartedAt time.Time `json:"started_at"` + + // CompletedAt is when the test run completed + CompletedAt time.Time `json:"completed_at"` + + // Version is the Yao version + Version string `json:"version"` +} + +// HasFailures returns true if there are any failed tests +func (r *ScriptTestReport) HasFailures() bool { + return r.Summary.Failed > 0 +} + +// PassRate returns the pass rate as a percentage (0-100) +func (r *ScriptTestReport) PassRate() float64 { + if r.Summary.Total == 0 { + return 0 + } + return float64(r.Summary.Passed) / float64(r.Summary.Total) * 100 +} + +// ToReport converts ScriptTestReport to a standard Report for unified reporting +func (r *ScriptTestReport) ToReport() *Report { + return &Report{ + Summary: &Summary{ + Total: r.Summary.Total, + Passed: r.Summary.Passed, + Failed: r.Summary.Failed, + Skipped: r.Summary.Skipped, + DurationMs: r.Summary.DurationMs, + AgentID: r.Script, + AgentPath: r.ScriptPath, + }, + Environment: r.Environment, + Results: r.toResults(), + Metadata: &ReportMetadata{ + StartedAt: r.Metadata.StartedAt, + CompletedAt: r.Metadata.CompletedAt, + Version: r.Metadata.Version, + }, + } +} + +// toResults converts script test results to standard results +func (r *ScriptTestReport) toResults() []*Result { + results := make([]*Result, len(r.Results)) + for i, sr := range r.Results { + results[i] = &Result{ + ID: sr.Name, + Status: sr.Status, + Input: sr.Name, + DurationMs: sr.DurationMs, + Error: sr.Error, + } + } + return results +} diff --git a/agent/test/types.go b/agent/test/types.go index b2ac9698..5b29bc53 100644 --- a/agent/test/types.go +++ b/agent/test/types.go @@ -1,7 +1,10 @@ package test import ( + "encoding/json" + "fmt" "math" + "os" "time" "github.com/yaoapp/yao/agent/context" @@ -57,6 +60,8 @@ const ( InputModeFile InputMode = "file" // InputModeMessage indicates input from a direct message string InputModeMessage InputMode = "message" + // InputModeScript indicates script test mode (testing agent handler scripts) + InputModeScript InputMode = "script" ) // Options represents the configuration options for running tests @@ -96,6 +101,14 @@ type Options struct { // Locale is the locale for the test context (default: "en-us") Locale string `json:"locale,omitempty"` + // ContextFile is the path to a JSON file containing custom context data (-ctx flag) + // This allows full customization of authorized info, metadata, etc. + ContextFile string `json:"context_file,omitempty"` + + // ContextData is the parsed context data from ContextFile + // This is populated internally after loading the file + ContextData *ContextConfig `json:"-"` + // Execution // =============================== @@ -126,6 +139,92 @@ type Options struct { // FailFast stops execution on first failure FailFast bool `json:"fail_fast,omitempty"` + + // Run is a regex pattern to filter which tests to run (similar to go test -run) + // Only tests matching the pattern will be executed + // Example: "TestSystem" matches TestSystemReady, TestSystemError, etc. + Run string `json:"run,omitempty"` +} + +// ContextConfig represents custom context configuration from JSON file +// This allows full customization of the test context including authorized info +type ContextConfig struct { + // Authorized contains custom authorization data + Authorized *AuthorizedConfig `json:"authorized,omitempty"` + + // Metadata contains custom metadata to pass to the context + Metadata map[string]interface{} `json:"metadata,omitempty"` + + // Client contains custom client information + Client *ClientConfig `json:"client,omitempty"` + + // Locale overrides the locale setting + Locale string `json:"locale,omitempty"` + + // Referer overrides the referer setting + Referer string `json:"referer,omitempty"` +} + +// AuthorizedConfig represents custom authorization configuration +// Matches the structure of types.AuthorizedInfo from openapi/oauth/types +type AuthorizedConfig struct { + // Sub is the subject identifier (JWT sub claim) + Sub string `json:"sub,omitempty"` + + // ClientID is the OAuth client ID + ClientID string `json:"client_id,omitempty"` + + // Scope is the access scope + Scope string `json:"scope,omitempty"` + + // SessionID is the session identifier + SessionID string `json:"session_id,omitempty"` + + // UserID is the user identifier + UserID string `json:"user_id,omitempty"` + + // TeamID is the team identifier + TeamID string `json:"team_id,omitempty"` + + // TenantID is the tenant identifier + TenantID string `json:"tenant_id,omitempty"` + + // RememberMe is the remember me flag + RememberMe bool `json:"remember_me,omitempty"` + + // Constraints contains data access constraints (set by ACL enforcement) + Constraints *DataConstraintsConfig `json:"constraints,omitempty"` +} + +// DataConstraintsConfig represents data access constraints +// Matches the structure of types.DataConstraints from openapi/oauth/types +type DataConstraintsConfig struct { + // OwnerOnly - only access owner's data + OwnerOnly bool `json:"owner_only,omitempty"` + + // CreatorOnly - only access creator's data + CreatorOnly bool `json:"creator_only,omitempty"` + + // EditorOnly - only access editor's data + EditorOnly bool `json:"editor_only,omitempty"` + + // TeamOnly - only access team's data (filter by team_id) + TeamOnly bool `json:"team_only,omitempty"` + + // Extra contains user-defined constraints (department, region, etc.) + Extra map[string]interface{} `json:"extra,omitempty"` +} + +// ClientConfig represents custom client configuration +type ClientConfig struct { + // Type is the client type (e.g., "web", "mobile", "test") + Type string `json:"type,omitempty"` + + // UserAgent is the client user agent string + UserAgent string `json:"user_agent,omitempty"` + + // IP is the client IP address + IP string `json:"ip,omitempty"` } // Environment configures the test execution context @@ -150,6 +249,9 @@ type Environment struct { // Accept is the accept format (default: "standard") Accept string `json:"accept"` + + // ContextConfig contains custom context configuration (from -ctx flag) + ContextConfig *ContextConfig `json:"-"` } // NewEnvironment creates a new test environment with defaults @@ -175,6 +277,61 @@ func NewEnvironment(userID, teamID string) *Environment { return env } +// NewEnvironmentWithContext creates a new test environment with custom context config +func NewEnvironmentWithContext(userID, teamID string, ctxConfig *ContextConfig) *Environment { + env := NewEnvironment(userID, teamID) + + if ctxConfig == nil { + return env + } + + env.ContextConfig = ctxConfig + + // Override with context config values + if ctxConfig.Locale != "" { + env.Locale = ctxConfig.Locale + } + if ctxConfig.Referer != "" { + env.Referer = ctxConfig.Referer + } + if ctxConfig.Client != nil { + if ctxConfig.Client.Type != "" { + env.ClientType = ctxConfig.Client.Type + } + if ctxConfig.Client.IP != "" { + env.ClientIP = ctxConfig.Client.IP + } + } + if ctxConfig.Authorized != nil { + if ctxConfig.Authorized.UserID != "" { + env.UserID = ctxConfig.Authorized.UserID + } + // TeamID takes precedence over TenantID for team override + if ctxConfig.Authorized.TeamID != "" { + env.TeamID = ctxConfig.Authorized.TeamID + } else if ctxConfig.Authorized.TenantID != "" { + env.TeamID = ctxConfig.Authorized.TenantID + } + } + + return env +} + +// LoadContextConfig loads context configuration from a JSON file +func LoadContextConfig(filePath string) (*ContextConfig, error) { + data, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read context file: %w", err) + } + + var config ContextConfig + if err := json.Unmarshal(data, &config); err != nil { + return nil, fmt.Errorf("failed to parse context file: %w", err) + } + + return &config, nil +} + // Case represents a single test case loaded from JSONL type Case struct { // ID is the unique identifier for this test case (e.g., "T001") diff --git a/cmd/README.md b/cmd/README.md index 544a96c5..13a718c3 100644 --- a/cmd/README.md +++ b/cmd/README.md @@ -14,17 +14,17 @@ go install github.com/yaoapp/yao@latest ## Global Flags -| Flag | Short | Description | -|------|-------|-------------| -| `--app` | `-a` | Application directory path | -| `--file` | `-f` | Application package file (.yaz) | -| `--key` | `-k` | Application license key | +| Flag | Short | Description | +| -------- | ----- | ------------------------------- | +| `--app` | `-a` | Application directory path | +| `--file` | `-f` | Application package file (.yaz) | +| `--key` | `-k` | Application license key | ## Environment Variables -| Variable | Description | -|----------|-------------| -| `YAO_ROOT` | Application root directory | +| Variable | Description | +| ---------- | -------------------------------------------- | +| `YAO_ROOT` | Application root directory | | `YAO_LANG` | Language setting (e.g., `zh-CN` for Chinese) | ## Commands @@ -46,10 +46,10 @@ yao start --debug **Flags:** -| Flag | Description | -|------|-------------| -| `--debug` | Enable development/debug mode | -| `--disable-watching` | Disable file watching | +| Flag | Description | +| -------------------- | ----------------------------- | +| `--debug` | Enable development/debug mode | +| `--disable-watching` | Disable file watching | --- @@ -70,9 +70,9 @@ yao run -s models.user.Find 1 **Flags:** -| Flag | Short | Description | -|------|-------|-------------| -| `--silent` | `-s` | Silent mode - output result as JSON only | +| Flag | Short | Description | +| ---------- | ----- | ---------------------------------------- | +| `--silent` | `-s` | Silent mode - output result as JSON only | **Argument Syntax:** @@ -102,11 +102,11 @@ yao migrate --reset **Flags:** -| Flag | Short | Description | -|------|-------|-------------| -| `--name` | `-n` | Specific model name to migrate | -| `--force` | | Force migrate in production mode | -| `--reset` | | Drop tables before migration | +| Flag | Short | Description | +| --------- | ----- | -------------------------------- | +| `--name` | `-n` | Specific model name to migrate | +| `--force` | | Force migrate in production mode | +| `--reset` | | Drop tables before migration | --- @@ -134,8 +134,8 @@ yao version --all **Flags:** -| Flag | Description | -|------|-------------| +| Flag | Description | +| ------- | -------------------------------------------------------------------- | | `--all` | Print all version information (Go version, commit, build time, etc.) | --- @@ -146,7 +146,7 @@ Commands for testing and managing AI agents. ### `yao agent test` -Test an agent with input cases from a JSONL file or direct message. +Test an agent with input cases from a JSONL file, direct message, or script tests. ```bash # Test with direct message (development mode) @@ -169,45 +169,104 @@ yao agent test -i tests/inputs.jsonl --parallel 4 # Verbose output yao agent test -i tests/inputs.jsonl -v + +# Script tests (test agent handler scripts) +yao agent test -i scripts.expense.setup -v + +# Script tests with test filtering +yao agent test -i scripts.expense.setup --run "TestSystemReady" + +# Script tests with custom context +yao agent test -i scripts.expense.setup --ctx tests/context.json -v ``` **Flags:** -| Flag | Short | Description | -|------|-------|-------------| -| `--input` | `-i` | Input: JSONL file path or direct message (required) | -| `--output` | `-o` | Output file path (default: `output-{timestamp}.jsonl`) | -| `--name` | `-n` | Agent ID (default: auto-detect from path) | -| `--connector` | `-c` | Override default connector | -| `--user` | `-u` | Test user ID (default: `test-user`) | -| `--team` | `-t` | Test team ID (default: `test-team`) | -| `--reporter` | `-r` | Reporter agent ID for custom report generation | -| `--runs` | | Number of runs per test case for stability analysis (default: 1) | -| `--timeout` | | Timeout per test case (default: `5m`) | -| `--parallel` | | Number of parallel test cases (default: 1) | -| `--verbose` | `-v` | Enable verbose output | -| `--fail-fast` | | Stop on first failure | -| `--app` | `-a` | Application directory | -| `--env` | `-e` | Environment file | +| Flag | Short | Description | +| ------------- | ----- | ---------------------------------------------------------------- | +| `--input` | `-i` | Input: JSONL file path, message, or script ID (required) | +| `--output` | `-o` | Output file path (default: `output-{timestamp}.jsonl`) | +| `--name` | `-n` | Agent ID (default: auto-detect from path) | +| `--connector` | `-c` | Override default connector | +| `--user` | `-u` | Test user ID (default: `test-user`) | +| `--team` | `-t` | Test team ID (default: `test-team`) | +| `--ctx` | | Path to context JSON file for custom authorization | +| `--reporter` | `-r` | Reporter agent ID for custom report generation | +| `--runs` | | Number of runs per test case for stability analysis (default: 1) | +| `--run` | | Regex pattern to filter which tests to run | +| `--timeout` | | Timeout per test case (default: `5m`) | +| `--parallel` | | Number of parallel test cases (default: 1) | +| `--verbose` | `-v` | Enable verbose output | +| `--fail-fast` | | Stop on first failure | +| `--app` | `-a` | Application directory | +| `--env` | `-e` | Environment file | **Input Modes:** 1. **Direct Message Mode**: For quick development/debugging + ```bash yao agent test -i "Hello world" -n my.agent ``` + - Outputs result directly to stdout - No report file generated - Ideal for iterative development 2. **File Mode**: For comprehensive testing + ```bash yao agent test -i tests/inputs.jsonl ``` + - Reads test cases from JSONL file - Generates detailed report - Supports stability analysis +3. **Script Test Mode**: For testing agent handler scripts + ```bash + yao agent test -i scripts.expense.setup -v + ``` + - Tests TypeScript/JavaScript handler scripts (hooks, tools, setup functions) + - Input format: `scripts..` (e.g., `scripts.expense.setup`) + - Automatically discovers and runs all `Test*` functions + - Uses Go-like testing interface with assertions + +**Script Test Function Signature:** + +```typescript +// assistants/expense/src/setup_test.ts +import { SystemReady } from "./setup"; + +export function TestSystemReady(t: testing.T, ctx: agent.Context) { + const result = SystemReady(ctx); + t.assert.True(result.success, "SystemReady should succeed"); + t.assert.Equal(result.status, "ready", "Status should be ready"); +} +``` + +**Context JSON Format (for `--ctx` flag):** + +```json +{ + "authorized": { + "sub": "user-12345", + "client_id": "my-app", + "user_id": "admin", + "team_id": "team-001", + "tenant_id": "acme-corp", + "constraints": { + "owner_only": true, + "team_only": false, + "extra": { "department": "engineering" } + } + }, + "metadata": { "request_id": "req-123" }, + "client": { "type": "web", "ip": "192.168.1.100" }, + "locale": "zh-cn" +} +``` + **JSONL Input Format:** ```jsonl @@ -221,12 +280,12 @@ yao agent test -i tests/inputs.jsonl -v **Output Formats:** -| Extension | Format | Description | -|-----------|--------|-------------| -| `.jsonl` | JSONL | Streaming format (default) | -| `.json` | JSON | Complete structured report | -| `.md` | Markdown | Human-readable with tables | -| `.html` | HTML | Interactive web report | +| Extension | Format | Description | +| --------- | -------- | -------------------------- | +| `.jsonl` | JSONL | Streaming format (default) | +| `.json` | JSON | Complete structured report | +| `.md` | Markdown | Human-readable with tables | +| `.html` | HTML | Interactive web report | **Agent Resolution:** @@ -281,11 +340,11 @@ yao sui trans default index -l "en-US,zh-CN,ja-JP" **SUI Flags:** -| Flag | Short | Description | -|------|-------|-------------| -| `--data` | `-d` | Session data as JSON (prefix with `::`) | -| `--debug` | `-D` | Enable debug mode | -| `--locales` | `-l` | Locales for translation (comma-separated) | +| Flag | Short | Description | +| ----------- | ----- | ----------------------------------------- | +| `--data` | `-d` | Session data as JSON (prefix with `::`) | +| `--debug` | `-D` | Enable debug mode | +| `--locales` | `-l` | Locales for translation (comma-separated) | --- @@ -313,6 +372,15 @@ yao sui watch default home # Run comprehensive agent tests yao agent test -i tests/inputs.jsonl -o report.html -v +# Run script tests for agent handlers +yao agent test -i scripts.expense.setup -v + +# Run specific script tests with filtering +yao agent test -i scripts.expense.setup --run "TestSystem.*" -v + +# Run script tests with custom context +yao agent test -i scripts.expense.setup --ctx tests/context.json -v + # Stability analysis (run each test 10 times) yao agent test -i tests/inputs.jsonl --runs 10 -o stability-report.json @@ -337,10 +405,10 @@ yao migrate -n user --reset --force ## Exit Codes -| Code | Description | -|------|-------------| -| 0 | Success | -| 1 | Error or test failure | +| Code | Description | +| ---- | --------------------- | +| 0 | Success | +| 1 | Error or test failure | --- @@ -370,4 +438,3 @@ myapp/ - [Yao Documentation](https://yaoapps.com/docs) - [Agent Test Design](../agent/test/DESIGN.md) - [SUI Documentation](https://yaoapps.com/docs/sui) - diff --git a/cmd/agent/agent.go b/cmd/agent/agent.go index d5fcffaf..4db2be45 100644 --- a/cmd/agent/agent.go +++ b/cmd/agent/agent.go @@ -22,8 +22,10 @@ var langs = map[string]string{ "Override connector": "覆盖连接器", "Test user ID (default: test-user)": "测试用户 ID (默认: test-user)", "Test team ID (default: test-team)": "测试团队 ID (默认: test-team)", + "Path to context JSON file for custom authorization": "自定义认证信息的 JSON 文件路径", "Reporter agent ID for custom report": "自定义报告生成器智能体 ID", "Number of runs for stability analysis": "稳定性分析的运行次数", + "Regex pattern to filter which tests to run": "用于过滤测试的正则表达式", "Default timeout per test case": "每个测试用例的默认超时时间", "Number of parallel test cases": "并行测试用例数", "Verbose output": "详细输出", diff --git a/cmd/agent/test.go b/cmd/agent/test.go index 3098a5e3..7af0ef03 100644 --- a/cmd/agent/test.go +++ b/cmd/agent/test.go @@ -25,8 +25,10 @@ var ( testConnector string testUser string testTeam string + testContext string // --ctx flag for custom context JSON file testReporter string testRuns int + testRun string // --run flag for test filtering (regex pattern) testTimeout string testParallel int testVerbose bool @@ -140,19 +142,21 @@ var TestCmd = &cobra.Command{ // Build test options opts := &test.Options{ - Input: testInput, - InputMode: inputMode, - OutputFile: testOutput, - AgentID: testAgent, - Connector: testConnector, - UserID: testUser, - TeamID: testTeam, - ReporterID: testReporter, - Runs: testRuns, - Timeout: timeout, - Parallel: testParallel, - Verbose: testVerbose, - FailFast: testFailFast, + Input: testInput, + InputMode: inputMode, + OutputFile: testOutput, + AgentID: testAgent, + Connector: testConnector, + UserID: testUser, + TeamID: testTeam, + ContextFile: testContext, + ReporterID: testReporter, + Runs: testRuns, + Run: testRun, + Timeout: timeout, + Parallel: testParallel, + Verbose: testVerbose, + FailFast: testFailFast, } // Merge with defaults @@ -232,8 +236,10 @@ func init() { TestCmd.Flags().StringVarP(&testConnector, "connector", "c", "", L("Override connector")) TestCmd.Flags().StringVarP(&testUser, "user", "u", "", L("Test user ID (default: test-user)")) TestCmd.Flags().StringVarP(&testTeam, "team", "t", "", L("Test team ID (default: test-team)")) + TestCmd.Flags().StringVar(&testContext, "ctx", "", L("Path to context JSON file for custom authorization")) TestCmd.Flags().StringVarP(&testReporter, "reporter", "r", "", L("Reporter agent ID for custom report")) TestCmd.Flags().IntVar(&testRuns, "runs", 1, L("Number of runs for stability analysis")) + TestCmd.Flags().StringVar(&testRun, "run", "", L("Regex pattern to filter which tests to run")) TestCmd.Flags().StringVar(&testTimeout, "timeout", "5m", L("Default timeout per test case")) TestCmd.Flags().IntVar(&testParallel, "parallel", 1, L("Number of parallel test cases")) TestCmd.Flags().BoolVarP(&testVerbose, "verbose", "v", false, L("Verbose output"))