diff --git a/.gitignore b/.gitignore index d11f893d..d1b23f38 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,4 @@ agent/assistant/hook/*.test.md agent/search/TODO.md agent/search/job-logs.txt agent/test/MULTI_TURN_DESIGN.md +agent/test/UPGRADE_PLAN.md diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md new file mode 100644 index 00000000..98fd99e1 --- /dev/null +++ b/agent/test/DESIGN_V2.md @@ -0,0 +1,1017 @@ +# Agent Test Framework V2 Design + +## Overview + +This document describes the design for Agent Test Framework V2, which extends the existing testing capabilities with: + +- **Message history support** - Test agents with conversation context via `input` array (already implemented) +- **Agent-driven testing** - Use agents to generate test cases and validate responses +- **Dynamic testing** - Simulator-driven testing with checkpoint validation + +## Quick Reference: Format Rules + +| Context | Format | Example | +| --------------------- | ------------------------ | ------------------------------------------------------- | +| `-i` flag (CLI) | Prefix required | `agents:workers.test.gen`, `scripts:tests.gen` | +| JSONL assertion `use` | Prefix required | `"use": "agents:workers.test.validator"` | +| JSONL `simulator.use` | No prefix (agent only) | `"use": "workers.test.user-simulator"` | +| `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-simulator` | +| `t.assert.Agent()` | No prefix (method-bound) | `t.assert.Agent(resp, "workers.test.validator", {...})` | +| JSONL `before/after` | No prefix (in src/) | `"before": "env_test.Before"` | +| `--before/--after` | No prefix (in src/) | `--before env_test.BeforeAll` | + +## Design Goals + +1. **Simple** - Single-turn with optional message history, no complex multi-turn state +2. **Stateless** - Each test is independent, no session management needed +3. **Parallel** - Tests can run in parallel since they don't share state +4. **Flexible** - Support both static (messages) and dynamic (simulator) testing +5. **Agent-driven** - Input generation, simulation, and validation can all be agent-powered + +## Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ yao agent test │ +├─────────────────────────────────────────────────────────────────────────┤ +│ │ +│ INPUT SOURCES (-i flag) │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ JSONL File │ │ Message │ │ Generator │ │ +│ │ ./test.jsonl│ │ "Hello..." │ │ agents:xxx │ │ +│ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ │ +│ │ │ │ │ +│ └────────────────┴────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌───────────────────────────────────────────────────────────────────┐ │ +│ │ Test Case Parser │ │ +│ │ │ │ +│ │ Standard Mode: {input: "..." | [...], assertions} │ │ +│ │ Dynamic Mode: {simulator: {...}, checkpoints: [...]} │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ┌───────────────┴───────────────┐ │ +│ ▼ ▼ │ +│ ┌───────────────────┐ ┌───────────────────────┐ │ +│ │ STANDARD MODE │ │ DYNAMIC MODE │ │ +│ │ │ │ │ │ +│ │ 1. Build messages │ │ LOOP: │ │ +│ │ 2. Call Agent │ │ 1. Simulator→input │ │ +│ │ 3. Run assertions │ │ 2. Call Agent │ │ +│ │ │ │ 3. Check checkpoints │ │ +│ │ → PASS/FAIL │ │ 4. Until done │ │ +│ └───────────────────┘ └───────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌───────────────────────────────────────────────────────────────────┐ │ +│ │ Reporter │ │ +│ │ - Console output │ │ +│ │ - JSON file output │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +## Test Modes + +### Standard Mode (Default) + +Single call to agent with optional message history. **No multi-turn state management needed.** + +| Field | Type | Description | +| ------------ | ------------------------------ | --------------------------------------------- | +| `input` | string \| Message \| Message[] | Text, single message, or conversation history | +| `assertions` | array | Assertions to validate response | +| `options` | object | `context.Options` passed to agent | + +### Dynamic Mode + +Simulator-driven testing with checkpoint validation. + +| Field | Type | Description | +| ------------- | ------ | -------------------------------- | +| `simulator` | object | Simulator agent configuration | +| `checkpoints` | array | Functional checkpoints to verify | +| `max_turns` | int | Maximum turns before timeout | +| `timeout` | string | Maximum time (e.g., "5m") | + +## Test Case Format + +### Simple Input (Existing) + +```jsonl +{ + "id": "T001", + "input": "Hello", + "assertions": [ + { + "type": "contains", + "value": "Hi" + } + ] +} +``` + +### With Message History (Existing) + +The `input` field already supports message arrays for conversation context: + +```jsonl +{ + "id": "T002", + "name": "Expense submission - final confirmation", + "input": [ + { + "role": "user", + "content": "I want to submit an expense" + }, + { + "role": "assistant", + "content": "What type of expense would you like to submit?" + }, + { + "role": "user", + "content": "Business travel to Beijing, $3500" + }, + { + "role": "assistant", + "content": "I'll create an expense for business travel, $3500. Please confirm." + }, + { + "role": "user", + "content": "Yes, confirm" + } + ], + "assertions": [ + { + "type": "contains", + "value": "submitted" + }, + { + "type": "tool_called", + "name": "create_expense" + } + ] +} +``` + +**Key insight**: Instead of executing 3 turns sequentially, we pass the full conversation history. The agent sees the context and responds to the last message. This is: + +- **Simpler** - No turn-by-turn execution, no session state +- **Faster** - Single API call instead of multiple +- **Parallelizable** - Each test is independent +- **Debuggable** - Clear input/output for each test + +### Testing Different Points in a Conversation + +To test agent behavior at different conversation stages, create separate test cases: + +```jsonl +// Test 1: First turn - agent should ask for expense type +{ + "id": "expense-turn1", + "input": [{"role": "user", "content": "I want to submit an expense"}], + "assertions": [{"type": "contains", "value": "type"}] +} + +// Test 2: Second turn - agent should create expense +{ + "id": "expense-turn2", + "input": [ + {"role": "user", "content": "I want to submit an expense"}, + {"role": "assistant", "content": "What type of expense would you like to submit?"}, + {"role": "user", "content": "Business travel, $3500"} + ], + "assertions": [{"type": "tool_called", "name": "create_expense"}] +} + +// Test 3: Final turn - agent should confirm submission +{ + "id": "expense-turn3", + "input": [ + {"role": "user", "content": "I want to submit an expense"}, + {"role": "assistant", "content": "What type of expense?"}, + {"role": "user", "content": "Business travel, $3500"}, + {"role": "assistant", "content": "Confirm $3500 expense?"}, + {"role": "user", "content": "Yes"} + ], + "assertions": [{"type": "contains", "value": "submitted"}] +} +``` + +### With Attachments + +```jsonl +{ + "id": "T003", + "input": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What's in this receipt?" + }, + { + "type": "image", + "source": "file://./fixtures/receipt.jpg" + } + ] + } + ], + "assertions": [ + { + "type": "contains", + "value": "amount" + } + ] +} +``` + +### Dynamic Mode (Simulator + Checkpoints) + +For coverage testing where conversation flow is unpredictable: + +```jsonl +{ + "id": "T004", + "name": "Expense Submission Coverage", + "simulator": { + "use": "workers.test.user-simulator", + "options": { + "metadata": { + "persona": "New employee unfamiliar with expense process", + "goal": "Submit a $3500 travel expense" + } + } + }, + "checkpoints": [ + { + "id": "ask_type", + "description": "Agent asks for expense type", + "assertion": { + "type": "contains", + "value": "type" + } + }, + { + "id": "call_create", + "description": "Agent calls create_expense", + "after": [ + "ask_type" + ], + "assertion": { + "type": "tool_called", + "name": "create_expense" + } + }, + { + "id": "confirm", + "description": "Agent confirms submission", + "after": [ + "call_create" + ], + "assertion": { + "type": "contains", + "value": "submitted" + } + } + ], + "max_turns": 10, + "timeout": "2m" +} +``` + +## Field Descriptions + +### Standard Mode Fields + +| Field | Type | Required | Description | +| ------------ | ------------------------------ | -------- | ------------------------------------------------- | +| `id` | string | Yes | Unique test identifier | +| `name` | string | No | Human-readable test name | +| `input` | string \| Message \| Message[] | Yes | Input: text, single message, or message array | +| `assertions` | array | No | Assertions to validate response (alias: `assert`) | +| `options` | object | No | `context.Options` passed to agent | +| `before` | string | No | Before script (e.g., `env_test.Before`) | +| `after` | string | No | After script (e.g., `env_test.After`) | + +**Note**: The `input` field supports three formats: + +- `string`: Simple text (converted to `[{role: "user", content: "..."}]`) +- `object`: Single message `{role: "...", content: "..."}` +- `array`: Message history `[{role: "user", ...}, {role: "assistant", ...}, ...]` + +### Dynamic Mode Fields + +| Field | Type | Required | Description | +| --------------------------- | ------ | -------- | ------------------------------------------ | +| `id` | string | Yes | Unique test identifier | +| `name` | string | No | Human-readable test name | +| `simulator` | object | Yes | User simulator configuration | +| `simulator.use` | string | Yes | Simulator agent ID (no prefix) | +| `simulator.options` | object | No | `context.Options` passed to simulator | +| `checkpoints` | array | Yes | Functionality checkpoints to verify | +| `checkpoints[].id` | string | Yes | Unique checkpoint identifier | +| `checkpoints[].description` | string | No | Human-readable description | +| `checkpoints[].assertion` | object | Yes | Assertion to verify | +| `checkpoints[].after` | array | No | Checkpoint IDs that must occur first | +| `max_turns` | int | No | Maximum turns before timeout (default: 20) | +| `timeout` | string | No | Maximum time (default: "5m") | +| `options` | object | No | `context.Options` passed to target agent | +| `before` | string | No | Before script function | +| `after` | string | No | After script function | + +## Before and After Scripts + +JSONL test cases can reference `*_test.ts` scripts for environment preparation: + +### Script Location + +Scripts are located in the agent's `src/` directory (as `*_test.ts` files): + +``` +assistants/expense/ +├── package.yao +├── prompts.yml +├── src/ +│ ├── index.ts # Main agent script +│ └── env_test.ts # Before/after functions +└── tests/ + ├── inputs.jsonl # Test cases + └── fixtures/ + └── receipt.jpg +``` + +### Script Interface + +```typescript +// src/env_test.ts + +// Before function - called before test case runs +// Returns context data that will be passed to After +export function Before(ctx: Context, testCase: TestCase): BeforeResult { + // Prepare database + const userId = Process("models.user.Create", { + name: "Test User", + email: "test@example.com", + }); + + // Prepare knowledge base + Process("knowledge.expense.Index", { + documents: [{ title: "Policy", content: "Max expense $5000" }], + }); + + return { + data: { userId, testId: testCase.id }, + }; +} + +// After function - called after test case completes (pass or fail) +export function After( + ctx: Context, + testCase: TestCase, + result: TestResult, + beforeData: any +) { + // Clean up database + if (beforeData?.userId) { + Process("models.user.Delete", beforeData.userId); + } + + // Clean up knowledge base + Process("knowledge.expense.Clear"); +} + +// Global before - called once before all test cases +export function BeforeAll(ctx: Context, testCases: TestCase[]): BeforeResult { + // One-time initialization + Process("models.migrate"); + return { data: { initialized: true } }; +} + +// Global after - called once after all test cases +export function AfterAll(ctx: Context, results: TestResult[], beforeData: any) { + // Final cleanup + Process("models.cleanup"); +} +``` + +### Test Case with Before/After + +```jsonl +{ + "id": "T001", + "name": "Submit expense with user context", + "before": "env_test.Before", + "after": "env_test.After", + "input": "Submit a $500 travel expense", + "assertions": [ + { + "type": "tool_called", + "name": "create_expense" + } + ] +} +``` + +### Global Before/After via CLI + +```bash +# Run with global before/after +yao agent test -i ./tests/inputs.jsonl \ + --before env_test.BeforeAll \ + --after env_test.AfterAll +``` + +### Execution Order + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Test Execution with Before/After │ +├─────────────────────────────────────────────────────────────────┤ +│ │ +│ 1. BeforeAll() - Global initialization (once) │ +│ ↓ │ +│ FOR EACH test case: │ +│ 2. Before() - Per-test initialization │ +│ ↓ │ +│ 3. Run test (call agent, check assertions) │ +│ ↓ │ +│ 4. After() - Per-test cleanup (always runs) │ +│ ↓ │ +│ 5. AfterAll() - Global cleanup (once) │ +│ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +**Note**: Script tests (`*_test.ts`) don't need before/after fields since they can call functions directly within the test. + +## Execution Flow + +### Standard Mode + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Standard Mode Execution │ +├─────────────────────────────────────────────────────────────────┤ +│ │ +│ 1. Parse test case │ +│ ├─ `input` is array? → Use as messages │ +│ └─ `input` is string? → Convert to [{role: "user", content}] │ +│ ↓ │ +│ 2. Call Agent.Stream(ctx, messages, options) │ +│ ↓ │ +│ 3. Run assertions against response │ +│ ├─ All PASS → Test PASSED ✅ │ +│ └─ Any FAIL → Test FAILED ❌ │ +│ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### Dynamic Mode + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Dynamic Mode Execution │ +├─────────────────────────────────────────────────────────────────┤ +│ │ +│ Initialize: │ +│ - pending_checkpoints = all checkpoints │ +│ - messages = [] │ +│ - turn_count = 0 │ +│ ↓ │ +│ LOOP: │ +│ 1. Call Simulator → get user input │ +│ 2. Append user message to messages │ +│ 3. Call Agent.Stream(ctx, messages, options) │ +│ 4. Append assistant response to messages │ +│ 5. Check response against pending_checkpoints │ +│ └─ If matched (and `after` satisfied) → move to reached │ +│ 6. Check termination: │ +│ ├─ All checkpoints reached → PASSED ✅ │ +│ ├─ Simulator signals goal_achieved → FAILED ❌ │ +│ ├─ turn_count >= max_turns → FAILED ❌ │ +│ └─ timeout exceeded → FAILED ❌ │ +│ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +## Assertion Types + +### Static Assertions + +| Type | Description | Example | +| ------------- | ---------------------- | ---------------------------------------------------------- | +| `contains` | Response contains text | `{"type": "contains", "value": "success"}` | +| `equals` | Exact match | `{"type": "equals", "value": "OK"}` | +| `regex` | Regex pattern match | `{"type": "regex", "pattern": "order-\\d+"}` | +| `json_path` | JSONPath value check | `{"type": "json_path", "path": "$.status", "value": "ok"}` | +| `tool_called` | Tool was invoked | `{"type": "tool_called", "name": "create_expense"}` | +| `type` | Value type check | `{"type": "type", "path": "$.count", "value": "number"}` | + +### Agent-Driven Assertions + +For semantic or fuzzy validation: + +```jsonl +{ + "type": "agent", + "use": "agents:workers.test.validator", + "options": { + "metadata": { + "criteria": "Response should be helpful and answer the user's question", + "tone": "professional and friendly" + } + } +} +``` + +### Script Assertions + +For custom validation logic: + +```jsonl +{ + "type": "script", + "use": "scripts:tests.validate-expense", + "options": { + "metadata": { + "min_amount": 100, + "max_amount": 10000 + } + } +} +``` + +## Script Testing with Agent Assertions + +Script tests can use Agent-driven assertions via `t.assert.Agent()`: + +```typescript +export function TestExpenseResponse(t: TestingT, ctx: Context) { + const messages = [ + { role: "user", content: "I want to submit an expense" }, + { role: "assistant", content: "What type of expense?" }, + { role: "user", content: "Travel, $500" }, + ]; + + const response = Process("agents.expense.Stream", ctx, messages); + + // Static assertion + t.assert.Contains(response.content, "confirm"); + + // Agent-driven assertion + t.assert.Agent(response.content, "workers.test.validator", { + metadata: { + criteria: "Response should ask for confirmation before creating expense", + conversation: messages, + }, + }); +} +``` + +## Standard Agent Interface + +All agent-driven features use `context.Options`: + +```go +type Options struct { + Skip *Skip `json:"skip,omitempty"` + Connector string `json:"connector,omitempty"` + Search any `json:"search,omitempty"` + Mode string `json:"mode,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` +} +``` + +### Generator Agent + +Called when `-i agents:xxx` is used: + +```go +options := &context.Options{ + Metadata: map[string]any{ + "test_mode": "generator", + "target_agent": "assistants.expense", + "count": 10, + "focus": "edge-cases", + }, +} +``` + +### Simulator Agent + +Called in dynamic mode to generate user input: + +```go +options := &context.Options{ + Metadata: map[string]any{ + "test_mode": "simulator", + "persona": "New employee", + "goal": "Submit expense", + "turn_number": 3, + }, +} +``` + +### Validator Agent + +Called for agent-driven assertions: + +```go +options := &context.Options{ + Metadata: map[string]any{ + "test_mode": "validator", + "criteria": "Response should be helpful", + }, +} +``` + +## Command Line Interface + +### Flags Reference + +| Flag | Long | Description | +| ---- | ------------- | ------------------------------------------------------------ | +| `-i` | `--input` | Input source: file path, message, or `agents:`/`scripts:` ID | +| `-n` | `--name` | Target agent ID (the agent being tested) | +| `-o` | `--output` | Output file path for results | +| `-c` | `--connector` | Override connector for the target agent | +| `-u` | `--user` | Test user ID (default: test-user) | +| `-t` | `--team` | Test team ID (default: test-team) | +| `-v` | `--verbose` | Verbose output | +| | `--ctx` | Path to context JSON file for custom authorization | +| | `--simulator` | Default simulator agent ID for dynamic mode | +| | `--before` | Global before script (e.g., `env_test.BeforeAll`) | +| | `--after` | Global after script (e.g., `env_test.AfterAll`) | +| | `--timeout` | Timeout per test case (default: 5m) | +| | `--parallel` | Number of parallel test cases | +| | `--runs` | Number of runs for stability analysis | +| | `--run` | Regex pattern to filter which tests to run | +| | `--fail-fast` | Stop on first failure | +| | `--dry-run` | Generate/parse tests without running | + +### Examples + +```bash +# Simple test +yao agent test -i "Hello, how are you?" -n assistants.chat + +# From JSONL file +yao agent test -i ./tests/expense.jsonl + +# Agent-generated tests +yao agent test -i "agents:workers.test.generator?count=10" -n assistants.expense + +# With simulator for dynamic mode +yao agent test -i ./tests/dynamic.jsonl --simulator workers.test.user-simulator + +# Parallel execution +yao agent test -i ./tests/expense.jsonl --parallel 5 + +# Verbose output +yao agent test -i ./tests/expense.jsonl -v +``` + +## Output Format + +### Console Output (Standard Mode) + +Standard mode shows each test case as a single line with input preview: + +``` +═══════════════════════════════════════════════════════════════ + Agent Test +═══════════════════════════════════════════════════════════════ +ℹ Agent: workers.system.keyword +ℹ Connector: deepseek.v3 +ℹ Input: ./tests/inputs.jsonl (42 test cases) +ℹ Timeout: 5m0s + +─────────────────────────────────────────────────────────────── + Running Tests +─────────────────────────────────────────────────────────────── +► [T001] 人工智能和机器学习正在改变我们�... PASSED (2.7s) +► [T002] The rapid development of cloud computing has re... PASSED (3.0s) +► [T003] 区块链技术是一种分布式账本技术�... PASSED (2.7s) +... + +─────────────────────────────────────────────────────────────── + Summary +─────────────────────────────────────────────────────────────── + Agent: workers.system.keyword + Connector: deepseek.v3 + Total: 42 + Passed: 42 + Failed: 0 + Pass Rate: 100.0% + Duration: 1.8m + + Output: ./tests/output-20251225185335.jsonl + +═══════════════════════════════════════════════════════════════ + ✨ ALL TESTS PASSED ✨ +═══════════════════════════════════════════════════════════════ +``` + +### Console Output (Dynamic Mode) + +Dynamic mode shows each test case as a tree with turns and checkpoints: + +``` +═══════════════════════════════════════════════════════════════ + Agent Test (Dynamic Mode) +═══════════════════════════════════════════════════════════════ +ℹ Agent: assistants.expense +ℹ Connector: openai.gpt4 +ℹ Input: ./tests/dynamic.jsonl (2 test cases) +ℹ Simulator: workers.test.user-simulator + +─────────────────────────────────────────────────────────────── + Running Tests +─────────────────────────────────────────────────────────────── +► [T001] Expense Submission Coverage + ├─ Turn 1: "Help me file an expense" → "What type of expense?" + │ └─ ✓ checkpoint: ask_type + ├─ Turn 2: "Client dinner, $250" → "I'll create... Please confirm." + │ └─ ✓ checkpoint: call_create (tool: create_expense) + └─ Turn 3: "Yes, confirm" → "Expense submitted! Reference: EXP-001" + └─ ✓ checkpoint: confirm + PASSED (6.8s) - 3 turns, 3/3 checkpoints + +► [T002] Expense with Attachment + ├─ Turn 1: "Submit receipt" + [receipt.jpg] → "What type?" + │ └─ ✓ checkpoint: ask_type + ├─ Turn 2: "Business lunch" → "Amount from receipt: $85.50. Confirm?" + │ └─ ✓ checkpoint: extract_amount + └─ Turn 3: "Yes" → "Submitted! Reference: EXP-002" + └─ ✓ checkpoint: confirm + PASSED (8.2s) - 3 turns, 3/3 checkpoints + +─────────────────────────────────────────────────────────────── + Summary +─────────────────────────────────────────────────────────────── + Agent: assistants.expense + Connector: openai.gpt4 + Simulator: workers.test.user-simulator + Total: 2 + Passed: 2 + Failed: 0 + Pass Rate: 100.0% + Duration: 15.0s + + Output: ./tests/output-20251225190000.jsonl + +═══════════════════════════════════════════════════════════════ + ✨ ALL TESTS PASSED ✨ +═══════════════════════════════════════════════════════════════ +``` + +### Console Output (Parallel Mode) + +When `--parallel N` is enabled, tests run concurrently. Output is buffered and displayed as complete test trees: + +``` +═══════════════════════════════════════════════════════════════ + Agent Test (Parallel: 5) +═══════════════════════════════════════════════════════════════ +ℹ Agent: assistants.expense +ℹ Input: ./tests/dynamic.jsonl (10 test cases) +ℹ Parallel: 5 concurrent + +─────────────────────────────────────────────────────────────── + Running Tests (5 parallel) +─────────────────────────────────────────────────────────────── +► [T003] Quick approval flow + ├─ Turn 1: "Approve expense EXP-001" → "Approved!" + └─ ✓ checkpoint: approved + PASSED (1.2s) - 1 turn, 1/1 checkpoints + +► [T001] Expense Submission Coverage + ├─ Turn 1: "Help me file an expense" → "What type?" + │ └─ ✓ checkpoint: ask_type + ├─ Turn 2: "Client dinner, $250" → "Confirm?" + │ └─ ✓ checkpoint: call_create + └─ Turn 3: "Yes" → "Submitted!" + └─ ✓ checkpoint: confirm + PASSED (6.8s) - 3 turns, 3/3 checkpoints + +► [T002] Expense with Attachment + ├─ Turn 1: "Submit receipt" + [receipt.jpg] → "What type?" + ... + PASSED (8.2s) - 3 turns, 3/3 checkpoints + +[Progress: 3/10 completed, 5 running...] + +► [T004] Rejection flow + ... + PASSED (4.5s) - 2 turns, 2/2 checkpoints + +─────────────────────────────────────────────────────────────── + Summary +─────────────────────────────────────────────────────────────── + Total: 10 + Passed: 10 + Failed: 0 + Pass Rate: 100.0% + Duration: 25.3s (effective: 2.5s/test with 5 parallel) + +═══════════════════════════════════════════════════════════════ + ✨ ALL TESTS PASSED ✨ +═══════════════════════════════════════════════════════════════ +``` + +**Note**: In parallel mode, test results appear in completion order (not input order). Each test's output is buffered and displayed as a complete tree to maintain readability. + +### JSON Output (Standard Mode) + +Output file is a JSON object with `summary`, `environment`, `results`, and `metadata`: + +```json +{ + "summary": { + "total": 3, + "passed": 3, + "failed": 0, + "skipped": 0, + "errors": 0, + "timeouts": 0, + "duration_ms": 5100, + "agent_id": "assistants.expense", + "agent_path": "/path/to/expense" + }, + "environment": { + "user_id": "test-user", + "team_id": "test-team", + "locale": "en-us" + }, + "results": [ + { + "id": "expense-turn1", + "status": "passed", + "input": [{ "role": "user", "content": "I want to submit an expense" }], + "output": "What type of expense would you like to submit?", + "duration_ms": 1200 + }, + { + "id": "expense-turn2", + "status": "passed", + "input": [ + { "role": "user", "content": "I want to submit an expense" }, + { "role": "assistant", "content": "What type?" }, + { "role": "user", "content": "Business travel, $3500" } + ], + "output": "Confirm $3500 expense?", + "duration_ms": 2100 + } + ], + "metadata": { + "started_at": "2025-12-25T10:00:00Z", + "completed_at": "2025-12-25T10:00:05Z", + "input_file": "./tests/expense.jsonl" + } +} +``` + +### JSON Output (Dynamic Mode) + +Dynamic mode adds `turns` and `checkpoints` to each result: + +```json +{ + "summary": { + "total": 1, + "passed": 1, + "failed": 0, + "duration_ms": 6800, + "agent_id": "assistants.expense" + }, + "results": [ + { + "id": "expense-dynamic", + "name": "Expense Coverage Test", + "status": "passed", + "turns": [ + { + "turn": 1, + "input": "Help me file an expense", + "output": "What type?" + }, + { "turn": 2, "input": "Client dinner, $250", "output": "Confirm?" }, + { "turn": 3, "input": "Yes", "output": "Submitted!" } + ], + "checkpoints": [ + { "id": "ask_type", "reached_at_turn": 1, "passed": true }, + { "id": "call_create", "reached_at_turn": 2, "passed": true }, + { "id": "confirm", "reached_at_turn": 3, "passed": true } + ], + "total_turns": 3, + "duration_ms": 6800 + } + ], + "metadata": { + "started_at": "2025-12-25T10:00:00Z", + "completed_at": "2025-12-25T10:00:07Z" + } +} +``` + +## User Simulator Agent + +### Interface + +```typescript +interface SimulatorInput { + persona: string; + goal: string; + conversation: Message[]; + turn_number: number; + max_turns: number; +} + +interface SimulatorOutput { + input: string; + goal_achieved: boolean; + reasoning?: string; +} +``` + +### Example Prompt + +``` +You are simulating a user with the following characteristics: + +Persona: {{persona}} +Goal: {{goal}} + +Current conversation: +{{conversation}} + +Generate the next user message to continue toward the goal. +If the goal has been achieved, set goal_achieved to true. + +Respond in JSON format: +{ + "input": "your response as the user", + "goal_achieved": true/false, + "reasoning": "brief explanation" +} +``` + +## Backward Compatibility + +Existing single-turn tests work unchanged: + +```jsonl +// Simple string input +{"id": "T001", "input": "Hello", "assertions": [...]} + +// Equivalent to array format +{"id": "T001", "input": [{"role": "user", "content": "Hello"}], "assertions": [...]} +``` + +## Error Handling + +### Standard Mode Errors + +| Error Type | Behavior | Output | +| ---------------- | ----------- | ---------------------------- | +| Agent timeout | Test FAILED | `error: "timeout after 30s"` | +| Agent error | Test FAILED | `error: "agent error: ..."` | +| Assertion failed | Test FAILED | `assertion_errors: [...]` | + +### Dynamic Mode Errors + +| Error Type | Behavior | Output | +| --------------------------- | ----------- | ----------------------------------- | +| All checkpoints reached | Test PASSED | `status: "passed"` | +| Checkpoints missing | Test FAILED | `error: "missing checkpoints: ..."` | +| Max turns exceeded | Test FAILED | `error: "max turns (20) exceeded"` | +| Timeout exceeded | Test FAILED | `error: "timeout after 5m"` | +| Simulator error | Test FAILED | `error: "simulator error: ..."` | +| Checkpoint assertion failed | Test FAILED | `error: "checkpoint X failed"` | + +## Current Implementation Status + +| Feature | Status | Notes | +| ----------------------- | ---------- | -------------------------------------------------- | +| Simple text input | ✅ Done | `input: "Hello"` | +| Message history | ✅ Done | `input: [{role, content}, ...]` | +| File attachments | ✅ Done | `file://` protocol in content parts | +| Static assertions | ✅ Done | contains, equals, regex, json_path, etc. | +| Before/After hooks | ✅ Done | `before/after` in JSONL, `--before/--after` in CLI | +| Agent-driven assertions | ✅ Done | `type: "agent"` + `t.assert.Agent()` JSAPI | +| Agent-driven input | ✅ Done | `-i agents:xxx` for test generation | +| Dry-run mode | ✅ Done | `--dry-run` to preview generated tests | +| Dynamic mode | ✅ Done | Simulator + Checkpoints | +| Console output | ✅ Done | Dynamic mode tree output, checkpoint display | + +## Open Questions + +1. **Message Generation**: Should we provide a helper to generate message history from a script? + +2. **Snapshot Testing**: Should we support "golden file" comparison for responses? + +3. **Retry Logic**: If a test fails, should we support automatic retry? diff --git a/agent/test/README.md b/agent/test/README.md index bdbd4e14..d49bdbd2 100644 --- a/agent/test/README.md +++ b/agent/test/README.md @@ -1,10 +1,10 @@ # Agent Test Framework -A testing framework for Yao AI agents with support for assertions, stability analysis, and CI integration. +A comprehensive testing framework for Yao AI agents with support for standard testing, dynamic (simulator-driven) testing, agent-driven assertions, and CI integration. ## Quick Start -### Agent Tests +### Standard Tests ```bash # Test with direct message (auto-detect agent from current directory) @@ -24,6 +24,26 @@ yao agent test -i tests/inputs.jsonl -o report.html yao agent test -i tests/inputs.jsonl --runs 5 ``` +### Agent-Driven Input + +```bash +# Generate test cases using an agent +yao agent test -i "agents:tests.generator-agent?count=10" -n assistants.expense + +# Preview generated tests without running (dry-run) +yao agent test -i "agents:tests.generator-agent?count=5" -n assistants.expense --dry-run +``` + +### Dynamic Mode (Simulator) + +```bash +# Run dynamic tests with simulator +yao agent test -i tests/dynamic.jsonl --simulator tests.simulator-agent + +# See detailed turn-by-turn output +yao agent test -i tests/dynamic.jsonl -v +``` + ### Script Tests ```bash @@ -39,7 +59,7 @@ yao agent test -i scripts.expense.setup --ctx tests/context.json -v ## Input Modes -The `-i` flag supports three input modes: +The `-i` flag supports multiple input modes: ### 1. JSONL File Mode @@ -64,64 +84,819 @@ yao agent test -i "Extract keywords from this text" yao agent test -i "Hello" -n workers.system.keyword ``` -Output is printed to stdout (or saved to `-o` if specified). +### 3. Agent-Driven Input Mode -### 3. Script Test Mode - -Test agent handler scripts (hooks, tools, setup functions): +Generate test cases using a generator agent: + +```bash +# Basic usage (-n specifies the target agent to test) +yao agent test -i "agents:tests.generator-agent" -n assistants.expense + +# With parameters +yao agent test -i "agents:tests.generator-agent?count=10&focus=edge-cases" -n assistants.expense + +# Dry-run to preview generated tests +yao agent test -i "agents:tests.generator-agent?count=5" -n assistants.expense --dry-run +``` + +**Note**: The `-n` flag is **required** for agent-driven input mode to specify which agent to test. The generator agent creates test cases for the target agent. + +### 4. Script Test Mode + +Test agent handler scripts: ```bash -# Run all tests in a script module yao agent test -i scripts.expense.setup -v - -# Run specific tests with filtering -yao agent test -i scripts.expense.setup --run "TestSystemReady" - -# Run with custom context -yao agent test -i scripts.expense.setup --ctx tests/context.json -v ``` Script test input format: `scripts..` (e.g., `scripts.expense.setup` → `assistants/expense/src/setup_test.ts`). -**Writing Test Scripts:** +### 5. Script-Generated Input Mode -Test scripts should be placed alongside the source files with `_test.ts` or `_test.js` suffix: +Generate test cases using a script: -``` -assistants/expense/src/ -├── setup.ts # Source file -├── setup_test.ts # Test file -├── tools.ts -└── tools_test.ts +```bash +yao agent test -i "scripts:tests.gen.Generate" -n assistants.expense ``` -Test functions must follow the naming convention `Test*` and accept `(t: testing.T, ctx: agent.Context)`: +**Note**: `scripts.xxx` (with dot) runs script tests, while `scripts:xxx` (with colon) generates test cases from a script. + +## Test Modes + +### Standard Mode + +Single call to agent with optional message history. Each test is independent and stateless. + +```jsonl +{ + "id": "T001", + "input": "Hello", + "assert": { + "type": "contains", + "value": "Hi" + } +} +``` + +### Dynamic Mode + +Simulator-driven testing with checkpoint validation. A simulator agent generates user messages while checkpoints verify agent behavior. + +```jsonl +{ + "id": "T001", + "input": "I want to order coffee", + "simulator": { + "use": "tests.simulator-agent", + "options": { + "metadata": { + "persona": "Customer", + "goal": "Order a latte" + } + } + }, + "checkpoints": [ + { + "id": "greeting", + "assert": { + "type": "regex", + "value": "(?i)hello" + } + }, + { + "id": "ask_size", + "after": [ + "greeting" + ], + "assert": { + "type": "regex", + "value": "(?i)size" + } + } + ], + "max_turns": 10 +} +``` + +## Command Line Options + +| Flag | Description | Default | +| ------------- | -------------------------------------------------------- | -------------------------- | +| `-i` | Input: JSONL file, message, `agents:xxx`, or `scripts:x` | (required) | +| `-o` | Output file path | `output-{timestamp}.jsonl` | +| `-n` | Agent ID (optional, auto-detected) | auto-detect | +| `-a` | Application directory | auto-detect | +| `-e` | Environment file | - | +| `-c` | Override connector | agent default | +| `-u` | Test user ID | `test-user` | +| `-t` | Test team ID | `test-team` | +| `-r` | Reporter agent ID for custom report | built-in | +| `-v` | Verbose output | false | +| `--ctx` | Path to context JSON file for custom authorization | - | +| `--simulator` | Default simulator agent ID for dynamic mode | - | +| `--before` | Global BeforeAll hook (e.g., `env_test.BeforeAll`) | - | +| `--after` | Global AfterAll hook (e.g., `env_test.AfterAll`) | - | +| `--runs` | Runs per test (stability analysis) | 1 | +| `--run` | Regex pattern to filter which tests to run | - | +| `--timeout` | Timeout per test | 5m | +| `--parallel` | Parallel test cases | 1 | +| `--fail-fast` | Stop on first failure | false | +| `--dry-run` | Generate test cases without running them | false | + +## Input Format (JSONL) + +Each line is a JSON object. Below are examples organized by scenario. + +### Scenario 1: Simple Text Input + +Basic test with string input: + +```jsonl +{"id": "greeting-basic", "input": "Hello, how are you?"} +{"id": "greeting-chinese", "input": "你好，请问有什么可以帮助你的？"} +``` + +### Scenario 2: With Assertions + +Validate response content: + +```jsonl +{"id": "keyword-extract", "input": "Extract keywords from: AI and machine learning", "assert": {"type": "contains", "value": "AI"}} +{"id": "json-response", "input": "What's the weather?", "assert": {"type": "json_path", "path": "need_search", "value": true}} +{"id": "no-error", "input": "Help me", "assert": {"type": "not_contains", "value": "error"}} +``` + +### Scenario 3: Multiple Assertions + +All assertions must pass: + +```jsonl +{ + "id": "expense-submit", + "input": "Submit $500 travel expense", + "assert": [ + { + "type": "contains", + "value": "expense" + }, + { + "type": "not_contains", + "value": "error" + }, + { + "type": "regex", + "value": "(?i)(submitted|created|confirmed)" + } + ] +} +``` + +### Scenario 4: Conversation History + +Test with multi-turn context: + +```jsonl +{ + "id": "expense-confirm", + "input": [ + { + "role": "user", + "content": "Submit an expense" + }, + { + "role": "assistant", + "content": "What type of expense?" + }, + { + "role": "user", + "content": "Travel, $500" + }, + { + "role": "assistant", + "content": "Please confirm: $500 travel expense" + }, + { + "role": "user", + "content": "Yes, confirm" + } + ], + "assert": { + "type": "regex", + "value": "(?i)(submitted|created)" + } +} +``` + +### Scenario 5: With File Attachments + +Test with images or documents: + +```jsonl +{ + "id": "receipt-analyze", + "input": { + "role": "user", + "content": [ + { + "type": "text", + "text": "Analyze this receipt" + }, + { + "type": "image", + "source": "file://fixtures/receipt.jpg" + } + ] + }, + "assert": { + "type": "contains", + "value": "amount" + } +} +``` + +### Scenario 6: Agent-Driven Assertion + +Use LLM to validate response semantics: + +```jsonl +{ + "id": "helpful-response", + "input": "How do I reset my password?", + "assert": { + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "Response should provide clear step-by-step instructions" + } +} +``` + +### Scenario 7: With Options + +Override connector or skip features: + +```jsonl +{"id": "fast-model", "input": "Quick question", "options": {"connector": "deepseek.v3", "skip": {"history": true, "trace": true}}} +{"id": "scenario-test", "input": "Query users", "options": {"metadata": {"scenario": "filter"}}, "assert": {"type": "json_path", "path": "from", "value": "users"}} +``` + +### Scenario 8: With Before/After Hooks + +Setup and teardown for each test: + +```jsonl +{ + "id": "with-user-data", + "input": "Show my expenses", + "before": "env_test.Before", + "after": "env_test.After", + "assert": { + "type": "contains", + "value": "expense" + } +} +``` + +### Scenario 9: Skip Test + +Temporarily disable a test: + +```jsonl +{ + "id": "wip-feature", + "input": "New feature test", + "skip": true +} +``` + +### Scenario 10: Dynamic Mode (Simulator) + +Multi-turn testing with user simulator: + +```jsonl +{ + "id": "coffee-order", + "input": "I want to order coffee", + "simulator": { + "use": "tests.simulator-agent", + "options": { + "metadata": { + "persona": "Regular customer", + "goal": "Order a medium latte" + } + } + }, + "checkpoints": [ + { + "id": "greeting", + "assert": { + "type": "regex", + "value": "(?i)(hello|hi|help)" + } + }, + { + "id": "ask-size", + "after": [ + "greeting" + ], + "assert": { + "type": "regex", + "value": "(?i)size" + } + }, + { + "id": "confirm", + "after": [ + "ask-size" + ], + "assert": { + "type": "regex", + "value": "(?i)confirm" + } + } + ], + "max_turns": 10 +} +``` + +### Scenario 11: Dynamic Mode with Optional Checkpoint + +Some checkpoints are optional: + +```jsonl +{ + "id": "expense-flow", + "input": "Submit expense", + "simulator": { + "use": "tests.simulator-agent", + "options": { + "metadata": { + "persona": "New employee", + "goal": "Submit $500 travel expense" + } + } + }, + "checkpoints": [ + { + "id": "ask-type", + "assert": { + "type": "regex", + "value": "(?i)type" + } + }, + { + "id": "suggest-category", + "required": false, + "assert": { + "type": "contains", + "value": "category" + } + }, + { + "id": "confirm", + "after": [ + "ask-type" + ], + "assert": { + "type": "regex", + "value": "(?i)confirm" + } + } + ], + "max_turns": 15 +} +``` + +### Standard Mode Fields + +| Field | Type | Required | Description | +| ---------- | ------------------------------ | -------- | ------------------------------------- | +| `id` | string | Yes | Test case ID | +| `input` | string \| Message \| []Message | Yes | Test input | +| `assert` | Assertion \| []Assertion | No | Assertion rules | +| `expected` | any | No | Expected output (exact match) | +| `user` | string | No | Override user ID for this test | +| `team` | string | No | Override team ID for this test | +| `metadata` | map | No | Additional metadata for hooks | +| `options` | Options | No | Context options | +| `timeout` | string | No | Override timeout (e.g., "30s") | +| `skip` | bool | No | Skip this test | +| `before` | string | No | Before hook (e.g., `env_test.Before`) | +| `after` | string | No | After hook (e.g., `env_test.After`) | + +### Dynamic Mode Fields + +| Field | Type | Required | Description | +| ----------------------------- | ------ | -------- | -------------------------------------- | +| `id` | string | Yes | Test case ID | +| `input` | string | Yes | Initial user message | +| `simulator` | object | Yes | Simulator configuration | +| `simulator.use` | string | Yes | Simulator agent ID (no prefix) | +| `simulator.options` | object | No | Simulator options | +| `simulator.options.metadata` | map | No | Metadata (persona, goal, etc.) | +| `simulator.options.connector` | string | No | Override simulator connector | +| `checkpoints` | array | Yes | Checkpoints to verify | +| `checkpoints[].id` | string | Yes | Checkpoint identifier | +| `checkpoints[].description` | string | No | Human-readable description | +| `checkpoints[].assert` | object | Yes | Assertion to validate | +| `checkpoints[].after` | array | No | Checkpoint IDs that must occur first | +| `checkpoints[].required` | bool | No | Is checkpoint required (default: true) | +| `max_turns` | int | No | Maximum turns (default: 20) | +| `timeout` | string | No | Override timeout (e.g., "2m") | + +### Options + +The `options` field allows per-test-case configuration: + +| Field | Type | Description | +| ------------------------ | ------ | ------------------------------------------ | +| `connector` | string | Override connector (e.g., `"deepseek.v3"`) | +| `mode` | string | Agent mode (default: `"chat"`) | +| `search` | bool | Enable/disable search mode | +| `disable_global_prompts` | bool | Temporarily disable global prompts | +| `metadata` | map | Custom data passed to hooks | +| `skip` | object | Skip configuration (see below) | + +### Options.skip + +| Field | Type | Description | +| --------- | ---- | ----------------------- | +| `history` | bool | Skip history loading | +| `trace` | bool | Skip trace logging | +| `output` | bool | Skip output to client | +| `keyword` | bool | Skip keyword extraction | +| `search` | bool | Skip auto search | + +### Input Types + +| Type | Description | Example | +| ----------- | -------------------- | ----------------------------------------------------- | +| `string` | Simple text | `"Hello world"` | +| `Message` | Single message | `{"role": "user", "content": "..."}` | +| `[]Message` | Conversation history | `[{"role": "user", ...}, {"role": "assistant", ...}]` | + +## Assertions + +Use `assert` for flexible validation. If `assert` is defined, it takes precedence over `expected`. + +### Static Assertions + +| Type | Description | Example | +| -------------- | ----------------------------- | --------------------------------------------------------- | +| `equals` | Exact match | `{"type": "equals", "value": {"key": "val"}}` | +| `contains` | Output contains value | `{"type": "contains", "value": "keyword"}` | +| `not_contains` | Output does not contain value | `{"type": "not_contains", "value": "error"}` | +| `json_path` | Extract JSON path and compare | `{"type": "json_path", "path": "$.field", "value": true}` | +| `regex` | Match regex pattern | `{"type": "regex", "value": "\\d+"}` | +| `type` | Check output type | `{"type": "type", "value": "object"}` | + +### Assertion Fields + +| Field | Type | Description | +| --------- | ------ | -------------------------------------------------------- | +| `type` | string | Assertion type (required) | +| `value` | any | Expected value or pattern | +| `path` | string | JSON path for `json_path` type | +| `script` | string | Script name for `script` type | +| `use` | string | Agent/script ID for `agent` type (with `agents:` prefix) | +| `options` | object | Options for agent assertions | +| `message` | string | Custom failure message | +| `negate` | bool | Invert the assertion result | + +### Agent-Driven Assertions + +For semantic or fuzzy validation using an LLM: + +```jsonl +{ + "id": "T001", + "input": "Hello", + "assert": { + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "Response should be friendly and helpful" + } +} +``` + +The validator agent receives the output and criteria, then returns `{"passed": true/false, "reason": "..."}`. + +### Script Assertions + +For custom validation logic: + +```jsonl +{ + "id": "T001", + "input": "Test", + "assert": { + "type": "script", + "script": "scripts.test.Validate" + } +} +``` + +### Multiple Assertions + +All assertions must pass: + +```jsonl +{ + "id": "T001", + "input": "Hello", + "assert": [ + { + "type": "contains", + "value": "Hi" + }, + { + "type": "not_contains", + "value": "error" + }, + { + "type": "json_path", + "path": "status", + "value": "ok" + } + ] +} +``` + +## File Attachments + +Test inputs support file attachments using the `file://` protocol: + +```jsonl +{ + "id": "T001", + "input": { + "role": "user", + "content": [ + { + "type": "text", + "text": "Analyze this image" + }, + { + "type": "image", + "source": "file://fixtures/receipt.jpg" + } + ] + } +} +``` + +Supported types: images (jpg, png, gif, webp), audio (wav, mp3), documents (pdf, doc, txt). + +## Before/After Hooks + +Hooks allow you to run setup and teardown code before and after tests. Hook scripts must be placed in the agent's `src/` directory with `_test.ts` suffix. + +### Hook Types + +| Hook | Scope | When Called | Use Case | +| ----------- | -------- | --------------------- | ------------------------------- | +| `Before` | Per-test | Before each test case | Create test data, setup context | +| `After` | Per-test | After each test case | Cleanup test data, log results | +| `BeforeAll` | Global | Once before all tests | Database migration, init | +| `AfterAll` | Global | Once after all tests | Global cleanup, report | + +### Execution Order + +``` +BeforeAll (global) + ├─ Before (test 1) + │ └─ Test 1 execution + │ └─ After (test 1) + ├─ Before (test 2) + │ └─ Test 2 execution + │ └─ After (test 2) + └─ ... +AfterAll (global) +``` + +### Per-Test Hooks + +Defined in JSONL, scripts located in agent's `src/` directory: + +```jsonl +{ + "id": "T001", + "input": "Test", + "before": "env_test.Before", + "after": "env_test.After" +} +``` + +### Global Hooks + +Via CLI flags: + +```bash +yao agent test -i tests/inputs.jsonl --before env_test.BeforeAll --after env_test.AfterAll +``` + +### Hook Function Signatures + +```typescript +// assistants/expense/src/env_test.ts + +/** + * Before - Called before each test case + * @param ctx - Agent context with user/team info + * @param testCase - The test case about to run + * @returns any - Data passed to After hook (optional) + */ +export function Before(ctx: Context, testCase: TestCase): any { + const userId = Process("models.user.Create", { name: "Test User" }); + return { userId }; // This data is passed to After +} + +/** + * After - Called after each test case (pass or fail) + * @param ctx - Agent context + * @param testCase - The test case that ran + * @param result - Test result with status, output, duration + * @param beforeData - Data returned from Before hook + */ +export function After( + ctx: Context, + testCase: TestCase, + result: TestResult, + beforeData: any +) { + if (beforeData?.userId) { + Process("models.user.Delete", beforeData.userId); + } + if (result.status === "failed") { + console.log(`Test ${testCase.id} failed: ${result.error}`); + } +} + +/** + * BeforeAll - Called once before all tests + * @param ctx - Agent context + * @param testCases - Array of all test cases + * @returns any - Data passed to AfterAll hook (optional) + */ +export function BeforeAll(ctx: Context, testCases: TestCase[]): any { + Process("models.migrate"); + return { initialized: true, count: testCases.length }; +} + +/** + * AfterAll - Called once after all tests complete + * @param ctx - Agent context + * @param results - Array of all test results + * @param beforeData - Data returned from BeforeAll hook + */ +export function AfterAll(ctx: Context, results: TestResult[], beforeData: any) { + const passed = results.filter((r) => r.status === "passed").length; + console.log(`Tests completed: ${passed}/${results.length} passed`); + Process("models.cleanup"); +} +``` + +### Hook Parameters + +**Context** - Agent execution context: + +```typescript +interface Context { + user_id: string; // Test user ID + team_id: string; // Test team ID + locale: string; // Locale (e.g., "en-us") + metadata: object; // Custom metadata from test case +} +``` + +**TestCase** - Test case definition: + +```typescript +interface TestCase { + id: string; // Test case ID + input: any; // Test input (string, Message, or Message[]) + assert?: object; // Assertion rules + expected?: any; // Expected output + user?: string; // Override user ID + team?: string; // Override team ID + metadata?: object; // Custom metadata + options?: object; // Context options + timeout?: string; // Timeout (e.g., "30s") + skip?: boolean; // Skip flag + before?: string; // Before hook reference + after?: string; // After hook reference +} +``` + +**TestResult** - Test execution result: + +```typescript +interface TestResult { + id: string; // Test case ID + status: string; // "passed" | "failed" | "error" | "skipped" | "timeout" + input: any; // Actual input sent + output: any; // Agent response + expected?: any; // Expected output (if defined) + error?: string; // Error message (if failed) + duration_ms: number; // Execution time in milliseconds + assertions?: object[]; // Assertion results +} +``` + +### Common Use Cases + +**Database Setup/Teardown**: + +```typescript +export function Before(ctx: Context, testCase: TestCase): any { + // Create test records + const user = Process("models.user.Create", { + name: "Test", + email: "test@example.com", + }); + const expense = Process("models.expense.Create", { + user_id: user.id, + amount: 100, + }); + return { user, expense }; +} + +export function After( + ctx: Context, + testCase: TestCase, + result: TestResult, + data: any +) { + // Clean up in reverse order + if (data?.expense) Process("models.expense.Delete", data.expense.id); + if (data?.user) Process("models.user.Delete", data.user.id); +} +``` + +**Conditional Setup Based on Metadata**: + +```typescript +export function Before(ctx: Context, testCase: TestCase): any { + const scenario = testCase.metadata?.scenario || "default"; + + if (scenario === "empty_db") { + Process("models.expense.DeleteAll"); + } else if (scenario === "with_data") { + Process("scripts.tests.seed.LoadTestData"); + } + + return { scenario }; +} +``` + +**Logging and Debugging**: + +```typescript +export function After( + ctx: Context, + testCase: TestCase, + result: TestResult, + data: any +) { + if (result.status === "failed") { + console.log("=== Test Failed ==="); + console.log("Test ID:", testCase.id); + console.log("Input:", JSON.stringify(testCase.input)); + console.log("Output:", JSON.stringify(result.output)); + console.log("Error:", result.error); + } +} +``` + +## Script Testing + +Test agent handler scripts with the `t.assert` API: ```typescript // assistants/expense/src/setup_test.ts import { SystemReady } from "./setup"; -// Test function signature: function Test*(t: testing.T, ctx: agent.Context) -export function TestSystemReady(t: testing.T, ctx: agent.Context) { +export function TestSystemReady(t: TestingT, ctx: Context) { const result = SystemReady(ctx); - // Use t.assert for assertions - t.assert.True(result.success, "SystemReady should succeed"); + t.assert.True(result.success, "Should succeed"); t.assert.Equal(result.status, "ready", "Status should be ready"); t.assert.NotNil(result.data, "Data should not be nil"); } -export function TestSystemReadyError(t: testing.T, ctx: agent.Context) { - // Access context properties - console.log("Testing with user:", ctx.authorized.user_id); +export function TestWithAgentAssertion(t: TestingT, ctx: Context) { + const response = Process("agents.expense.Stream", ctx, messages); - // Test error handling - const result = SystemReady(ctx); - t.assert.False(result.error, "Should not have error"); + // Static assertion + t.assert.Contains(response.content, "confirm"); + + // Agent-driven assertion + t.assert.Agent(response.content, "tests.validator-agent", { + criteria: "Response should ask for confirmation", + }); } ``` -**Available Assertions:** +### Available Assertions | Method | Description | | -------------------------------- | ------------------------------ | @@ -133,385 +908,78 @@ export function TestSystemReadyError(t: testing.T, ctx: agent.Context) { | `t.assert.NotNil(value, msg)` | Assert value is not nil | | `t.assert.Contains(s, sub, msg)` | Assert string contains substr | | `t.assert.Len(arr, n, msg)` | Assert array/string length | +| `t.assert.Agent(resp, id, opts)` | Agent-driven assertion | -**Test Control:** +## Dynamic Mode -| Method | Description | -| -------------- | ---------------------------- | -| `t.Log(msg)` | Log a message | -| `t.Error(msg)` | Mark test as failed with msg | -| `t.Fatal(msg)` | Mark failed and stop test | -| `t.Skip(msg)` | Skip this test | - -## Command Line Options - -| Flag | Description | Default | -| ------------- | -------------------------------------------------- | -------------------------- | -| `-i` | Input: JSONL file path, message, or script ID | (required) | -| `-o` | Output file path | `output-{timestamp}.jsonl` | -| `-n` | Agent ID (optional, auto-detected) | auto-detect | -| `-c` | Override connector | agent default | -| `-u` | Test user ID | `test-user` | -| `-t` | Test team ID | `test-team` | -| `--ctx` | Path to context JSON file for custom authorization | - | -| `-r` | Reporter agent ID | built-in | -| `--runs` | Runs per test (stability analysis) | 1 | -| `--run` | Regex pattern to filter which tests to run | - | -| `--timeout` | Timeout per test | 5m | -| `--parallel` | Parallel test cases | 1 | -| `-v` | Verbose output | false | -| `--fail-fast` | Stop on first failure | false | - -## Agent Resolution - -The agent is resolved in the following priority order: - -1. **Explicit `-n` flag**: `yao agent test -i "msg" -n my.agent` -2. **Path-based detection**: Traverse up from input file to find `package.yao` -3. **Current directory**: For direct message mode, look for `package.yao` in cwd - -Example directory structure: - -``` -assistants/workers/system/keyword/ -├── package.yao <- Agent definition (auto-detected) -├── prompts.yml -├── src/ -│ └── index.ts -└── tests/ - └── inputs.jsonl <- Input file -``` - -## Input Format (JSONL) - -Each line is a JSON object: - -```jsonl -{"id": "T001", "input": "Simple text"} -{"id": "T002", "input": {"role": "user", "content": "Message with role"}} -{"id": "T003", "input": [{"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello"}, {"role": "user", "content": "Follow-up"}]} -{"id": "T004", "input": "Test", "assert": {"type": "json_path", "path": "field", "value": true}} -{"id": "T005", "input": "Skip this", "skip": true} -``` - -### Fields - -| Field | Type | Required | Description | -| ---------- | ------------------------------ | -------- | ------------------------------ | -| `id` | string | Yes | Test case ID | -| `input` | string \| Message \| []Message | Yes | Test input | -| `assert` | Assertion \| []Assertion | No | Assertion rules | -| `expected` | any | No | Expected output (exact match) | -| `user` | string | No | Override user ID | -| `team` | string | No | Override team ID | -| `options` | Options | No | Context options (see below) | -| `timeout` | string | No | Override timeout (e.g., "30s") | -| `skip` | bool | No | Skip this test | -| `metadata` | map | No | Additional metadata | - -### Options - -The `options` field allows per-test-case configuration that maps to `context.Options`: - -| Field | Type | Description | -| ------------------------ | ------ | -------------------------------------------- | -| `connector` | string | Override connector (e.g., `"deepseek.v3"`) | -| `mode` | string | Agent mode (default: `"chat"`) | -| `search` | bool | Enable/disable search mode (default: `true`) | -| `disable_global_prompts` | bool | Temporarily disable global prompts | -| `metadata` | map | Custom data passed to hooks (e.g., scenario) | -| `skip` | object | Skip configuration (see below) | - -#### Options.skip - -| Field | Type | Description | -| --------- | ---- | ----------------------- | -| `history` | bool | Skip history loading | -| `trace` | bool | Skip trace logging | -| `output` | bool | Skip output to client | -| `keyword` | bool | Skip keyword extraction | -| `search` | bool | Skip auto search | - -**Example with options:** +For testing complex conversation flows where the path is unpredictable: ```jsonl { - "id": "T001", - "input": "Query users with status active", - "options": { - "connector": "deepseek.v3", - "metadata": { - "scenario": "filter" - }, - "skip": { - "trace": true + "id": "coffee-order", + "input": "I want to order coffee", + "simulator": { + "use": "tests.simulator-agent", + "options": { + "metadata": { + "persona": "Customer ordering a latte", + "goal": "Complete the coffee order" + } } }, - "assert": { - "type": "json_path", - "path": "from", - "value": "users" - } -} -``` - -**Using metadata for hook scenarios:** - -The `options.metadata` field is passed to agent hooks. For example, a Create Hook can read `options.metadata.scenario` to select different prompt presets: - -```jsonl -{"id": "T001", "input": "...", "options": {"metadata": {"scenario": "aggregation"}}} -{"id": "T002", "input": "...", "options": {"metadata": {"scenario": "join"}}} -``` - -### Input Types - -| Type | Description | Example | -| ----------- | -------------------- | ----------------------------------------------------- | -| `string` | Simple text | `"Hello world"` | -| `Message` | Single message | `{"role": "user", "content": "..."}` | -| `[]Message` | Conversation history | `[{"role": "user", ...}, {"role": "assistant", ...}]` | - -### File Attachments - -Test inputs support file attachments (images, audio, documents) using the `file://` protocol. Files are loaded and converted to appropriate formats for the LLM. - -**Supported file types:** - -| Type | Extensions | Format | -| ------ | ---------------------------------------------------------------------- | ------------------------------ | -| Image | `.jpg`, `.jpeg`, `.png`, `.gif`, `.webp`, `.bmp` | Base64 data URL in `image_url` | -| Audio | `.wav`, `.mp3`, `.flac`, `.ogg`, `.m4a` | Base64 in `input_audio` | -| Doc | `.pdf`, `.doc`, `.docx`, `.xls`, `.xlsx`, `.txt`, `.csv`, `.json` | Base64 data URL in `file` | -| Source | `.yao`, `.ts`, `.js`, `.go`, `.py`, `.rs`, `.java`, `.sql`, `.yaml`... | Base64 data URL in `file` | - -**File path resolution:** - -- **Relative paths**: Resolved relative to the JSONL input file's directory (for file mode) or current working directory (for message mode) -- **Absolute paths**: Used as-is - -**Example with image attachment:** - -```jsonl -{ - "id": "T001", - "input": { - "role": "user", - "content": [ - { - "type": "text", - "text": "Please analyze this invoice" - }, - { - "type": "image", - "source": "file://fixtures/invoice.jpg" - } - ] - }, - "assert": { - "type": "contains", - "value": "amount" - } -} -``` - -**Example with multiple attachments:** - -```jsonl -{ - "id": "T002", - "input": { - "role": "user", - "content": [ - { - "type": "text", - "text": "Process these receipts" - }, - { - "type": "image", - "source": "file://fixtures/receipt1.png" - }, - { - "type": "image", - "source": "file://fixtures/receipt2.png" - }, - { - "type": "file", - "source": "file://fixtures/policy.pdf", - "name": "expense_policy.pdf" - } - ] - } -} -``` - -**Example with audio:** - -```jsonl -{ - "id": "T003", - "input": { - "role": "user", - "content": [ - { - "type": "text", - "text": "Transcribe this audio" - }, - { - "type": "audio", - "source": "file://fixtures/recording.wav" - } - ] - } -} -``` - -**Content part types:** - -| Type | Fields | Description | -| ----------- | --------------------------------------- | -------------------------------- | -| `text` | `text` | Text content | -| `image` | `source` (file://) or `url` | Image attachment | -| `image_url` | `image_url: {url, detail?}` | Direct image URL (OpenAI format) | -| `audio` | `source` (file://) or `data`, `format` | Audio attachment | -| `file` | `source` (file://) or `url`, `filename` | Document attachment | -| `data` | `data: {sources: [...]}` | Data source references | - -## Assertions - -Use `assert` for flexible validation. If `assert` is defined, it takes precedence over `expected`. - -### Assertion Types - -| Type | Description | Example | -| -------------- | ----------------------------- | --------------------------------------------------------- | -| `equals` | Exact match | `{"type": "equals", "value": {"key": "val"}}` | -| `contains` | Output contains value | `{"type": "contains", "value": "keyword"}` | -| `not_contains` | Output does not contain value | `{"type": "not_contains", "value": "error"}` | -| `json_path` | Extract JSON path and compare | `{"type": "json_path", "path": "$.field", "value": true}` | -| `regex` | Match regex pattern | `{"type": "regex", "value": "\\d+"}` | -| `type` | Check output type | `{"type": "type", "value": "object"}` | -| `script` | Run custom assertion script | `{"type": "script", "script": "scripts.test.Check"}` | - -### Assertion Options - -| Field | Type | Description | -| --------- | ------ | --------------------------- | -| `type` | string | Assertion type (required) | -| `value` | any | Expected value or pattern | -| `path` | string | JSON path (for `json_path`) | -| `script` | string | Script name (for `script`) | -| `message` | string | Custom failure message | -| `negate` | bool | Invert the result | - -### Examples - -**JSON path validation:** - -```jsonl -{ - "id": "T001", - "input": "What's the weather?", - "assert": { - "type": "json_path", - "path": "need_search", - "value": true - } -} -``` - -**Multiple assertions (all must pass):** - -```jsonl -{ - "id": "T002", - "input": "Hello", - "assert": [ + "checkpoints": [ { - "type": "json_path", - "path": "need_search", - "value": false + "id": "greeting", + "description": "Agent greets customer", + "assert": { + "type": "regex", + "value": "(?i)(hello|hi|help)" + } }, { - "type": "not_contains", - "value": "error" + "id": "ask_size", + "description": "Agent asks for size", + "after": [ + "greeting" + ], + "assert": { + "type": "regex", + "value": "(?i)size" + } + }, + { + "id": "confirm", + "description": "Agent confirms order", + "after": [ + "ask_size" + ], + "assert": { + "type": "regex", + "value": "(?i)confirm" + } } - ] + ], + "max_turns": 10 } ``` -**Custom script assertion:** +### Console Output (Dynamic Mode) -```jsonl -{ - "id": "T003", - "input": "Test", - "assert": { - "type": "script", - "script": "scripts.test.Validate" - } -} ``` - -Script receives `(output, input, expected)` and returns: - -```javascript -// Boolean -return true; - -// Or detailed result -return { pass: true, message: "Validation passed" }; +► [coffee-order] (dynamic, 3 checkpoints) +ℹ Dynamic test: coffee-order (max 10 turns) +ℹ Turn 1: User: I want to order coffee +ℹ Turn 1: Agent: Hello! What can I get for you? +ℹ ✓ checkpoint: greeting +ℹ Turn 2: User: A medium latte please +ℹ Turn 2: Agent: What size would you like? +ℹ ✓ checkpoint: ask_size +ℹ Turn 3: User: Medium +ℹ Turn 3: Agent: Let me confirm: Medium latte. Correct? +ℹ ✓ checkpoint: confirm + └─ PASSED (3 turns, 3 checkpoints, 8.5s) ``` -**Negated assertion:** - -```jsonl -{ - "id": "T004", - "input": "Hello", - "assert": { - "type": "contains", - "value": "error", - "negate": true - } -} -``` - -### JSON Path Notes - -- Supports dot-notation: `$.field.subfield` or `field.subfield` -- Supports array indexing: `field[0]`, `field[0].subfield`, `field[0].nested[1]` -- Supports multiple expected values (OR logic): `"value": ["a", "b"]` - passes if actual matches any -- Auto-extracts JSON from markdown code blocks (` ```json ... ``` `) -- Works with both string output and structured objects - -**Array index examples:** - -```jsonl -{"id": "T001", "assert": {"type": "json_path", "path": "wheres[0].like", "value": "%test%"}} -{"id": "T002", "assert": {"type": "json_path", "path": "wheres[0].in[0]", "value": "pending"}} -{"id": "T003", "assert": {"type": "json_path", "path": "joins[0].from", "value": "users"}} -{"id": "T004", "assert": {"type": "json_path", "path": "groups[0]", "value": "category"}} -``` - -**Multiple expected values (OR logic):** - -```jsonl -{ - "id": "T005", - "assert": { - "type": "json_path", - "path": "error", - "value": [ - "missing_schema", - "missing_query" - ] - } -} -``` - -This passes if `error` equals either `"missing_schema"` or `"missing_query"`. - ## Output Formats Determined by `-o` file extension: @@ -523,18 +991,6 @@ Determined by `-o` file extension: | `.md` | Markdown | Human-readable | | `.html` | HTML | Interactive web report | -### Default Output Path - -When `-o` is not specified in file mode: - -``` -{input_directory}/output-{timestamp}.jsonl -``` - -Example: `tests/output-20241217100000.jsonl` - -In direct message mode without `-o`, output is printed to stdout. - ## Stability Analysis Run each test multiple times to measure consistency: @@ -543,15 +999,6 @@ Run each test multiple times to measure consistency: yao agent test -i tests/inputs.jsonl --runs 5 -o stability.json ``` -Output includes: - -- Pass rate per test -- Stability classification (stable, mostly_stable, unstable, highly_unstable) -- Average/min/max duration -- Standard deviation - -### Stability Classification - | Pass Rate | Classification | | --------- | --------------- | | 100% | Stable | @@ -559,49 +1006,14 @@ Output includes: | 50-79% | Unstable | | < 50% | Highly Unstable | -## Test Environment - -The test framework creates a context with configurable environment: - -| Setting | Flag | Default | -| ---------- | ---- | ----------- | -| User ID | `-u` | `test-user` | -| Team ID | `-t` | `test-team` | -| Locale | - | `en-us` | -| ClientType | - | `test` | -| ClientIP | - | `127.0.0.1` | - -Priority: Command line flags > Test case fields > Defaults - -## Custom Reporter Agent - -Use `-r` to specify a custom agent for report generation: - -```bash -yao agent test -i tests/inputs.jsonl -r report.beautiful -o report.html -``` - -The reporter agent receives: - -```json -{ - "report": { "summary": {...}, "results": [...] }, - "format": "html", - "options": { "verbose": true } -} -``` - ## CI Integration ```bash # Exit code: 0 = all passed, 1 = failures -yao agent test -i tests/inputs.jsonl -o results.jsonl --fail-fast +yao agent test -i tests/inputs.jsonl --fail-fast -# Parse JSONL results -cat results.jsonl | jq 'select(.type == "summary")' - -# Run script tests -yao agent test -i scripts.expense.setup --fail-fast +# Run with parallel execution +yao agent test -i tests/inputs.jsonl --parallel 4 ``` ### GitHub Actions Example @@ -609,95 +1021,226 @@ yao agent test -i scripts.expense.setup --fail-fast ```yaml - name: Run Agent Tests run: | - yao agent test -i assistants/keyword/tests/inputs.jsonl \ + yao agent test -i assistants/expense/tests/inputs.jsonl \ -u ci-user -t ci-team \ --runs 3 \ -o report.json +- name: Run Dynamic Tests + run: | + yao agent test -i assistants/expense/tests/dynamic.jsonl \ + --simulator tests.simulator-agent \ + -v + - name: Run Script Tests run: | yao agent test -i scripts.expense.setup -v - yao agent test -i scripts.expense.tools -v - -- name: Run Script Tests with Custom Context - run: | - yao agent test -i scripts.expense.setup \ - --ctx tests/context.json \ - --run "TestSystem.*" \ - -v - -- name: Check Stability - run: | - jq -e '.results | all(.pass_rate >= 80)' report.json ``` -## Examples +## Format Rules Reference -### Agent Tests +| Context | Format | Example | +| ---------------------- | ------------------------ | ----------------------------------------- | +| `-i agents:xxx` (CLI) | Colon prefix | `agents:tests.generator` | +| `-i scripts:xxx` (CLI) | Colon prefix | `scripts:tests.gen.Generate` | +| `-i scripts.xxx` (CLI) | Dot prefix (test mode) | `scripts.expense.setup` | +| JSONL assertion `use` | Prefix required | `"use": "agents:tests.validator"` | +| JSONL `simulator.use` | No prefix (agent only) | `"use": "tests.simulator-agent"` | +| `--simulator` flag | No prefix (agent only) | `--simulator tests.simulator-agent` | +| `t.assert.Agent()` | No prefix (method-bound) | `t.assert.Agent(resp, "tests.validator")` | +| JSONL `before/after` | No prefix (in src/) | `"before": "env_test.Before"` | +| `--before/--after` | No prefix (in src/) | `--before env_test.BeforeAll` | -```bash -# Quick development test (auto-detect agent) -cd assistants/keyword -yao agent test -i "Extract keywords: AI and ML" +**Script input modes**: -# Quick development test (specify agent) -yao agent test -i "Hello" -n workers.system.keyword +- `scripts.xxx` (dot) - Run script tests (`*_test.ts` functions) +- `scripts:xxx` (colon) - Generate test cases from a script -# Full test suite with HTML report -yao agent test -i tests/inputs.jsonl -o report.html -v +## Built-in Test Agents -# Override connector -yao agent test -i tests/inputs.jsonl -c openai.gpt4 +The framework provides three specialized agents for testing: -# Stability analysis -yao agent test -i tests/inputs.jsonl --runs 10 -o stability.json +### Generator Agent (`tests.generator-agent`) -# Parallel execution with timeout -yao agent test -i tests/inputs.jsonl --parallel 4 --timeout 2m +Generates test cases based on target agent description. -# Custom test environment -yao agent test -i tests/inputs.jsonl -u admin -t prod-team +**package.yao**: -# Custom reporter agent -yao agent test -i tests/inputs.jsonl -r report.beautiful -o custom-report.md - -# Full example with all options -yao agent test -i tests/inputs.jsonl \ - -n keyword.agent \ - -c deepseek.v3 \ - -u test-user \ - -t test-team \ - --runs 3 \ - --timeout 10m \ - --parallel 4 \ - -r report.html \ - -o report.html +```json +{ + "name": "Test Case Generator", + "connector": "gpt-4o", + "description": "Generates test cases for agent testing", + "options": { "temperature": 0.7 }, + "automated": true +} ``` -### Script Tests +**prompts.yml**: + +```yaml +- role: system + content: | + You are a test case generator. Generate test cases based on the target agent. + + ## Input Format + - `target_agent`: Agent info (id, description, tools) + - `count`: Number of test cases (default: 5) + - `focus`: Focus area (e.g., "edge-cases", "happy-path") + + ## Output Format + JSON array of test cases: + [ + { + "id": "test-id", + "input": "User message", + "assert": [{"type": "contains", "value": "expected"}] + } + ] +``` + +**Usage**: ```bash -# Run all tests in a script module -yao agent test -i scripts.expense.setup -v +yao agent test -i "agents:tests.generator-agent?count=10" -n assistants.expense +``` -# Run specific tests with regex filter -yao agent test -i scripts.expense.setup --run "TestSystemReady" +### Validator Agent (`tests.validator-agent`) -# Run tests matching a pattern -yao agent test -i scripts.expense.setup --run "TestSystem.*" -v +Validates agent responses for agent-driven assertions. -# Run with custom context (authorization, metadata, etc.) -yao agent test -i scripts.expense.setup --ctx tests/context.json -v +**package.yao**: -# Run with specific user/team -yao agent test -i scripts.expense.setup -u admin -t ops-team -v +```json +{ + "name": "Response Validator", + "connector": "gpt-4o", + "description": "Validates responses against criteria", + "options": { "temperature": 0 }, + "automated": true +} +``` -# Combine options -yao agent test -i scripts.expense.setup \ - --ctx tests/context.json \ - --run "TestSystem.*" \ - --timeout 30s \ - -v +**prompts.yml**: + +```yaml +- role: system + content: | + You are a response validator. Evaluate whether the response meets the criteria. + + ## Input Format + - `output`: The response to validate + - `criteria`: The validation rules + - `input`: Original input (optional) + + ## Output Format + JSON object (no markdown): + {"passed": true/false, "reason": "explanation"} + + ## Examples + Input: {"output": "Paris is the capital", "criteria": "factually accurate"} + Output: {"passed": true, "reason": "Statement is correct"} +``` + +**Usage in JSONL**: + +```jsonl +{ + "id": "T001", + "input": "Hello", + "assert": { + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "Response should be friendly" + } +} +``` + +**Usage in script tests**: + +```typescript +t.assert.Agent(response, "tests.validator-agent", { + criteria: "Response should be helpful", +}); +``` + +### Simulator Agent (`tests.simulator-agent`) + +Simulates user behavior for dynamic mode testing. + +**package.yao**: + +```json +{ + "name": "User Simulator", + "connector": "gpt-4o", + "description": "Simulates user behavior for dynamic testing", + "options": { "temperature": 0.7 }, + "automated": true +} +``` + +**prompts.yml**: + +```yaml +- role: system + content: | + You are a user simulator. Generate realistic user messages based on persona and goal. + + ## Input Format + - `persona`: User description (e.g., "New employee") + - `goal`: What user wants to achieve + - `conversation`: Previous messages + - `turn_number`: Current turn + - `max_turns`: Maximum turns + + ## Output Format + JSON object: + { + "message": "User response", + "goal_achieved": false, + "reasoning": "Strategy explanation" + } + + ## Guidelines + 1. Stay in character + 2. Work toward the goal + 3. Be realistic (include natural variations) + 4. Set goal_achieved: true when done +``` + +**Usage in JSONL**: + +```jsonl +{ + "id": "dynamic-test", + "input": "I need help", + "simulator": { + "use": "tests.simulator-agent", + "options": { + "metadata": { + "persona": "New employee", + "goal": "Submit expense report" + } + } + }, + "checkpoints": [ + { + "id": "greeting", + "assert": { + "type": "regex", + "value": "(?i)hello" + } + } + ], + "max_turns": 10 +} +``` + +**Usage via CLI**: + +```bash +yao agent test -i tests/dynamic.jsonl --simulator tests.simulator-agent ``` ## Exit Codes diff --git a/agent/test/assert.go b/agent/test/assert.go index fc3c97e7..505f0e62 100644 --- a/agent/test/assert.go +++ b/agent/test/assert.go @@ -9,6 +9,9 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/yaoapp/gou/process" + goutext "github.com/yaoapp/gou/text" + "github.com/yaoapp/yao/agent/assistant" + "github.com/yaoapp/yao/agent/context" ) // Asserter handles test assertions @@ -115,6 +118,9 @@ func (a *Asserter) mapToAssertion(m map[string]interface{}) *Assertion { if s, ok := m["script"].(string); ok { assertion.Script = s } + if u, ok := m["use"].(string); ok { + assertion.Use = u + } if msg, ok := m["message"].(string); ok { assertion.Message = msg } @@ -122,6 +128,17 @@ func (a *Asserter) mapToAssertion(m map[string]interface{}) *Assertion { assertion.Negate = n } + // Parse options for agent assertions + if opts, ok := m["options"].(map[string]interface{}); ok { + assertion.Options = &AssertionOptions{} + if c, ok := opts["connector"].(string); ok { + assertion.Options.Connector = c + } + if meta, ok := opts["metadata"].(map[string]interface{}); ok { + assertion.Options.Metadata = meta + } + } + return assertion } @@ -147,6 +164,8 @@ func (a *Asserter) evaluateAssertion(assertion *Assertion, output, input interfa result = a.assertType(assertion, output) case "script": result = a.assertScript(assertion, output, input) + case "agent": + result = a.assertAgent(assertion, output, input) default: result.Passed = false result.Message = fmt.Sprintf("unknown assertion type: %s", assertion.Type) @@ -229,17 +248,14 @@ func (a *Asserter) assertJSONPath(assertion *Assertion, output interface{}) *Ass var jsonData interface{} switch v := output.(type) { case string: - // Try to parse as JSON - if err := jsoniter.Unmarshal([]byte(v), &jsonData); err != nil { - // Try to extract JSON from markdown code blocks - extracted := extractJSONFromText(v) - if extracted != nil { - jsonData = extracted - } else { - result.Passed = false - result.Message = fmt.Sprintf("output is not valid JSON: %s", err.Error()) - return result - } + // Use gou/text to extract JSON (handles markdown, auto-repair, etc.) + extracted := goutext.ExtractJSON(v) + if extracted != nil { + jsonData = extracted + } else { + result.Passed = false + result.Message = fmt.Sprintf("output is not valid JSON: %s", v) + return result } case map[string]interface{}, []interface{}: jsonData = v @@ -478,6 +494,158 @@ func (a *Asserter) getType(v interface{}) string { } } +// assertAgent uses an agent to validate the output +func (a *Asserter) assertAgent(assertion *Assertion, output, input interface{}) *AssertionResult { + result := &AssertionResult{ + Assertion: assertion, + Actual: output, + } + + // Parse use field: "agents:tests.validator-agent" + if !strings.HasPrefix(assertion.Use, "agents:") { + result.Passed = false + result.Message = "agent assertion requires 'use' field with 'agents:' prefix" + return result + } + + agentID := strings.TrimPrefix(assertion.Use, "agents:") + + // Get assistant + ast, err := assistant.Get(agentID) + if err != nil { + result.Passed = false + result.Message = fmt.Sprintf("failed to get validator agent: %s", err.Error()) + return result + } + + // Build validation request + validationInput := map[string]interface{}{ + "output": output, + "input": input, + } + + // Add criteria from Value field + if assertion.Value != nil { + validationInput["criteria"] = assertion.Value + } + + // Add metadata from options + if assertion.Options != nil && assertion.Options.Metadata != nil { + for k, v := range assertion.Options.Metadata { + validationInput[k] = v + } + } + + // Build context options - skip history and trace for validator + opts := &context.Options{ + Skip: &context.Skip{ + History: true, + Trace: true, + Output: true, + }, + Metadata: map[string]interface{}{ + "test_mode": "validator", + }, + } + if assertion.Options != nil && assertion.Options.Connector != "" { + opts.Connector = assertion.Options.Connector + } + + // Create context and call agent + env := NewEnvironment("", "") + ctx := NewTestContext("validator", agentID, env) + defer ctx.Release() + + // Convert validation input to JSON string for the message + inputJSON, err := json.Marshal(validationInput) + if err != nil { + result.Passed = false + result.Message = fmt.Sprintf("failed to marshal validation input: %s", err.Error()) + return result + } + + messages := []context.Message{{ + Role: context.RoleUser, + Content: string(inputJSON), + }} + + response, err := ast.Stream(ctx, messages, opts) + if err != nil { + result.Passed = false + result.Message = fmt.Sprintf("validator agent error: %s", err.Error()) + return result + } + + // Parse response + return a.parseValidatorResponse(response, result) +} + +// parseValidatorResponse parses the validator agent's response +func (a *Asserter) parseValidatorResponse(response *context.Response, result *AssertionResult) *AssertionResult { + output := extractValidatorOutput(response) + + // Expected format: { "passed": bool, "reason": string, "score": float, "suggestions": [] } + if outputMap, ok := output.(map[string]interface{}); ok { + if passed, ok := outputMap["passed"].(bool); ok { + result.Passed = passed + } else { + result.Passed = false + result.Message = "validator response missing 'passed' field" + return result + } + if reason, ok := outputMap["reason"].(string); ok { + result.Message = reason + } + // Store score and suggestions in expected field for reference + result.Expected = outputMap + } else { + result.Passed = false + result.Message = "validator agent returned invalid response format" + } + + return result +} + +// extractValidatorOutput extracts the output from a validator response +func extractValidatorOutput(response *context.Response) interface{} { + if response == nil || response.Completion == nil { + return nil + } + + // Get content from completion + content := response.Completion.Content + if content == nil { + return nil + } + + // Try to get text content + var text string + switch v := content.(type) { + case string: + text = v + default: + // Try to marshal and use as-is + data, err := json.Marshal(content) + if err != nil { + return nil + } + text = string(data) + } + + if text == "" { + return nil + } + + // Use gou/text to extract JSON (handles markdown code blocks, auto-repair, etc.) + result := goutext.ExtractJSON(text) + if result != nil { + return result + } + + // Return raw text if extraction fails + return text +} + // assertScript runs a custom assertion script func (a *Asserter) assertScript(assertion *Assertion, output, input interface{}) *AssertionResult { result := &AssertionResult{ @@ -559,30 +727,3 @@ func (a *Asserter) toString(v interface{}) string { return string(b) } } - -// extractJSONFromText tries to extract JSON from text (e.g., markdown code blocks) -func extractJSONFromText(text string) interface{} { - // Try to find JSON in code blocks - patterns := []string{ - "```json\n", - "```\n", - } - - for _, start := range patterns { - if idx := strings.Index(text, start); idx >= 0 { - text = text[idx+len(start):] - if endIdx := strings.Index(text, "```"); endIdx >= 0 { - text = text[:endIdx] - } - break - } - } - - // Try to parse - var result interface{} - if err := jsoniter.Unmarshal([]byte(strings.TrimSpace(text)), &result); err == nil { - return result - } - - return nil -} diff --git a/agent/test/assert_agent_test.go b/agent/test/assert_agent_test.go new file mode 100644 index 00000000..20dbc4c9 --- /dev/null +++ b/agent/test/assert_agent_test.go @@ -0,0 +1,275 @@ +package test_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v8 "github.com/yaoapp/gou/runtime/v8" + "github.com/yaoapp/yao/agent" + agenttest "github.com/yaoapp/yao/agent/test" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" + "rogchap.com/v8go" +) + +func TestAsserter_AgentAssertion(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent (includes assistants) + err := agent.Load(config.Conf) + if err != nil { + t.Fatalf("Failed to load agent: %v", err) + } + + asserter := agenttest.NewAsserter() + + tests := []struct { + name string + tc *agenttest.Case + output interface{} + expected bool + skipMsg string + }{ + { + name: "agent assertion - pass", + tc: &agenttest.Case{ + Assert: map[string]interface{}{ + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "Response should be a greeting", + }, + }, + output: "Hello! How can I help you today?", + expected: true, + }, + { + name: "agent assertion - fail", + tc: &agenttest.Case{ + Assert: map[string]interface{}{ + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "Response should provide a detailed technical answer", + }, + }, + output: "I don't know.", + expected: false, + }, + { + name: "agent assertion - missing prefix", + tc: &agenttest.Case{ + Assert: map[string]interface{}{ + "type": "agent", + "use": "tests.validator-agent", // Missing agents: prefix + "value": "Should pass", + }, + }, + output: "Hello", + expected: false, // Should fail due to missing prefix + }, + { + name: "agent assertion - with metadata", + tc: &agenttest.Case{ + Assert: map[string]interface{}{ + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "Response is helpful", + "options": map[string]interface{}{ + "metadata": map[string]interface{}{ + "context": "customer support", + }, + }, + }, + }, + output: "I'd be happy to help you with your order. Let me look that up for you.", + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.skipMsg != "" { + t.Skip(tt.skipMsg) + } + + passed, errMsg := asserter.Validate(tt.tc, tt.output) + if passed != tt.expected { + t.Errorf("Expected passed=%v, got passed=%v, error: %s", tt.expected, passed, errMsg) + } + }) + } +} + +func TestAsserter_AgentAssertion_InvalidAgent(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent (includes assistants) + err := agent.Load(config.Conf) + if err != nil { + t.Fatalf("Failed to load agent: %v", err) + } + + asserter := agenttest.NewAsserter() + + tc := &agenttest.Case{ + Assert: map[string]interface{}{ + "type": "agent", + "use": "agents:nonexistent.agent", + "value": "Should fail", + }, + } + + passed, errMsg := asserter.Validate(tc, "Hello") + assert.False(t, passed, "Should fail for nonexistent agent") + assert.Contains(t, errMsg, "failed to get validator agent", "Error should mention agent loading failure") +} + +func TestAsserter_MapToAssertion_WithUseAndOptions(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent (includes assistants) + err := agent.Load(config.Conf) + if err != nil { + t.Fatalf("Failed to load agent: %v", err) + } + + asserter := agenttest.NewAsserter() + + // Test that mapToAssertion correctly parses use and options fields + tc := &agenttest.Case{ + Assert: map[string]interface{}{ + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "criteria here", + "options": map[string]interface{}{ + "connector": "gpt-4o", + "metadata": map[string]interface{}{ + "key": "value", + }, + }, + }, + } + + // Validate triggers parseAssertions internally + // We just verify it doesn't panic and processes correctly + _, _ = asserter.Validate(tc, "test output") + // If we get here without panic, the parsing worked +} + +// TestTestingT_AssertAgent tests the JSAPI t.assert.Agent() method +func TestTestingT_AssertAgent(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent (includes assistants) + err := agent.Load(config.Conf) + if err != nil { + t.Fatalf("Failed to load agent: %v", err) + } + + tests := []struct { + name string + script string + shouldFail bool + }{ + { + name: "JSAPI agent assertion - pass", + script: ` + function test(t) { + var response = "Hello! How can I help you today?"; + t.assert.Agent(response, "tests.validator-agent", { + criteria: "Response should be a friendly greeting" + }); + } + test(__test_t); + `, + shouldFail: false, + }, + { + name: "JSAPI agent assertion - JSON response", + script: ` + function test(t) { + var response = { + status: "success", + data: { user: "john", email: "john@example.com" }, + message: "User created successfully" + }; + t.assert.Agent(response, "tests.validator-agent", { + criteria: "Response should be a successful API response with user data" + }); + } + test(__test_t); + `, + shouldFail: false, + }, + { + name: "JSAPI agent assertion - with metadata", + script: ` + function test(t) { + var response = "I'd be happy to help you with your order."; + t.assert.Agent(response, "tests.validator-agent", { + criteria: "Response is helpful and professional", + metadata: { context: "customer support" } + }); + } + test(__test_t); + `, + shouldFail: false, + }, + { + name: "JSAPI agent assertion - fail case", + script: ` + function test(t) { + var response = "I don't know."; + t.assert.Agent(response, "tests.validator-agent", { + criteria: "Response should provide a detailed technical explanation" + }); + } + test(__test_t); + `, + shouldFail: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create TestingT + testingT := agenttest.NewTestingT(tt.name) + + // Create V8 isolate and context + iso := v8go.NewIsolate() + defer iso.Dispose() + + v8ctx := v8go.NewContext(iso) + defer v8ctx.Close() + + // Create testing object + testObj, err := agenttest.NewTestingTObject(v8ctx, testingT) + if err != nil { + t.Fatalf("Failed to create testing object: %v", err) + } + + // Set testing object as global + global := v8ctx.Global() + global.Set("__test_t", testObj) + + // Run the test script + _, err = v8ctx.RunScript(tt.script, "test.js") + + // Check results + if tt.shouldFail { + assert.True(t, testingT.Failed(), "Test should have failed") + } else { + if err != nil { + t.Errorf("Script execution error: %v", err) + } + assert.False(t, testingT.Failed(), "Test should have passed, errors: %v", testingT.Errors()) + } + }) + } +} + +// Ensure v8 is used (for script loading) +var _ = v8.Scripts diff --git a/agent/test/dynamic_integration_test.go b/agent/test/dynamic_integration_test.go new file mode 100644 index 00000000..b37c7554 --- /dev/null +++ b/agent/test/dynamic_integration_test.go @@ -0,0 +1,250 @@ +package test_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/yaoapp/yao/agent" + agenttest "github.com/yaoapp/yao/agent/test" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +// TestDynamicRunner_CoffeeOrder tests a complete dynamic mode flow: +// Simulator acts as a customer ordering coffee, agent handles the order +func TestDynamicRunner_CoffeeOrder(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Create a temporary JSONL file with a dynamic test case + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "dynamic-inputs.jsonl") + + // Dynamic test case: customer ordering coffee (JSONL must be single line) + testCase := `{"id": "coffee-order-flow", "name": "Complete Coffee Order", "input": "Hi, I would like to order a coffee please", "simulator": {"use": "tests.simulator-agent", "options": {"metadata": {"persona": "A customer who wants to order a medium latte with oat milk", "goal": "Successfully complete a coffee order"}}}, "checkpoints": [{"id": "greeting", "description": "Agent greets and asks for order", "assert": {"type": "regex", "value": "(?i)(order|like|help)"}}, {"id": "ask_size", "description": "Agent asks for size", "after": ["greeting"], "assert": {"type": "regex", "value": "(?i)size"}}, {"id": "confirm_order", "description": "Agent confirms the order", "after": ["ask_size"], "assert": {"type": "regex", "value": "(?i)confirm"}}], "max_turns": 8}` + + err = os.WriteFile(inputFile, []byte(testCase), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run dynamic test + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.dynamic-test-agent", + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + require.NotNil(t, report.Summary, "Summary should not be nil") + + // Log results + t.Logf("Total: %d, Passed: %d, Failed: %d", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) + + // Check results + if len(report.Results) > 0 { + result := report.Results[0] + t.Logf("Test [%s] Status: %s", result.ID, result.Status) + + // Check metadata for dynamic mode info + if result.Metadata != nil { + if mode, ok := result.Metadata["mode"].(string); ok { + assert.Equal(t, "dynamic", mode, "Should be dynamic mode") + } + if turns, ok := result.Metadata["total_turns"].(int); ok { + t.Logf("Total turns: %d", turns) + } + } + + if result.Error != "" { + t.Logf("Error: %s", result.Error) + } + } +} + +// TestDynamicRunner_WithInitialInput tests dynamic mode with initial user input +func TestDynamicRunner_WithInitialInput(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Create a test case with initial input + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "dynamic-inputs.jsonl") + + // Start with user's first message (JSONL must be single line) + testCase := `{"id": "coffee-with-initial", "name": "Coffee Order with Initial Message", "input": "Hi, I want to order a coffee", "simulator": {"use": "tests.simulator-agent", "options": {"metadata": {"persona": "Customer ordering a large cappuccino", "goal": "Complete the coffee order"}}}, "checkpoints": [{"id": "acknowledge", "description": "Agent acknowledges the order request", "assert": {"type": "regex", "value": "(?i)(coffee|order|help)"}}, {"id": "ask_details", "description": "Agent asks for more details", "after": ["acknowledge"], "assert": {"type": "regex", "value": "(?i)(size|type|what)"}}], "max_turns": 5}` + + err = os.WriteFile(inputFile, []byte(testCase), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run test + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.dynamic-test-agent", + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + t.Logf("Total: %d, Passed: %d, Failed: %d", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) +} + +// TestDynamicRunner_OptionalCheckpoint tests optional checkpoint behavior +func TestDynamicRunner_OptionalCheckpoint(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "dynamic-inputs.jsonl") + + // Test with one required and one optional checkpoint (JSONL must be single line) + testCase := `{"id": "optional-checkpoint-test", "name": "Test with Optional Checkpoint", "input": "Hello", "simulator": {"use": "tests.simulator-agent", "options": {"metadata": {"persona": "Simple customer", "goal": "Get a greeting response"}}}, "checkpoints": [{"id": "greeting_response", "description": "Agent responds with greeting", "assert": {"type": "regex", "value": "(?i)(hello|hi|help)"}}, {"id": "special_offer", "description": "Agent mentions special offer (optional)", "required": false, "assert": {"type": "contains", "value": "special offer"}}], "max_turns": 3}` + + err = os.WriteFile(inputFile, []byte(testCase), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run test + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.dynamic-test-agent", + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + // Test should pass even if optional checkpoint is not reached + t.Logf("Total: %d, Passed: %d, Failed: %d", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) + + // If the required checkpoint is reached, the test should pass + if len(report.Results) > 0 && report.Results[0].Metadata != nil { + if checkpoints, ok := report.Results[0].Metadata["checkpoints"].(map[string]*agenttest.CheckpointResult); ok { + for id, cp := range checkpoints { + t.Logf("Checkpoint [%s]: reached=%v, required=%v", id, cp.Reached, cp.Required) + } + } + } +} + +// TestDynamicRunner_MaxTurnsExceeded tests behavior when max turns is exceeded +func TestDynamicRunner_MaxTurnsExceeded(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "dynamic-inputs.jsonl") + + // Test case with impossible checkpoint and low max_turns (JSONL must be single line) + testCase := `{"id": "max-turns-test", "name": "Test Max Turns Exceeded", "input": "Hello", "simulator": {"use": "tests.simulator-agent", "options": {"metadata": {"persona": "Persistent customer", "goal": "Keep talking"}}}, "checkpoints": [{"id": "impossible", "description": "This checkpoint will never be reached", "assert": {"type": "contains", "value": "IMPOSSIBLE_STRING_NEVER_APPEARS_12345"}}], "max_turns": 2}` + + err = os.WriteFile(inputFile, []byte(testCase), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run test + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.dynamic-test-agent", + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + // Test should fail due to max turns exceeded + assert.Equal(t, 1, report.Summary.Failed, "Test should fail") + + if len(report.Results) > 0 { + result := report.Results[0] + assert.Equal(t, agenttest.StatusFailed, result.Status, "Status should be failed") + assert.Contains(t, result.Error, "max turns", "Error should mention max turns") + t.Logf("Error (expected): %s", result.Error) + } +} + +// TestDynamicRunner_CheckpointOrdering tests that checkpoint ordering is enforced +func TestDynamicRunner_CheckpointOrderingEnforced(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "dynamic-inputs.jsonl") + + // Test case with ordered checkpoints (JSONL must be single line) + testCase := `{"id": "ordered-checkpoints", "name": "Test Checkpoint Ordering", "input": "I want to order coffee", "simulator": {"use": "tests.simulator-agent", "options": {"metadata": {"persona": "Customer ordering step by step", "goal": "Complete coffee order following the flow"}}}, "checkpoints": [{"id": "step1_greeting", "description": "Agent greets", "assert": {"type": "regex", "value": "(?i)(hello|hi|help|order)"}}, {"id": "step2_size", "description": "Agent asks about size", "after": ["step1_greeting"], "assert": {"type": "regex", "value": "(?i)size"}}, {"id": "step3_confirm", "description": "Agent confirms", "after": ["step2_size"], "assert": {"type": "regex", "value": "(?i)confirm"}}], "max_turns": 10}` + + err = os.WriteFile(inputFile, []byte(testCase), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run test + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.dynamic-test-agent", + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + t.Logf("Total: %d, Passed: %d, Failed: %d", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) + + // Log checkpoint order + if len(report.Results) > 0 && report.Results[0].Metadata != nil { + if checkpoints, ok := report.Results[0].Metadata["checkpoints"].(map[string]*agenttest.CheckpointResult); ok { + for id, cp := range checkpoints { + t.Logf("Checkpoint [%s]: reached=%v, at_turn=%d", id, cp.Reached, cp.ReachedAtTurn) + } + } + } +} diff --git a/agent/test/dynamic_runner.go b/agent/test/dynamic_runner.go new file mode 100644 index 00000000..2b6020df --- /dev/null +++ b/agent/test/dynamic_runner.go @@ -0,0 +1,413 @@ +package test + +import ( + "fmt" + "time" + + jsoniter "github.com/json-iterator/go" + goutext "github.com/yaoapp/gou/text" + "github.com/yaoapp/yao/agent/assistant" + "github.com/yaoapp/yao/agent/context" +) + +// DynamicRunner handles dynamic (simulator-driven) test execution +type DynamicRunner struct { + opts *Options + output *OutputWriter + asserter *Asserter +} + +// NewDynamicRunner creates a new dynamic runner +func NewDynamicRunner(opts *Options) *DynamicRunner { + return &DynamicRunner{ + opts: opts, + output: NewOutputWriter(opts.Verbose), + asserter: NewAsserter(), + } +} + +// RunDynamic executes a dynamic test case +func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID string) *DynamicResult { + startTime := time.Now() + + result := &DynamicResult{ + ID: tc.ID, + Turns: make([]*TurnResult, 0), + Checkpoints: make(map[string]*CheckpointResult), + } + + // Initialize checkpoints + for _, cp := range tc.Checkpoints { + result.Checkpoints[cp.ID] = &CheckpointResult{ + ID: cp.ID, + Reached: false, + Required: cp.IsRequired(), + } + } + + // Get simulator agent + simAST, err := assistant.Get(tc.Simulator.Use) + if err != nil { + result.Status = StatusError + result.Error = fmt.Sprintf("failed to get simulator agent: %s", err.Error()) + result.DurationMs = time.Since(startTime).Milliseconds() + return result + } + + // Get configuration + maxTurns := tc.GetMaxTurns() + timeout := tc.GetTimeout(r.opts.Timeout) + + // Build simulator metadata + simMetadata := make(map[string]interface{}) + if tc.Simulator.Options != nil && tc.Simulator.Options.Metadata != nil { + for k, v := range tc.Simulator.Options.Metadata { + simMetadata[k] = v + } + } + + // Conversation history + messages := make([]context.Message, 0) + + // Get initial input if provided + initialMessages, err := tc.GetMessages() + if err == nil && len(initialMessages) > 0 { + messages = append(messages, initialMessages...) + } + + // Output dynamic test start + if r.opts.Verbose { + r.output.Info("Dynamic test: %s (max %d turns)", tc.ID, maxTurns) + } + + // Conversation loop + for turn := 1; turn <= maxTurns; turn++ { + turnStart := time.Now() + turnResult := &TurnResult{Turn: turn} + + // Check timeout + if time.Since(startTime) > timeout { + result.Status = StatusTimeout + result.Error = fmt.Sprintf("timeout after %s", timeout) + result.DurationMs = time.Since(startTime).Milliseconds() + result.TotalTurns = turn - 1 + return result + } + + // For turns after the first, get input from simulator + if turn > 1 || len(messages) == 0 { + simInput := r.buildSimulatorInput(tc, messages, result, turn, maxTurns, simMetadata) + simOutput, err := r.callSimulator(simAST, tc, simInput) + if err != nil { + turnResult.Error = fmt.Sprintf("simulator error: %s", err.Error()) + result.Turns = append(result.Turns, turnResult) + result.Status = StatusError + result.Error = turnResult.Error + result.DurationMs = time.Since(startTime).Milliseconds() + result.TotalTurns = turn + return result + } + + // Check if goal achieved + if simOutput.GoalAchieved { + if r.opts.Verbose { + r.output.Info(" Turn %d: Simulator signaled goal achieved", turn) + } + + // Check if all required checkpoints reached + if r.allRequiredCheckpointsReached(result) { + result.Status = StatusPassed + } else { + result.Status = StatusFailed + result.Error = "simulator signaled goal achieved but not all required checkpoints reached" + } + result.DurationMs = time.Since(startTime).Milliseconds() + result.TotalTurns = turn - 1 + return result + } + + // Add user message + userMessage := context.Message{ + Role: context.RoleUser, + Content: simOutput.Message, + } + messages = append(messages, userMessage) + turnResult.Input = simOutput.Message + + if r.opts.Verbose { + r.output.Info(" Turn %d: User: %s", turn, truncateOutput(simOutput.Message, 50)) + } + } else { + // Use initial input for first turn + if len(messages) > 0 { + lastMsg := messages[len(messages)-1] + turnResult.Input = lastMsg.Content + if r.opts.Verbose { + r.output.Info(" Turn %d: User: %s", turn, truncateOutput(lastMsg.Content, 50)) + } + } + } + + // Call target agent + ctx := NewTestContextFromOptions( + fmt.Sprintf("dynamic-%s-%d", tc.ID, turn), + agentID, + r.opts, + tc, + ) + + opts := buildContextOptions(tc, r.opts) + response, err := ast.Stream(ctx, messages, opts) + ctx.Release() + + if err != nil { + turnResult.Error = err.Error() + turnResult.DurationMs = time.Since(turnStart).Milliseconds() + result.Turns = append(result.Turns, turnResult) + result.Status = StatusError + result.Error = fmt.Sprintf("agent error at turn %d: %s", turn, err.Error()) + result.DurationMs = time.Since(startTime).Milliseconds() + result.TotalTurns = turn + return result + } + + // Extract output + output := extractOutput(response) + turnResult.Output = output + turnResult.DurationMs = time.Since(turnStart).Milliseconds() + + if r.opts.Verbose { + r.output.Info(" Turn %d: Agent: %s", turn, truncateOutput(output, 50)) + } + + // Add assistant response to messages + messages = append(messages, context.Message{ + Role: context.RoleAssistant, + Content: output, + }) + + // Check checkpoints against this response + reachedIDs := r.checkCheckpoints(tc.Checkpoints, output, result) + turnResult.CheckpointsReached = reachedIDs + + if r.opts.Verbose && len(reachedIDs) > 0 { + for _, id := range reachedIDs { + r.output.Info(" ✓ checkpoint: %s", id) + } + } + + result.Turns = append(result.Turns, turnResult) + + // Check if all required checkpoints reached + if r.allRequiredCheckpointsReached(result) { + result.Status = StatusPassed + result.DurationMs = time.Since(startTime).Milliseconds() + result.TotalTurns = turn + return result + } + } + + // Max turns exceeded + result.Status = StatusFailed + result.Error = fmt.Sprintf("max turns (%d) exceeded without reaching all checkpoints", maxTurns) + result.DurationMs = time.Since(startTime).Milliseconds() + result.TotalTurns = maxTurns + return result +} + +// buildSimulatorInput builds the input for the simulator agent +func (r *DynamicRunner) buildSimulatorInput( + tc *Case, + messages []context.Message, + result *DynamicResult, + turn, maxTurns int, + metadata map[string]interface{}, +) *SimulatorInput { + input := &SimulatorInput{ + Conversation: messages, + TurnNumber: turn, + MaxTurns: maxTurns, + } + + // Extract persona and goal from metadata + if persona, ok := metadata["persona"].(string); ok { + input.Persona = persona + } + if goal, ok := metadata["goal"].(string); ok { + input.Goal = goal + } + + // Build checkpoint lists + input.CheckpointsReached = make([]string, 0) + input.CheckpointsPending = make([]string, 0) + for id, cp := range result.Checkpoints { + if cp.Reached { + input.CheckpointsReached = append(input.CheckpointsReached, id) + } else { + input.CheckpointsPending = append(input.CheckpointsPending, id) + } + } + + // Store extra metadata + input.Extra = make(map[string]interface{}) + for k, v := range metadata { + if k != "persona" && k != "goal" { + input.Extra[k] = v + } + } + + return input +} + +// callSimulator calls the simulator agent and parses the response +func (r *DynamicRunner) callSimulator(simAST *assistant.Assistant, tc *Case, input *SimulatorInput) (*SimulatorOutput, error) { + // Create context + env := NewEnvironment("", "") + ctx := NewTestContext("simulator", tc.Simulator.Use, env) + defer ctx.Release() + + // Build options - skip history and trace + opts := &context.Options{ + Skip: &context.Skip{ + History: true, + Trace: true, + Output: true, + }, + Metadata: map[string]interface{}{ + "test_mode": "simulator", + }, + } + + // Override connector if specified + if tc.Simulator.Options != nil && tc.Simulator.Options.Connector != "" { + opts.Connector = tc.Simulator.Options.Connector + } + + // Build message + inputJSON, err := jsoniter.Marshal(input) + if err != nil { + return nil, fmt.Errorf("failed to marshal simulator input: %w", err) + } + + messages := []context.Message{{ + Role: context.RoleUser, + Content: string(inputJSON), + }} + + // Call simulator + response, err := simAST.Stream(ctx, messages, opts) + if err != nil { + return nil, fmt.Errorf("simulator agent error: %w", err) + } + + // Parse response + return r.parseSimulatorResponse(response) +} + +// parseSimulatorResponse parses the simulator agent's response +func (r *DynamicRunner) parseSimulatorResponse(response *context.Response) (*SimulatorOutput, error) { + if response == nil || response.Completion == nil { + return nil, fmt.Errorf("empty response from simulator") + } + + // Extract content + content := response.Completion.Content + if content == nil { + return nil, fmt.Errorf("no content in simulator response") + } + + // Convert to string + var text string + switch v := content.(type) { + case string: + text = v + default: + data, err := jsoniter.Marshal(content) + if err != nil { + return nil, fmt.Errorf("failed to marshal content: %w", err) + } + text = string(data) + } + + // Use goutext.ExtractJSON for fault-tolerant parsing + parsed := goutext.ExtractJSON(text) + if parsed == nil { + // Try to use the text as the message directly + return &SimulatorOutput{ + Message: text, + GoalAchieved: false, + }, nil + } + + // Parse as SimulatorOutput + output := &SimulatorOutput{} + if m, ok := parsed.(map[string]interface{}); ok { + if msg, ok := m["message"].(string); ok { + output.Message = msg + } + if achieved, ok := m["goal_achieved"].(bool); ok { + output.GoalAchieved = achieved + } + if reasoning, ok := m["reasoning"].(string); ok { + output.Reasoning = reasoning + } + } + + if output.Message == "" { + return nil, fmt.Errorf("simulator returned empty message") + } + + return output, nil +} + +// checkCheckpoints validates checkpoints against current output +func (r *DynamicRunner) checkCheckpoints(checkpoints []*Checkpoint, output interface{}, result *DynamicResult) []string { + reachedIDs := make([]string, 0) + + for _, cp := range checkpoints { + cpResult := result.Checkpoints[cp.ID] + if cpResult.Reached { + continue // Already reached + } + + // Check "after" constraint + if len(cp.After) > 0 { + allAfterReached := true + for _, afterID := range cp.After { + if afterResult, ok := result.Checkpoints[afterID]; ok { + if !afterResult.Reached { + allAfterReached = false + break + } + } + } + if !allAfterReached { + continue // Dependencies not met + } + } + + // Validate using asserter + tempCase := &Case{Assert: cp.Assert} + passed, msg := r.asserter.Validate(tempCase, output) + + if passed { + cpResult.Reached = true + cpResult.Passed = true + cpResult.ReachedAtTurn = len(result.Turns) + 1 + cpResult.Message = msg + reachedIDs = append(reachedIDs, cp.ID) + } + } + + return reachedIDs +} + +// allRequiredCheckpointsReached checks if all required checkpoints are reached +func (r *DynamicRunner) allRequiredCheckpointsReached(result *DynamicResult) bool { + for _, cp := range result.Checkpoints { + if cp.Required && !cp.Reached { + return false + } + } + return true +} diff --git a/agent/test/dynamic_runner_test.go b/agent/test/dynamic_runner_test.go new file mode 100644 index 00000000..a262e564 --- /dev/null +++ b/agent/test/dynamic_runner_test.go @@ -0,0 +1,319 @@ +package test_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/yao/agent" + "github.com/yaoapp/yao/agent/test" + "github.com/yaoapp/yao/config" + testutils "github.com/yaoapp/yao/test" +) + +func TestCase_IsDynamicMode(t *testing.T) { + tests := []struct { + name string + tc *test.Case + expected bool + }{ + { + name: "standard mode - no simulator", + tc: &test.Case{ + ID: "T001", + Input: "Hello", + }, + expected: false, + }, + { + name: "standard mode - simulator but no checkpoints", + tc: &test.Case{ + ID: "T002", + Input: "Hello", + Simulator: &test.Simulator{Use: "tests.simulator-agent"}, + }, + expected: false, + }, + { + name: "standard mode - checkpoints but no simulator", + tc: &test.Case{ + ID: "T003", + Input: "Hello", + Checkpoints: []*test.Checkpoint{ + {ID: "cp1", Assert: map[string]interface{}{"type": "contains", "value": "hi"}}, + }, + }, + expected: false, + }, + { + name: "dynamic mode - has both simulator and checkpoints", + tc: &test.Case{ + ID: "T004", + Simulator: &test.Simulator{Use: "tests.simulator-agent"}, + Checkpoints: []*test.Checkpoint{ + {ID: "cp1", Assert: map[string]interface{}{"type": "contains", "value": "hi"}}, + }, + }, + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.tc.IsDynamicMode() + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestCase_GetMaxTurns(t *testing.T) { + tests := []struct { + name string + tc *test.Case + expected int + }{ + { + name: "default max turns", + tc: &test.Case{ID: "T001"}, + expected: 20, + }, + { + name: "custom max turns", + tc: &test.Case{ID: "T002", MaxTurns: 10}, + expected: 10, + }, + { + name: "zero max turns uses default", + tc: &test.Case{ID: "T003", MaxTurns: 0}, + expected: 20, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.tc.GetMaxTurns() + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestCheckpoint_IsRequired(t *testing.T) { + boolTrue := true + boolFalse := false + + tests := []struct { + name string + cp *test.Checkpoint + expected bool + }{ + { + name: "default is required", + cp: &test.Checkpoint{ID: "cp1"}, + expected: true, + }, + { + name: "explicitly required", + cp: &test.Checkpoint{ID: "cp2", Required: &boolTrue}, + expected: true, + }, + { + name: "explicitly not required", + cp: &test.Checkpoint{ID: "cp3", Required: &boolFalse}, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.cp.IsRequired() + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestDynamicResult_ToResult(t *testing.T) { + dr := &test.DynamicResult{ + ID: "T001", + Status: test.StatusPassed, + TotalTurns: 3, + DurationMs: 5000, + Turns: []*test.TurnResult{ + {Turn: 1, Input: "Hello", Output: "Hi there!"}, + {Turn: 2, Input: "How are you?", Output: "I'm doing well!"}, + {Turn: 3, Input: "Goodbye", Output: "Bye!"}, + }, + Checkpoints: map[string]*test.CheckpointResult{ + "greet": {ID: "greet", Reached: true, ReachedAtTurn: 1, Required: true}, + "bye": {ID: "bye", Reached: true, ReachedAtTurn: 3, Required: true}, + }, + } + + result := dr.ToResult() + + assert.Equal(t, "T001", result.ID) + assert.Equal(t, test.StatusPassed, result.Status) + assert.Equal(t, int64(5000), result.DurationMs) + assert.Equal(t, "Hello", result.Input) + assert.Equal(t, "Bye!", result.Output) + + // Check metadata + assert.NotNil(t, result.Metadata) + assert.Equal(t, "dynamic", result.Metadata["mode"]) + assert.Equal(t, 3, result.Metadata["total_turns"]) +} + +func TestDynamicRunner_Integration(t *testing.T) { + // Skip if running in short mode + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + // Prepare test environment + testutils.Prepare(t, config.Conf) + defer testutils.Clean() + + // Load agents + err := agent.Load(config.Conf) + if err != nil { + t.Skipf("Failed to load agents: %v", err) + } + + // Create a dynamic test case + tc := &test.Case{ + ID: "dynamic-greeting", + Simulator: &test.Simulator{ + Use: "tests.simulator-agent", + Options: &test.SimulatorOptions{ + Metadata: map[string]interface{}{ + "persona": "Friendly user", + "goal": "Have a brief greeting exchange", + }, + }, + }, + Input: "Hello!", + Checkpoints: []*test.Checkpoint{ + { + ID: "greeting", + Description: "Agent should greet back", + Assert: map[string]interface{}{ + "type": "regex", + "value": "(?i)(hello|hi|hey|greetings)", + }, + }, + }, + MaxTurns: 3, + } + + // Verify it's dynamic mode + assert.True(t, tc.IsDynamicMode()) + + // Create runner options + opts := &test.Options{ + Verbose: true, + Timeout: 30 * time.Second, + } + + // Create dynamic runner + runner := test.NewDynamicRunner(opts) + assert.NotNil(t, runner) + + // Note: Full integration test would require the simulator agent to be loaded + // and would make actual LLM calls. For CI, we test the structure and logic. +} + +func TestDynamicRunner_CheckpointOrdering(t *testing.T) { + // Test that checkpoints with "after" constraints are properly ordered + testutils.Prepare(t, config.Conf) + defer testutils.Clean() + + // Load agents + err := agent.Load(config.Conf) + if err != nil { + t.Skipf("Failed to load agents: %v", err) + } + + // Create a test case with ordered checkpoints + tc := &test.Case{ + ID: "ordered-checkpoints", + Simulator: &test.Simulator{ + Use: "tests.simulator-agent", + Options: &test.SimulatorOptions{ + Metadata: map[string]interface{}{ + "persona": "Customer", + "goal": "Complete a purchase", + }, + }, + }, + Checkpoints: []*test.Checkpoint{ + { + ID: "ask_product", + Description: "Agent asks about product", + Assert: map[string]interface{}{ + "type": "contains", + "value": "product", + }, + }, + { + ID: "confirm_order", + Description: "Agent confirms order", + After: []string{"ask_product"}, + Assert: map[string]interface{}{ + "type": "contains", + "value": "confirm", + }, + }, + { + ID: "complete", + Description: "Order completed", + After: []string{"confirm_order"}, + Assert: map[string]interface{}{ + "type": "contains", + "value": "complete", + }, + }, + }, + MaxTurns: 10, + } + + // Verify checkpoint structure + assert.Len(t, tc.Checkpoints, 3) + assert.Empty(t, tc.Checkpoints[0].After) + assert.Equal(t, []string{"ask_product"}, tc.Checkpoints[1].After) + assert.Equal(t, []string{"confirm_order"}, tc.Checkpoints[2].After) +} + +func TestSimulatorInput_Structure(t *testing.T) { + // Test SimulatorInput structure + input := &test.SimulatorInput{ + Persona: "Test user", + Goal: "Complete task", + TurnNumber: 3, + MaxTurns: 10, + CheckpointsReached: []string{"cp1", "cp2"}, + CheckpointsPending: []string{"cp3"}, + Extra: map[string]interface{}{ + "style": "formal", + }, + } + + assert.Equal(t, "Test user", input.Persona) + assert.Equal(t, "Complete task", input.Goal) + assert.Equal(t, 3, input.TurnNumber) + assert.Equal(t, 10, input.MaxTurns) + assert.Len(t, input.CheckpointsReached, 2) + assert.Len(t, input.CheckpointsPending, 1) + assert.Equal(t, "formal", input.Extra["style"]) +} + +func TestSimulatorOutput_Structure(t *testing.T) { + // Test SimulatorOutput structure + output := &test.SimulatorOutput{ + Message: "I'd like to buy a product", + GoalAchieved: false, + Reasoning: "Continuing toward purchase goal", + } + + assert.Equal(t, "I'd like to buy a product", output.Message) + assert.False(t, output.GoalAchieved) + assert.Equal(t, "Continuing toward purchase goal", output.Reasoning) +} diff --git a/agent/test/dynamic_types.go b/agent/test/dynamic_types.go new file mode 100644 index 00000000..d75ec2c4 --- /dev/null +++ b/agent/test/dynamic_types.go @@ -0,0 +1,159 @@ +package test + +import "github.com/yaoapp/yao/agent/context" + +// DynamicResult represents the result of a dynamic (simulator-driven) test +type DynamicResult struct { + // ID is the test case identifier + ID string `json:"id"` + + // Status is the overall test status + Status Status `json:"status"` + + // Turns contains results for each conversation turn + Turns []*TurnResult `json:"turns"` + + // Checkpoints maps checkpoint ID to its result + Checkpoints map[string]*CheckpointResult `json:"checkpoints"` + + // TotalTurns is the number of turns executed + TotalTurns int `json:"total_turns"` + + // DurationMs is the total execution time in milliseconds + DurationMs int64 `json:"duration_ms"` + + // Error contains error message if status is failed/error/timeout + Error string `json:"error,omitempty"` +} + +// TurnResult represents the result of a single conversation turn +type TurnResult struct { + // Turn is the turn number (1-based) + Turn int `json:"turn"` + + // Input is the user message (from simulator or initial input) + Input interface{} `json:"input"` + + // Output is the agent's response + Output interface{} `json:"output,omitempty"` + + // CheckpointsReached lists checkpoint IDs reached in this turn + CheckpointsReached []string `json:"checkpoints_reached,omitempty"` + + // DurationMs is the turn execution time in milliseconds + DurationMs int64 `json:"duration_ms"` + + // Error contains error message if this turn failed + Error string `json:"error,omitempty"` +} + +// CheckpointResult represents the result of a checkpoint validation +type CheckpointResult struct { + // ID is the checkpoint identifier + ID string `json:"id"` + + // Reached indicates if the checkpoint was reached + Reached bool `json:"reached"` + + // ReachedAtTurn is the turn number when checkpoint was reached (0 if not reached) + ReachedAtTurn int `json:"reached_at_turn,omitempty"` + + // Required indicates if this checkpoint is required + Required bool `json:"required"` + + // Passed indicates if the checkpoint assertion passed + Passed bool `json:"passed"` + + // Message contains assertion result message + Message string `json:"message,omitempty"` +} + +// SimulatorInput is the input sent to the simulator agent +type SimulatorInput struct { + // Persona describes the user being simulated + Persona string `json:"persona,omitempty"` + + // Goal is what the user is trying to achieve + Goal string `json:"goal,omitempty"` + + // Conversation is the message history + Conversation []context.Message `json:"conversation"` + + // TurnNumber is the current turn (1-based) + TurnNumber int `json:"turn_number"` + + // MaxTurns is the maximum allowed turns + MaxTurns int `json:"max_turns"` + + // CheckpointsReached lists checkpoint IDs already reached + CheckpointsReached []string `json:"checkpoints_reached,omitempty"` + + // CheckpointsPending lists checkpoint IDs still pending + CheckpointsPending []string `json:"checkpoints_pending,omitempty"` + + // Extra metadata from simulator options + Extra map[string]interface{} `json:"extra,omitempty"` +} + +// SimulatorOutput is the expected output from the simulator agent +type SimulatorOutput struct { + // Message is the simulated user message + Message string `json:"message"` + + // GoalAchieved indicates if the user's goal has been accomplished + GoalAchieved bool `json:"goal_achieved"` + + // Reasoning explains the simulator's response strategy + Reasoning string `json:"reasoning,omitempty"` +} + +// ToResult converts DynamicResult to standard Result for reporting +func (dr *DynamicResult) ToResult() *Result { + result := &Result{ + ID: dr.ID, + Status: dr.Status, + DurationMs: dr.DurationMs, + Error: dr.Error, + } + + // Store dynamic-specific data in metadata + result.Metadata = map[string]interface{}{ + "mode": "dynamic", + "total_turns": dr.TotalTurns, + "turns": dr.Turns, + "checkpoints": dr.Checkpoints, + } + + // Set input from first turn + if len(dr.Turns) > 0 { + result.Input = dr.Turns[0].Input + } + + // Set output from last turn + if len(dr.Turns) > 0 { + result.Output = dr.Turns[len(dr.Turns)-1].Output + } + + return result +} + +// IsDynamicMode checks if a test case should run in dynamic mode +func (tc *Case) IsDynamicMode() bool { + return tc.Simulator != nil && len(tc.Checkpoints) > 0 +} + +// GetMaxTurns returns the max turns for dynamic mode +func (tc *Case) GetMaxTurns() int { + if tc.MaxTurns > 0 { + return tc.MaxTurns + } + return 20 // Default max turns +} + +// IsRequired returns true if the checkpoint is required +func (cp *Checkpoint) IsRequired() bool { + if cp.Required == nil { + return true // Default to required + } + return *cp.Required +} diff --git a/agent/test/input_source.go b/agent/test/input_source.go new file mode 100644 index 00000000..0b905f80 --- /dev/null +++ b/agent/test/input_source.go @@ -0,0 +1,392 @@ +package test + +import ( + "fmt" + "net/url" + "strconv" + "strings" + + jsoniter "github.com/json-iterator/go" + goutext "github.com/yaoapp/gou/text" + "github.com/yaoapp/yao/agent/assistant" + "github.com/yaoapp/yao/agent/context" +) + +// InputSourceType represents the type of input source +type InputSourceType string + +const ( + // InputSourceFile indicates input from a JSONL file + InputSourceFile InputSourceType = "file" + // InputSourceMessage indicates input from a direct message string + InputSourceMessage InputSourceType = "message" + // InputSourceScript indicates script test mode + InputSourceScript InputSourceType = "script" + // InputSourceAgent indicates input generated by an agent + InputSourceAgent InputSourceType = "agent" +) + +// InputSource represents a parsed input source +type InputSource struct { + Type InputSourceType // file, message, script, agent + Value string // path, message, script ref, or agent ID + Params map[string]interface{} // query parameters (for agent source) +} + +// ParseInputSource parses the -i flag value into an InputSource +// Supported formats: +// - "agents:workers.test.generator" - Agent-generated test cases +// - "agents:workers.test.generator?count=10&focus=edge-cases" - With parameters +// - "scripts.tests.gen" - Script-generated test cases +// - "./tests/inputs.jsonl" - JSONL file +// - "Hello, how are you?" - Direct message +func ParseInputSource(input string) *InputSource { + // Check for agents: prefix + if strings.HasPrefix(input, "agents:") { + return parseAgentSource(strings.TrimPrefix(input, "agents:")) + } + + // Check for scripts: prefix (for generator scripts) + if strings.HasPrefix(input, "scripts:") { + return &InputSource{ + Type: InputSourceScript, + Value: strings.TrimPrefix(input, "scripts:"), + } + } + + // Check for script test mode (scripts.xxx format without prefix) + if strings.HasPrefix(input, "scripts.") { + return &InputSource{ + Type: InputSourceScript, + Value: input, + } + } + + // Check for file extension + if strings.HasSuffix(input, ".jsonl") || strings.HasSuffix(input, ".json") { + return &InputSource{ + Type: InputSourceFile, + Value: input, + } + } + + // Check if it looks like a file path + if strings.Contains(input, "/") || strings.Contains(input, "\\") { + return &InputSource{ + Type: InputSourceFile, + Value: input, + } + } + + // Default to message + return &InputSource{ + Type: InputSourceMessage, + Value: input, + } +} + +// parseAgentSource parses an agent source string with optional query parameters +// Format: "agent.id" or "agent.id?count=10&focus=edge-cases" +func parseAgentSource(input string) *InputSource { + source := &InputSource{ + Type: InputSourceAgent, + Params: make(map[string]interface{}), + } + + // Check for query parameters + if idx := strings.Index(input, "?"); idx >= 0 { + source.Value = input[:idx] + queryStr := input[idx+1:] + + // Parse query parameters + values, err := url.ParseQuery(queryStr) + if err == nil { + for key, vals := range values { + if len(vals) > 0 { + // Try to parse as number + if num, err := strconv.Atoi(vals[0]); err == nil { + source.Params[key] = num + } else if num, err := strconv.ParseFloat(vals[0], 64); err == nil { + source.Params[key] = num + } else if vals[0] == "true" { + source.Params[key] = true + } else if vals[0] == "false" { + source.Params[key] = false + } else { + source.Params[key] = vals[0] + } + } + } + } + } else { + source.Value = input + } + + return source +} + +// GeneratorInput represents the input sent to a generator agent +type GeneratorInput struct { + TargetAgent *TargetAgentInfo `json:"target_agent"` + Count int `json:"count,omitempty"` + Focus string `json:"focus,omitempty"` + Extra map[string]interface{} `json:"extra,omitempty"` +} + +// TargetAgentInfo contains information about the agent being tested +type TargetAgentInfo struct { + ID string `json:"id"` + Description string `json:"description,omitempty"` + Tools []map[string]interface{} `json:"tools,omitempty"` +} + +// GenerateTestCases generates test cases using a generator agent +func GenerateTestCases(agentID string, targetInfo *TargetAgentInfo, params map[string]interface{}) ([]*Case, error) { + // Get generator assistant + ast, err := assistant.Get(agentID) + if err != nil { + return nil, fmt.Errorf("failed to get generator agent %s: %w", agentID, err) + } + + // Build generation request + genInput := &GeneratorInput{ + TargetAgent: targetInfo, + Count: 5, // Default count + } + + // Apply parameters + if params != nil { + if count, ok := params["count"].(int); ok { + genInput.Count = count + } + if focus, ok := params["focus"].(string); ok { + genInput.Focus = focus + } + // Store extra parameters + genInput.Extra = make(map[string]interface{}) + for k, v := range params { + if k != "count" && k != "focus" { + genInput.Extra[k] = v + } + } + } + + // Create context + env := NewEnvironment("", "") + ctx := NewTestContext("generator", agentID, env) + defer ctx.Release() + + // Build options - skip history and trace for efficiency + opts := &context.Options{ + Skip: &context.Skip{ + History: true, + Trace: true, + Output: true, + }, + Metadata: map[string]interface{}{ + "test_mode": "generator", + }, + } + + // Build message + inputJSON, err := jsoniter.Marshal(genInput) + if err != nil { + return nil, fmt.Errorf("failed to marshal generator input: %w", err) + } + + messages := []context.Message{{ + Role: context.RoleUser, + Content: string(inputJSON), + }} + + // Call generator agent + response, err := ast.Stream(ctx, messages, opts) + if err != nil { + return nil, fmt.Errorf("generator agent error: %w", err) + } + + // Extract and parse response + return parseGeneratedCases(response) +} + +// parseGeneratedCases parses the generator agent's response into test cases +func parseGeneratedCases(response *context.Response) ([]*Case, error) { + if response == nil || response.Completion == nil { + return nil, fmt.Errorf("empty response from generator agent") + } + + // Extract content + content := response.Completion.Content + if content == nil { + return nil, fmt.Errorf("no content in generator response") + } + + // Convert content to string + var text string + switch v := content.(type) { + case string: + text = v + default: + data, err := jsoniter.Marshal(content) + if err != nil { + return nil, fmt.Errorf("failed to marshal content: %w", err) + } + text = string(data) + } + + // Use goutext.ExtractJSON for fault-tolerant parsing + parsed := goutext.ExtractJSON(text) + if parsed == nil { + return nil, fmt.Errorf("failed to parse generator response as JSON: %s", truncateOutput(text, 200)) + } + + // Convert to []*Case + return convertToCases(parsed) +} + +// convertToCases converts parsed JSON to test cases +func convertToCases(parsed interface{}) ([]*Case, error) { + // Handle array of cases + arr, ok := parsed.([]interface{}) + if !ok { + // Maybe it's a single case wrapped in an object + if obj, ok := parsed.(map[string]interface{}); ok { + if cases, ok := obj["cases"].([]interface{}); ok { + arr = cases + } else if testCases, ok := obj["test_cases"].([]interface{}); ok { + arr = testCases + } else { + // Single case + arr = []interface{}{obj} + } + } else { + return nil, fmt.Errorf("expected array of test cases, got %T", parsed) + } + } + + cases := make([]*Case, 0, len(arr)) + for i, item := range arr { + caseMap, ok := item.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("test case %d is not an object", i) + } + + tc, err := mapToCase(caseMap) + if err != nil { + return nil, fmt.Errorf("failed to parse test case %d: %w", i, err) + } + + cases = append(cases, tc) + } + + return cases, nil +} + +// mapToCase converts a map to a Case struct +func mapToCase(m map[string]interface{}) (*Case, error) { + tc := &Case{} + + // Required: id + if id, ok := m["id"].(string); ok { + tc.ID = id + } else { + return nil, fmt.Errorf("missing required field 'id'") + } + + // Required: input + if input, ok := m["input"]; ok { + tc.Input = input + } else { + return nil, fmt.Errorf("missing required field 'input'") + } + + // Optional: assertions/assert + if assertions, ok := m["assertions"]; ok { + tc.Assert = assertions + } else if assert, ok := m["assert"]; ok { + tc.Assert = assert + } + + // Optional: options - convert map to CaseOptions + if options, ok := m["options"].(map[string]interface{}); ok { + tc.Options = mapToCaseOptions(options) + } + + // Optional: before/after + if before, ok := m["before"].(string); ok { + tc.Before = before + } + if after, ok := m["after"].(string); ok { + tc.After = after + } + + // Optional: timeout + if timeout, ok := m["timeout"].(string); ok { + tc.Timeout = timeout + } + + return tc, nil +} + +// ToInputMode converts InputSourceType to InputMode for backward compatibility +func (s *InputSource) ToInputMode() InputMode { + switch s.Type { + case InputSourceFile: + return InputModeFile + case InputSourceMessage: + return InputModeMessage + case InputSourceScript: + return InputModeScript + case InputSourceAgent: + // Agent source generates cases, then runs in file mode + return InputModeFile + default: + return InputModeMessage + } +} + +// mapToCaseOptions converts a map to CaseOptions +func mapToCaseOptions(m map[string]interface{}) *CaseOptions { + opts := &CaseOptions{} + + if connector, ok := m["connector"].(string); ok { + opts.Connector = connector + } + + if mode, ok := m["mode"].(string); ok { + opts.Mode = mode + } + + if disableGlobalPrompts, ok := m["disable_global_prompts"].(bool); ok { + opts.DisableGlobalPrompts = disableGlobalPrompts + } + + if search, ok := m["search"].(bool); ok { + opts.Search = &search + } + + if metadata, ok := m["metadata"].(map[string]interface{}); ok { + opts.Metadata = metadata + } + + if skip, ok := m["skip"].(map[string]interface{}); ok { + opts.Skip = &CaseSkipOptions{} + if history, ok := skip["history"].(bool); ok { + opts.Skip.History = history + } + if trace, ok := skip["trace"].(bool); ok { + opts.Skip.Trace = trace + } + if output, ok := skip["output"].(bool); ok { + opts.Skip.Output = output + } + if keyword, ok := skip["keyword"].(bool); ok { + opts.Skip.Keyword = keyword + } + if searchSkip, ok := skip["search"].(bool); ok { + opts.Skip.Search = searchSkip + } + } + + return opts +} diff --git a/agent/test/input_source_test.go b/agent/test/input_source_test.go new file mode 100644 index 00000000..7e89a9ae --- /dev/null +++ b/agent/test/input_source_test.go @@ -0,0 +1,181 @@ +package test_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/yao/agent" + agenttest "github.com/yaoapp/yao/agent/test" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +func TestParseInputSource(t *testing.T) { + tests := []struct { + name string + input string + wantType agenttest.InputSourceType + wantValue string + wantParams map[string]interface{} + }{ + { + name: "JSONL file", + input: "./tests/inputs.jsonl", + wantType: agenttest.InputSourceFile, + wantValue: "./tests/inputs.jsonl", + }, + { + name: "JSON file", + input: "./tests/inputs.json", + wantType: agenttest.InputSourceFile, + wantValue: "./tests/inputs.json", + }, + { + name: "direct message", + input: "Hello, how are you?", + wantType: agenttest.InputSourceMessage, + wantValue: "Hello, how are you?", + }, + { + name: "agent source simple", + input: "agents:tests.generator-agent", + wantType: agenttest.InputSourceAgent, + wantValue: "tests.generator-agent", + }, + { + name: "agent source with params", + input: "agents:tests.generator-agent?count=10&focus=edge-cases", + wantType: agenttest.InputSourceAgent, + wantValue: "tests.generator-agent", + wantParams: map[string]interface{}{ + "count": 10, + "focus": "edge-cases", + }, + }, + { + name: "agent source with boolean param", + input: "agents:tests.generator-agent?verbose=true", + wantType: agenttest.InputSourceAgent, + wantValue: "tests.generator-agent", + wantParams: map[string]interface{}{ + "verbose": true, + }, + }, + { + name: "script source with prefix", + input: "scripts:tests.gen.Generate", + wantType: agenttest.InputSourceScript, + wantValue: "tests.gen.Generate", + }, + { + name: "script test mode", + input: "scripts.tests.gen", + wantType: agenttest.InputSourceScript, + wantValue: "scripts.tests.gen", + }, + { + name: "path with separator", + input: "/path/to/inputs.jsonl", + wantType: agenttest.InputSourceFile, + wantValue: "/path/to/inputs.jsonl", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := agenttest.ParseInputSource(tt.input) + + assert.Equal(t, tt.wantType, source.Type, "Type mismatch") + assert.Equal(t, tt.wantValue, source.Value, "Value mismatch") + + if tt.wantParams != nil { + for k, v := range tt.wantParams { + assert.Equal(t, v, source.Params[k], "Param %s mismatch", k) + } + } + }) + } +} + +func TestInputSource_ToInputMode(t *testing.T) { + tests := []struct { + name string + source *agenttest.InputSource + wantMode agenttest.InputMode + }{ + { + name: "file source", + source: &agenttest.InputSource{Type: agenttest.InputSourceFile}, + wantMode: agenttest.InputModeFile, + }, + { + name: "message source", + source: &agenttest.InputSource{Type: agenttest.InputSourceMessage}, + wantMode: agenttest.InputModeMessage, + }, + { + name: "script source", + source: &agenttest.InputSource{Type: agenttest.InputSourceScript}, + wantMode: agenttest.InputModeScript, + }, + { + name: "agent source", + source: &agenttest.InputSource{Type: agenttest.InputSourceAgent}, + wantMode: agenttest.InputModeFile, // Agent generates cases, then runs in file mode + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mode := tt.source.ToInputMode() + assert.Equal(t, tt.wantMode, mode) + }) + } +} + +func TestGenerateTestCases(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent (includes assistants) + err := agent.Load(config.Conf) + if err != nil { + t.Fatalf("Failed to load agent: %v", err) + } + + // Test generating test cases from the generator agent + targetInfo := &agenttest.TargetAgentInfo{ + ID: "tests.next", + Description: "A simple test agent for greeting", + } + + params := map[string]interface{}{ + "count": 3, + "focus": "happy-path", + } + + cases, err := agenttest.GenerateTestCases("tests.generator-agent", targetInfo, params) + if err != nil { + t.Fatalf("Failed to generate test cases: %v", err) + } + + // Verify we got some test cases + assert.NotEmpty(t, cases, "Should generate at least one test case") + + // Verify each case has required fields + for _, tc := range cases { + assert.NotEmpty(t, tc.ID, "Test case should have ID") + assert.NotNil(t, tc.Input, "Test case should have Input") + } + + t.Logf("Generated %d test cases", len(cases)) + for _, tc := range cases { + t.Logf(" - %s", tc.ID) + } +} + +func TestMapToCaseOptions(t *testing.T) { + // Test that options map is correctly converted + source := agenttest.ParseInputSource("agents:test?count=5") + assert.Equal(t, 5, source.Params["count"]) +} diff --git a/agent/test/interfaces.go b/agent/test/interfaces.go index a5996010..d0083d2e 100644 --- a/agent/test/interfaces.go +++ b/agent/test/interfaces.go @@ -43,6 +43,12 @@ type Loader interface { // LoadFile loads test cases from a JSONL file LoadFile(path string) ([]*Case, error) + + // LoadFromAgent generates test cases using a generator agent + LoadFromAgent(agentID string, targetInfo *TargetAgentInfo, params map[string]interface{}) ([]*Case, error) + + // LoadFromScript generates test cases using a script + LoadFromScript(scriptRef string, targetInfo *TargetAgentInfo) ([]*Case, error) } // Resolver is the interface for resolving agent information diff --git a/agent/test/loader.go b/agent/test/loader.go index 49de1a77..52092b68 100644 --- a/agent/test/loader.go +++ b/agent/test/loader.go @@ -8,6 +8,7 @@ import ( "time" jsoniter "github.com/json-iterator/go" + "github.com/yaoapp/gou/process" ) // JSONLLoader loads test cases from JSONL files @@ -143,3 +144,35 @@ func FilterByIDs(cases []*Case, ids []string) []*Case { return idSet[tc.ID] }) } + +// LoadFromAgent generates test cases using a generator agent +func (l *JSONLLoader) LoadFromAgent(agentID string, targetInfo *TargetAgentInfo, params map[string]interface{}) ([]*Case, error) { + return GenerateTestCases(agentID, targetInfo, params) +} + +// LoadFromScript generates test cases using a script +// scriptRef format: "module.FunctionName" (e.g., "tests.gen.Generate") +func (l *JSONLLoader) LoadFromScript(scriptRef string, targetInfo *TargetAgentInfo) ([]*Case, error) { + // Parse script reference + parts := strings.Split(scriptRef, ".") + if len(parts) < 2 { + return nil, fmt.Errorf("invalid script reference format: %s (expected 'module.Function')", scriptRef) + } + + // Build process name: scripts.module.Function + processName := "scripts." + scriptRef + + // Execute via process + p, err := process.Of(processName, targetInfo) + if err != nil { + return nil, fmt.Errorf("failed to create process %s: %w", processName, err) + } + + result, err := p.Exec() + if err != nil { + return nil, fmt.Errorf("script execution failed: %w", err) + } + + // Parse result as test cases + return convertToCases(result) +} diff --git a/agent/test/output.go b/agent/test/output.go index a49f1f6e..aaedcf04 100644 --- a/agent/test/output.go +++ b/agent/test/output.go @@ -303,6 +303,45 @@ func (w *OutputWriter) ScriptTestSummary(summary *ScriptTestSummary, duration ti fmt.Printf("%s\n", formatDuration(duration)) } +// DynamicTestStart outputs the start of a dynamic test +func (w *OutputWriter) DynamicTestStart(id string, checkpointCount int) { + color.New(color.FgWhite).Printf("► [%s] ", id) + color.New(color.FgCyan).Printf("(dynamic, %d checkpoints)\n", checkpointCount) +} + +// DynamicTurn outputs a single turn in dynamic testing +func (w *OutputWriter) DynamicTurn(turn int, inputSummary string, checkpointsReached, total int) { + if w.verbose { + color.New(color.FgHiBlack).Printf("│ ├─ Turn %d: %s ", turn, inputSummary) + color.New(color.FgCyan).Printf("[%d/%d checkpoints]\n", checkpointsReached, total) + } +} + +// DynamicCheckpoint outputs a checkpoint being reached +func (w *OutputWriter) DynamicCheckpoint(checkpointID string) { + if w.verbose { + color.New(color.FgGreen).Printf("│ │ └─ ✓ checkpoint: %s\n", checkpointID) + } +} + +// DynamicTestResult outputs the result of a dynamic test +func (w *OutputWriter) DynamicTestResult(status Status, turns int, checkpoints int, duration time.Duration) { + color.New(color.FgHiBlack).Printf(" └─ ") + + switch status { + case StatusPassed: + color.New(color.FgGreen).Printf("PASSED") + case StatusFailed: + color.New(color.FgRed).Printf("FAILED") + case StatusError: + color.New(color.FgRed).Printf("ERROR") + case StatusTimeout: + color.New(color.FgRed).Printf("TIMEOUT") + } + + color.New(color.FgHiBlack).Printf(" (%d turns, %d checkpoints, %s)\n", turns, checkpoints, formatDuration(duration)) +} + // StabilityResult prints stability analysis result for a test case func (w *OutputWriter) StabilityResult(sr *StabilityResult) { color.New(color.FgWhite).Printf(" [%s] ", sr.ID) diff --git a/agent/test/runner.go b/agent/test/runner.go index 32658fdb..3f18e9a8 100644 --- a/agent/test/runner.go +++ b/agent/test/runner.go @@ -1,11 +1,12 @@ package test import ( - "bufio" stdContext "context" "fmt" "os" "path/filepath" + "reflect" + "strings" "sync" "time" @@ -16,19 +17,22 @@ import ( // Executor executes test cases against an agent type Executor struct { - opts *Options - output *OutputWriter - resolver Resolver - loader Loader + opts *Options + output *OutputWriter + resolver Resolver + loader Loader + hookExecutor *HookExecutor + agentPath string // Path to the agent being tested } // NewRunner creates a new test runner func NewRunner(opts *Options) *Executor { return &Executor{ - opts: opts, - output: NewOutputWriter(opts.Verbose), - resolver: NewResolver(), - loader: NewLoader(), + opts: opts, + output: NewOutputWriter(opts.Verbose), + resolver: NewResolver(), + loader: NewLoader(), + hookExecutor: NewHookExecutor(opts.Verbose), } } @@ -161,21 +165,65 @@ func (r *Executor) RunTests() (*Report, error) { } r.output.Info("Agent: %s", agentInfo.ID) + r.agentPath = agentInfo.Path // Store agent path for hook execution if r.opts.Connector != "" { r.output.Info("Connector: %s (override)", r.opts.Connector) } else if agentInfo.Connector != "" { r.output.Info("Connector: %s", agentInfo.Connector) } - // Load test cases + // Load test cases based on input source var testCases []*Case + inputSource := ParseInputSource(r.opts.Input) - // File mode - load from JSONL - testCases, err = r.loader.LoadFile(r.opts.Input) - if err != nil { - return nil, fmt.Errorf("failed to load test cases: %w", err) + switch inputSource.Type { + case InputSourceAgent: + // Generate test cases using agent + r.output.Info("Generating test cases from agent: %s", inputSource.Value) + targetInfo := &TargetAgentInfo{ + ID: agentInfo.ID, + Description: agentInfo.Description, + } + testCases, err = r.loader.LoadFromAgent(inputSource.Value, targetInfo, inputSource.Params) + if err != nil { + return nil, fmt.Errorf("failed to generate test cases: %w", err) + } + r.output.Info("Generated: %d test cases", len(testCases)) + + case InputSourceScript: + // Generate test cases using script (if it's a generator script, not test script) + // Note: scripts. prefix without "scripts:" is handled by RunScriptTests + if strings.HasPrefix(r.opts.Input, "scripts:") { + scriptRef := strings.TrimPrefix(r.opts.Input, "scripts:") + r.output.Info("Generating test cases from script: %s", scriptRef) + targetInfo := &TargetAgentInfo{ + ID: agentInfo.ID, + Description: agentInfo.Description, + } + testCases, err = r.loader.LoadFromScript(scriptRef, targetInfo) + if err != nil { + return nil, fmt.Errorf("failed to generate test cases from script: %w", err) + } + r.output.Info("Generated: %d test cases", len(testCases)) + } else { + // This is a test script (scripts.xxx format), handled by RunScriptTests + return nil, fmt.Errorf("script test mode should be handled by RunScriptTests") + } + + default: + // File mode - load from JSONL + testCases, err = r.loader.LoadFile(r.opts.Input) + if err != nil { + return nil, fmt.Errorf("failed to load test cases: %w", err) + } + r.output.Info("Input: %s (%d test cases)", r.opts.Input, len(testCases)) + } + + // Handle dry-run mode - just output the generated test cases + if r.opts.DryRun { + r.output.Info("Dry-run mode: outputting generated test cases") + return r.outputDryRun(testCases, agentInfo) } - r.output.Info("Input: %s (%d test cases)", r.opts.Input, len(testCases)) // Filter skipped tests activeTests := FilterSkipped(testCases) @@ -222,6 +270,27 @@ func (r *Executor) RunTests() (*Report, error) { }, } + // Execute global BeforeAll if specified + var globalBeforeData interface{} + if r.opts.BeforeAll != "" { + r.output.Info("BeforeAll: %s", r.opts.BeforeAll) + var err error + globalBeforeData, err = r.hookExecutor.ExecuteBeforeAll(r.opts.BeforeAll, activeTests, agentInfo.Path) + if err != nil { + return nil, fmt.Errorf("beforeAll script failed: %w", err) + } + } + + // Ensure AfterAll runs even if tests fail + defer func() { + if r.opts.AfterAll != "" { + r.output.Info("AfterAll: %s", r.opts.AfterAll) + if err := r.hookExecutor.ExecuteAfterAll(r.opts.AfterAll, report.Results, globalBeforeData, agentInfo.Path); err != nil { + r.output.Warning("afterAll script failed: %s", err.Error()) + } + } + }() + // Run tests r.output.SubHeader("Running Tests") @@ -308,6 +377,11 @@ func (r *Executor) runParallel(ast *assistant.Assistant, testCases []*Case, agen // runSingleTest runs a single test case func (r *Executor) runSingleTest(ast *assistant.Assistant, tc *Case, agentID string, runNum int) *Result { + // Check if this is a dynamic mode test + if tc.IsDynamicMode() { + return r.runDynamicTest(ast, tc, agentID) + } + // Get input summary for display inputSummary := SummarizeInput(tc.Input, 50) r.output.TestStart(tc.ID, inputSummary, runNum) @@ -322,6 +396,31 @@ func (r *Executor) runSingleTest(ast *assistant.Assistant, tc *Case, agentID str Options: tc.Options, } + // Execute before script if specified + var beforeData interface{} + if tc.Before != "" { + var err error + beforeData, err = r.hookExecutor.ExecuteBefore(tc.Before, tc, r.agentPath) + if err != nil { + result.Status = StatusError + result.Error = fmt.Sprintf("before script failed: %s", err.Error()) + result.DurationMs = time.Since(startTime).Milliseconds() + r.output.TestResult(result.Status, time.Since(startTime)) + r.output.TestError(result.Error) + // Note: after script is NOT called when before fails + return result + } + } + + // Ensure after script runs even if test fails (but only if before succeeded) + defer func() { + if tc.After != "" && (tc.Before == "" || beforeData != nil || result.Status != StatusError || !isBeforeError(result.Error)) { + if err := r.hookExecutor.ExecuteAfter(tc.After, tc, result, beforeData, r.agentPath); err != nil { + r.output.Warning("after script failed: %s", err.Error()) + } + } + }() + // Parse input to messages with file loading support // BaseDir is derived from the input file directory inputOpts := r.getInputOptions() @@ -395,6 +494,71 @@ func (r *Executor) runSingleTest(ast *assistant.Assistant, tc *Case, agentID str return result } +// runDynamicTest runs a dynamic (simulator-driven) test case +func (r *Executor) runDynamicTest(ast *assistant.Assistant, tc *Case, agentID string) *Result { + // Output test start for dynamic mode + r.output.DynamicTestStart(tc.ID, len(tc.Checkpoints)) + + startTime := time.Now() + + // Execute before script if specified + var beforeData interface{} + if tc.Before != "" { + var err error + beforeData, err = r.hookExecutor.ExecuteBefore(tc.Before, tc, r.agentPath) + if err != nil { + result := &Result{ + ID: tc.ID, + Status: StatusError, + Error: fmt.Sprintf("before script failed: %s", err.Error()), + DurationMs: time.Since(startTime).Milliseconds(), + } + r.output.TestResult(result.Status, time.Since(startTime)) + r.output.TestError(result.Error) + return result + } + } + + // Create dynamic runner and execute + dynamicRunner := NewDynamicRunner(r.opts) + dynamicResult := dynamicRunner.RunDynamic(ast, tc, agentID) + + // Convert to standard result + result := dynamicResult.ToResult() + + // Execute after script if specified + defer func() { + if tc.After != "" && (tc.Before == "" || beforeData != nil || result.Status != StatusError || !isBeforeError(result.Error)) { + if err := r.hookExecutor.ExecuteAfter(tc.After, tc, result, beforeData, r.agentPath); err != nil { + r.output.Warning("after script failed: %s", err.Error()) + } + } + }() + + // Output result + duration := time.Duration(result.DurationMs) * time.Millisecond + r.output.DynamicTestResult(result.Status, dynamicResult.TotalTurns, len(tc.Checkpoints), duration) + + if result.Error != "" { + r.output.TestError(result.Error) + } + + return result +} + +// isBeforeError checks if the error message indicates a before script failure +func isBeforeError(errMsg string) bool { + return len(errMsg) > 0 && errMsg[:min(len(errMsg), 20)] == "before script failed" +} + +// min returns the minimum of two integers +func min(a, b int) int { + if a < b { + return a + } + return b +} + // runStabilityTests runs each test case multiple times for stability analysis func (r *Executor) runStabilityTests(ast *assistant.Assistant, testCases []*Case, agentID string) []*StabilityResult { results := make([]*StabilityResult, 0, len(testCases)) @@ -499,20 +663,6 @@ func (r *Executor) writeOutput(report *Report) error { return reporter.Write(report, file) } -// writeJSONLine writes a JSON line to the writer -func writeJSONLine(writer *bufio.Writer, data interface{}) error { - line, err := jsoniter.Marshal(data) - if err != nil { - return err - } - _, err = writer.Write(line) - if err != nil { - return err - } - _, err = writer.WriteString("\n") - return err -} - // buildContextOptions builds context.Options from test case and runner options // Priority: test case options > runner options > defaults func buildContextOptions(tc *Case, runnerOpts *Options) *context.Options { @@ -593,6 +743,12 @@ func isEmptyValue(v interface{}) bool { return true } + // Use reflection to check for typed nil (e.g., *NextHookResponse(nil)) + rv := reflect.ValueOf(v) + if rv.Kind() == reflect.Ptr && rv.IsNil() { + return true + } + switch val := v.(type) { case string: return val == "" @@ -600,6 +756,12 @@ func isEmptyValue(v interface{}) bool { return len(val) == 0 case []interface{}: return len(val) == 0 + case *context.NextHookResponse: + // Check if NextHookResponse is effectively empty + if val == nil { + return true + } + return val.Data == nil && val.Delegate == nil } return false @@ -633,3 +795,51 @@ func (r *Executor) getInputOptions() *InputOptions { return opts } + +// outputDryRun outputs generated test cases without running them +func (r *Executor) outputDryRun(testCases []*Case, agentInfo *AgentInfo) (*Report, error) { + r.output.Info("Generated Test Cases:") + + // Output each test case as JSONL + for _, tc := range testCases { + data, err := jsoniter.Marshal(tc) + if err != nil { + r.output.Warning("Failed to marshal test case %s: %s", tc.ID, err.Error()) + continue + } + fmt.Println(string(data)) + } + + // Write to output file if specified + if r.opts.OutputFile != "" { + file, err := os.Create(r.opts.OutputFile) + if err != nil { + return nil, fmt.Errorf("failed to create output file: %w", err) + } + defer file.Close() + + for _, tc := range testCases { + data, err := jsoniter.Marshal(tc) + if err != nil { + continue + } + file.WriteString(string(data) + "\n") + } + + r.output.Info("Output written to: %s", r.opts.OutputFile) + } + + // Return a minimal report + connector := r.opts.Connector + if connector == "" { + connector = agentInfo.Connector + } + + return &Report{ + Summary: &Summary{ + Total: len(testCases), + AgentID: agentInfo.ID, + Connector: connector, + }, + }, nil +} diff --git a/agent/test/runner_integration_test.go b/agent/test/runner_integration_test.go new file mode 100644 index 00000000..aae1c398 --- /dev/null +++ b/agent/test/runner_integration_test.go @@ -0,0 +1,280 @@ +package test_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/yaoapp/yao/agent" + agenttest "github.com/yaoapp/yao/agent/test" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +// TestRunner_AgentDrivenInput tests the complete flow: +// 1. Use generator-agent to generate test cases +// 2. Run the generated tests against simple-greeting agent +func TestRunner_AgentDrivenInput(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Test with agent-driven input + opts := &agenttest.Options{ + Input: "agents:tests.generator-agent?count=3", + AgentID: "tests.simple-greeting", + Verbose: true, + InputMode: agenttest.InputModeFile, // Will be overridden by ParseInputSource + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + require.NotNil(t, report.Summary, "Summary should not be nil") + + // Verify report + assert.Greater(t, report.Summary.Total, 0, "Should have at least one test case") + t.Logf("Total: %d, Passed: %d, Failed: %d", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) +} + +// TestRunner_AgentDrivenInput_DryRun tests dry-run mode +func TestRunner_AgentDrivenInput_DryRun(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Test with dry-run mode + opts := &agenttest.Options{ + Input: "agents:tests.generator-agent?count=2", + AgentID: "tests.simple-greeting", + DryRun: true, + Verbose: true, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Dry-run should not return error") + require.NotNil(t, report, "Report should not be nil") + + // In dry-run mode, tests are generated but not executed + // So Passed and Failed should both be 0, but Total should have the count + assert.Greater(t, report.Summary.Total, 0, "Should have generated test cases") + + t.Logf("Generated %d test cases in dry-run mode", report.Summary.Total) +} + +// TestRunner_FileInput tests loading test cases from JSONL file +func TestRunner_FileInput(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Create a temporary JSONL file with test cases + // Use case-insensitive contains for robustness + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "inputs.jsonl") + + testCases := `{"id": "greeting-hello", "input": "Hello", "assert": {"type": "regex", "value": "(?i)hello"}} +{"id": "greeting-hi", "input": "Hi there", "assert": {"type": "regex", "value": "(?i)(hi|hello)"}} +{"id": "greeting-morning", "input": "Good morning", "assert": {"type": "regex", "value": "(?i)(hello|morning|good)"}}` + + err = os.WriteFile(inputFile, []byte(testCases), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run tests from file + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.simple-greeting", + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + // Verify report + assert.Equal(t, 3, report.Summary.Total, "Should have 3 test cases") + t.Logf("Total: %d, Passed: %d, Failed: %d", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) + + // Check results for debugging + if report.Results != nil { + for _, r := range report.Results { + t.Logf(" [%s] Status: %s, Output: %v", r.ID, r.Status, r.Output) + } + } +} + +// TestRunner_DirectMessage tests direct message mode +func TestRunner_DirectMessage(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Test with direct message + opts := &agenttest.Options{ + Input: "Hello, how are you?", + AgentID: "tests.simple-greeting", + Verbose: true, + InputMode: agenttest.InputModeMessage, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + // Direct message mode returns a minimal report + assert.Equal(t, 1, report.Summary.Total, "Should have 1 test case") + assert.Equal(t, 1, report.Summary.Passed, "Direct message should pass") +} + +// TestRunner_WithBeforeAfter tests before/after hooks +func TestRunner_WithBeforeAfter(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Create a temporary JSONL file with test cases that use hooks + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "inputs.jsonl") + + // Note: hooks-test agent has env_test.ts with Before/After functions + testCases := `{"id": "hook-test-1", "input": "Hello", "assert": {"type": "contains", "value": "hello"}, "before": "env_test.Before", "after": "env_test.After"}` + + err = os.WriteFile(inputFile, []byte(testCases), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run tests with hooks (using hooks-test agent which has the hook scripts) + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.hooks-test", + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + t.Logf("Total: %d, Passed: %d, Failed: %d", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) +} + +// TestRunner_Parallel tests parallel execution +func TestRunner_Parallel(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Create a temporary JSONL file with multiple test cases + // Use regex for case-insensitive matching + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "inputs.jsonl") + + testCases := `{"id": "parallel-1", "input": "Hello", "assert": {"type": "regex", "value": "(?i)(hello|hi)"}} +{"id": "parallel-2", "input": "Hi", "assert": {"type": "regex", "value": "(?i)(hello|hi)"}} +{"id": "parallel-3", "input": "Hey", "assert": {"type": "regex", "value": "(?i)(hello|hi|hey)"}} +{"id": "parallel-4", "input": "Good day", "assert": {"type": "regex", "value": "(?i)(hello|good|day)"}}` + + err = os.WriteFile(inputFile, []byte(testCases), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run tests in parallel + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.simple-greeting", + Parallel: 2, // Run 2 tests in parallel + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + assert.Equal(t, 4, report.Summary.Total, "Should have 4 test cases") + t.Logf("Total: %d, Passed: %d, Failed: %d (parallel: 2)", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) +} + +// TestRunner_FailFast tests fail-fast behavior +func TestRunner_FailFast(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Create a temporary JSONL file with a failing test first + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "inputs.jsonl") + + // First test will fail (expects "impossible" which won't be in response) + testCases := `{"id": "fail-first", "input": "Hello", "assert": {"type": "contains", "value": "IMPOSSIBLE_STRING_12345"}} +{"id": "should-skip", "input": "Hi", "assert": {"type": "contains", "value": "hi"}}` + + err = os.WriteFile(inputFile, []byte(testCases), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run tests with fail-fast + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.simple-greeting", + FailFast: true, + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error (fail-fast is not an error)") + require.NotNil(t, report, "Report should not be nil") + + // With fail-fast, only the first test should run + assert.Equal(t, 1, report.Summary.Failed, "First test should fail") + // The second test might not run due to fail-fast + t.Logf("Total: %d, Passed: %d, Failed: %d (fail-fast enabled)", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) +} diff --git a/agent/test/script_assert.go b/agent/test/script_assert.go index c0cc0245..c2c22ffd 100644 --- a/agent/test/script_assert.go +++ b/agent/test/script_assert.go @@ -261,6 +261,9 @@ func newAssertObject(v8ctx *v8go.Context, t *TestingT) (*v8go.Value, error) { // JSON path assertion assertObj.Set("JSONPath", assertJSONPathMethod(iso, t)) + // Agent-driven assertion + assertObj.Set("Agent", assertAgentMethod(iso, t)) + // Create instance instance, err := assertObj.NewInstance(v8ctx) if err != nil { @@ -924,6 +927,70 @@ func assertJSONPathMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate }) } +// assertAgentMethod implements assert.Agent(response, agentID, options?) +// Uses a validator agent to check the response +// agentID is the direct agent ID (no "agents:" prefix needed) +func assertAgentMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + v8ctx := info.Context() + args := info.Args() + if len(args) < 2 { + t.fail("Agent requires response and agentID arguments", &ScriptAssertionInfo{Type: "Agent"}) + return v8go.Undefined(iso) + } + + response, _ := bridge.GoValue(args[0], v8ctx) + agentID := args[1].String() + + // Get options if provided + var options map[string]interface{} + if len(args) > 2 && args[2].IsObject() { + optVal, _ := bridge.GoValue(args[2], v8ctx) + options, _ = optVal.(map[string]interface{}) + } + + // Build assertion with agents: prefix + assertion := &Assertion{ + Type: "agent", + Use: "agents:" + agentID, + } + + // Extract criteria and metadata from options + if options != nil { + if criteria, ok := options["criteria"]; ok { + assertion.Value = criteria + } + if metadata, ok := options["metadata"].(map[string]interface{}); ok { + assertion.Options = &AssertionOptions{Metadata: metadata} + } + if connector, ok := options["connector"].(string); ok { + if assertion.Options == nil { + assertion.Options = &AssertionOptions{} + } + assertion.Options.Connector = connector + } + } + + // Use the asserter to validate + asserter := &Asserter{} + result := asserter.assertAgent(assertion, response, nil) + + if !result.Passed { + msg := result.Message + if msg == "" { + msg = "agent assertion failed" + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "Agent", + Actual: response, + Message: msg, + }) + } + + return v8go.Undefined(iso) + }) +} + // Helper functions // deepEqual performs deep equality comparison diff --git a/agent/test/script_hooks.go b/agent/test/script_hooks.go new file mode 100644 index 00000000..ea9349e1 --- /dev/null +++ b/agent/test/script_hooks.go @@ -0,0 +1,592 @@ +package test + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/yaoapp/gou/application" + v8 "github.com/yaoapp/gou/runtime/v8" + "github.com/yaoapp/gou/runtime/v8/bridge" + "github.com/yaoapp/yao/agent/context" + "rogchap.com/v8go" +) + +// HookExecutor executes before/after scripts from *_test.ts files +// Scripts are loaded via V8 and executed directly, not via Process() +type HookExecutor struct { + verbose bool + output *OutputWriter + loadedDirs map[string]bool // Track which directories have been loaded + agentContext *context.Context +} + +// NewHookExecutor creates a new hook executor +func NewHookExecutor(verbose bool) *HookExecutor { + return &HookExecutor{ + verbose: verbose, + output: NewOutputWriter(verbose), + loadedDirs: make(map[string]bool), + } +} + +// SetAgentContext sets the agent context for script execution +func (h *HookExecutor) SetAgentContext(ctx *context.Context) { + h.agentContext = ctx +} + +// HookRef represents a parsed hook reference +// Format: "src/env_test.ts:Before" or just "Before" (uses default test file) +type HookRef struct { + ScriptFile string // e.g., "env_test.ts" + Function string // e.g., "Before" +} + +// ParseHookRef parses a hook reference string +// Formats: +// - "Before" -> uses first *_test.ts file found +// - "env_test.Before" -> uses src/env_test.ts +// - "src/env_test.Before" -> uses src/env_test.ts +func ParseHookRef(ref string) (*HookRef, error) { + if ref == "" { + return nil, fmt.Errorf("empty hook reference") + } + + // Split by last dot to get function name + lastDot := strings.LastIndex(ref, ".") + if lastDot == -1 { + // Just function name, will use default test file + return &HookRef{ + ScriptFile: "", // Will be resolved later + Function: ref, + }, nil + } + + scriptPart := ref[:lastDot] + funcName := ref[lastDot+1:] + + // Normalize script file name + scriptFile := scriptPart + if !strings.HasSuffix(scriptFile, "_test") { + scriptFile += "_test" + } + scriptFile += ".ts" + + // Remove "src/" prefix if present + scriptFile = strings.TrimPrefix(scriptFile, "src/") + + return &HookRef{ + ScriptFile: scriptFile, + Function: funcName, + }, nil +} + +// LoadTestScripts loads all *_test.ts scripts from the agent's src directory +// Returns the script IDs that were loaded +func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) { + srcDir := filepath.Join(agentPath, "src") + + // Check if already loaded + if h.loadedDirs[srcDir] { + return nil, nil + } + + // Check if src directory exists + exists, err := application.App.Exists(srcDir) + if err != nil { + return nil, err + } + if !exists { + return nil, nil // No src directory, not an error + } + + var loadedScripts []string + exts := []string{"*_test.ts", "*_test.js"} + + err = application.App.Walk(srcDir, func(root, file string, isdir bool) error { + if isdir { + return nil + } + + // Only load *_test.ts/js files + base := filepath.Base(file) + if !strings.HasSuffix(base, "_test.ts") && !strings.HasSuffix(base, "_test.js") { + return nil + } + + // Generate script ID + scriptID := generateHookScriptID(file, srcDir) + + // Load the script + _, err := v8.Load(file, scriptID) + if err != nil { + if h.verbose { + h.output.Warning("Failed to load hook script %s: %v", base, err) + } + return nil // Continue loading other scripts + } + + loadedScripts = append(loadedScripts, scriptID) + if h.verbose { + h.output.Verbose("Loaded hook script: %s (id: %s)", base, scriptID) + } + + return nil + }, exts...) + + if err != nil { + return nil, fmt.Errorf("failed to walk src directory: %w", err) + } + + h.loadedDirs[srcDir] = true + return loadedScripts, nil +} + +// generateHookScriptID generates a script ID for hook scripts +// Example: assistants/test/src/env_test.ts -> hook.env_test +func generateHookScriptID(filePath string, srcDir string) string { + filePath = filepath.ToSlash(filePath) + srcDir = filepath.ToSlash(srcDir) + + relPath := strings.TrimPrefix(filePath, srcDir+"/") + relPath = strings.TrimPrefix(relPath, "/") + relPath = strings.TrimSuffix(relPath, filepath.Ext(relPath)) + + return "hook." + strings.ReplaceAll(relPath, "/", ".") +} + +// FindTestScript finds a loaded test script by pattern +// If scriptFile is empty, returns the first *_test script found +func (h *HookExecutor) FindTestScript(scriptFile string) (*v8.Script, string, error) { + if scriptFile != "" { + // Look for specific script + scriptID := "hook." + strings.TrimSuffix(scriptFile, ".ts") + scriptID = strings.TrimSuffix(scriptID, ".js") + + if script, ok := v8.Scripts[scriptID]; ok { + return script, scriptID, nil + } + return nil, "", fmt.Errorf("hook script not found: %s (id: %s)", scriptFile, scriptID) + } + + // Find first *_test script + for id, script := range v8.Scripts { + if strings.HasPrefix(id, "hook.") && strings.Contains(id, "_test") { + return script, id, nil + } + } + + return nil, "", fmt.Errorf("no hook test script found") +} + +// ExecuteBefore executes a Before function from a test script +func (h *HookExecutor) ExecuteBefore(ref string, testCase *Case, agentPath string) (interface{}, error) { + hookRef, err := ParseHookRef(ref) + if err != nil { + return nil, err + } + + // Ensure scripts are loaded + if _, err := h.LoadTestScripts(agentPath); err != nil { + return nil, fmt.Errorf("failed to load test scripts: %w", err) + } + + // Find the script + script, scriptID, err := h.FindTestScript(hookRef.ScriptFile) + if err != nil { + return nil, err + } + + if h.verbose { + h.output.Verbose("Executing %s from %s", hookRef.Function, scriptID) + } + + // Execute the function + return h.executeHookFunction(script, hookRef.Function, testCase, nil, nil) +} + +// ExecuteAfter executes an After function from a test script +func (h *HookExecutor) ExecuteAfter(ref string, testCase *Case, result *Result, beforeData interface{}, agentPath string) error { + hookRef, err := ParseHookRef(ref) + if err != nil { + return err + } + + // Ensure scripts are loaded + if _, err := h.LoadTestScripts(agentPath); err != nil { + return fmt.Errorf("failed to load test scripts: %w", err) + } + + // Find the script + script, scriptID, err := h.FindTestScript(hookRef.ScriptFile) + if err != nil { + return err + } + + if h.verbose { + h.output.Verbose("Executing %s from %s", hookRef.Function, scriptID) + } + + // Execute the function + _, err = h.executeHookFunction(script, hookRef.Function, testCase, result, beforeData) + return err +} + +// ExecuteBeforeAll executes a BeforeAll function +func (h *HookExecutor) ExecuteBeforeAll(ref string, testCases []*Case, agentPath string) (interface{}, error) { + hookRef, err := ParseHookRef(ref) + if err != nil { + return nil, err + } + + // Ensure scripts are loaded + if _, err := h.LoadTestScripts(agentPath); err != nil { + return nil, fmt.Errorf("failed to load test scripts: %w", err) + } + + // Find the script + script, scriptID, err := h.FindTestScript(hookRef.ScriptFile) + if err != nil { + return nil, err + } + + if h.verbose { + h.output.Verbose("Executing %s from %s", hookRef.Function, scriptID) + } + + // Execute with test cases array + return h.executeHookFunctionWithCases(script, hookRef.Function, testCases) +} + +// ExecuteAfterAll executes an AfterAll function +func (h *HookExecutor) ExecuteAfterAll(ref string, results []*Result, beforeData interface{}, agentPath string) error { + hookRef, err := ParseHookRef(ref) + if err != nil { + return err + } + + // Ensure scripts are loaded + if _, err := h.LoadTestScripts(agentPath); err != nil { + return fmt.Errorf("failed to load test scripts: %w", err) + } + + // Find the script + script, scriptID, err := h.FindTestScript(hookRef.ScriptFile) + if err != nil { + return err + } + + if h.verbose { + h.output.Verbose("Executing %s from %s", hookRef.Function, scriptID) + } + + // Execute with results array + _, err = h.executeHookFunctionWithResults(script, hookRef.Function, results, beforeData) + return err +} + +// executeHookFunction executes a hook function with test case context +func (h *HookExecutor) executeHookFunction(script *v8.Script, funcName string, testCase *Case, result *Result, beforeData interface{}) (interface{}, error) { + // Create script context + scriptCtx, err := script.NewContext("", nil) + if err != nil { + return nil, fmt.Errorf("failed to create script context: %w", err) + } + defer scriptCtx.Close() + + v8ctx := scriptCtx.Context + + // Set share data + if err := h.setShareData(v8ctx); err != nil { + return nil, err + } + + // Get the function + global := v8ctx.Global() + fnValue, err := global.Get(funcName) + if err != nil { + return nil, fmt.Errorf("failed to get function %s: %w", funcName, err) + } + + if fnValue.IsUndefined() || fnValue.IsNull() { + return nil, fmt.Errorf("function %s not defined", funcName) + } + + if !fnValue.IsFunction() { + return nil, fmt.Errorf("%s is not a function", funcName) + } + + fn, err := fnValue.AsFunction() + if err != nil { + return nil, fmt.Errorf("failed to convert to function: %w", err) + } + + // Build arguments + args, err := h.buildHookArgs(v8ctx, testCase, result, beforeData) + if err != nil { + return nil, err + } + + // Convert to v8go.Valuer slice for Call + valuerArgs := make([]v8go.Valuer, len(args)) + for i, arg := range args { + valuerArgs[i] = arg + } + + // Call the function + jsResult, err := fn.Call(global, valuerArgs...) + if err != nil { + return nil, fmt.Errorf("hook function %s failed: %w", funcName, err) + } + + // Convert result to Go value + if jsResult == nil || jsResult.IsUndefined() || jsResult.IsNull() { + return nil, nil + } + + goResult, err := bridge.GoValue(jsResult, v8ctx) + if err != nil { + return nil, fmt.Errorf("failed to convert result: %w", err) + } + + // Extract data field if present + if resultMap, ok := goResult.(map[string]interface{}); ok { + if data, exists := resultMap["data"]; exists { + return data, nil + } + } + + return goResult, nil +} + +// executeHookFunctionWithCases executes BeforeAll with test cases array +func (h *HookExecutor) executeHookFunctionWithCases(script *v8.Script, funcName string, testCases []*Case) (interface{}, error) { + scriptCtx, err := script.NewContext("", nil) + if err != nil { + return nil, fmt.Errorf("failed to create script context: %w", err) + } + defer scriptCtx.Close() + + v8ctx := scriptCtx.Context + + if err := h.setShareData(v8ctx); err != nil { + return nil, err + } + + global := v8ctx.Global() + fnValue, err := global.Get(funcName) + if err != nil { + return nil, fmt.Errorf("failed to get function %s: %w", funcName, err) + } + + if fnValue.IsUndefined() || fnValue.IsNull() { + return nil, fmt.Errorf("function %s not defined", funcName) + } + + if !fnValue.IsFunction() { + return nil, fmt.Errorf("%s is not a function", funcName) + } + + fn, err := fnValue.AsFunction() + if err != nil { + return nil, fmt.Errorf("failed to convert to function: %w", err) + } + + // Convert test cases to JS array + casesJS, err := h.testCasesToJS(v8ctx, testCases) + if err != nil { + return nil, err + } + + jsResult, err := fn.Call(global, casesJS) + if err != nil { + return nil, fmt.Errorf("hook function %s failed: %w", funcName, err) + } + + if jsResult == nil || jsResult.IsUndefined() || jsResult.IsNull() { + return nil, nil + } + + goResult, err := bridge.GoValue(jsResult, v8ctx) + if err != nil { + return nil, fmt.Errorf("failed to convert result: %w", err) + } + + if resultMap, ok := goResult.(map[string]interface{}); ok { + if data, exists := resultMap["data"]; exists { + return data, nil + } + } + + return goResult, nil +} + +// executeHookFunctionWithResults executes AfterAll with results array +func (h *HookExecutor) executeHookFunctionWithResults(script *v8.Script, funcName string, results []*Result, beforeData interface{}) (interface{}, error) { + scriptCtx, err := script.NewContext("", nil) + if err != nil { + return nil, fmt.Errorf("failed to create script context: %w", err) + } + defer scriptCtx.Close() + + v8ctx := scriptCtx.Context + + if err := h.setShareData(v8ctx); err != nil { + return nil, err + } + + global := v8ctx.Global() + fnValue, err := global.Get(funcName) + if err != nil { + return nil, fmt.Errorf("failed to get function %s: %w", funcName, err) + } + + if fnValue.IsUndefined() || fnValue.IsNull() { + return nil, fmt.Errorf("function %s not defined", funcName) + } + + if !fnValue.IsFunction() { + return nil, fmt.Errorf("%s is not a function", funcName) + } + + fn, err := fnValue.AsFunction() + if err != nil { + return nil, fmt.Errorf("failed to convert to function: %w", err) + } + + // Convert results to JS array + resultsJS, err := h.resultsToJS(v8ctx, results) + if err != nil { + return nil, err + } + + // Convert beforeData to JS + beforeDataJS, err := bridge.JsValue(v8ctx, beforeData) + if err != nil { + return nil, fmt.Errorf("failed to convert beforeData: %w", err) + } + + jsResult, err := fn.Call(global, resultsJS, beforeDataJS) + if err != nil { + return nil, fmt.Errorf("hook function %s failed: %w", funcName, err) + } + + if jsResult == nil || jsResult.IsUndefined() || jsResult.IsNull() { + return nil, nil + } + + goResult, err := bridge.GoValue(jsResult, v8ctx) + if err != nil { + return nil, fmt.Errorf("failed to convert result: %w", err) + } + + return goResult, nil +} + +// setShareData sets the share data for script execution +func (h *HookExecutor) setShareData(v8ctx *v8go.Context) error { + var authorized map[string]interface{} + if h.agentContext != nil && h.agentContext.Authorized != nil { + authorized = h.agentContext.Authorized.AuthorizedToMap() + } + + return bridge.SetShareData(v8ctx, v8ctx.Global(), &bridge.Share{ + Sid: "", + Root: false, + Global: nil, + Authorized: authorized, + }) +} + +// buildHookArgs builds the arguments for a hook function call +func (h *HookExecutor) buildHookArgs(v8ctx *v8go.Context, testCase *Case, result *Result, beforeData interface{}) ([]*v8go.Value, error) { + var args []*v8go.Value + + // Arg 1: testCase + if testCase != nil { + tcMap := map[string]interface{}{ + "id": testCase.ID, + "input": testCase.Input, + } + if testCase.Metadata != nil { + tcMap["metadata"] = testCase.Metadata + } + if testCase.Assert != nil { + tcMap["assert"] = testCase.Assert + } + + tcJS, err := bridge.JsValue(v8ctx, tcMap) + if err != nil { + return nil, fmt.Errorf("failed to convert testCase: %w", err) + } + args = append(args, tcJS) + } + + // Arg 2: result (for After) + if result != nil { + resultMap := map[string]interface{}{ + "id": result.ID, + "status": string(result.Status), + "duration_ms": result.DurationMs, + } + if result.Output != nil { + resultMap["output"] = result.Output + } + if result.Error != "" { + resultMap["error"] = result.Error + } + + resultJS, err := bridge.JsValue(v8ctx, resultMap) + if err != nil { + return nil, fmt.Errorf("failed to convert result: %w", err) + } + args = append(args, resultJS) + } + + // Arg 3: beforeData (for After) + if beforeData != nil { + beforeDataJS, err := bridge.JsValue(v8ctx, beforeData) + if err != nil { + return nil, fmt.Errorf("failed to convert beforeData: %w", err) + } + args = append(args, beforeDataJS) + } + + return args, nil +} + +// testCasesToJS converts test cases to a JS array +func (h *HookExecutor) testCasesToJS(v8ctx *v8go.Context, testCases []*Case) (*v8go.Value, error) { + cases := make([]map[string]interface{}, len(testCases)) + for i, tc := range testCases { + cases[i] = map[string]interface{}{ + "id": tc.ID, + "input": tc.Input, + } + if tc.Metadata != nil { + cases[i]["metadata"] = tc.Metadata + } + } + + return bridge.JsValue(v8ctx, cases) +} + +// resultsToJS converts results to a JS array +func (h *HookExecutor) resultsToJS(v8ctx *v8go.Context, results []*Result) (*v8go.Value, error) { + resultMaps := make([]map[string]interface{}, len(results)) + for i, r := range results { + resultMaps[i] = map[string]interface{}{ + "id": r.ID, + "status": string(r.Status), + "duration_ms": r.DurationMs, + } + if r.Output != nil { + resultMaps[i]["output"] = r.Output + } + if r.Error != "" { + resultMaps[i]["error"] = r.Error + } + } + + return bridge.JsValue(v8ctx, resultMaps) +} diff --git a/agent/test/script_hooks_test.go b/agent/test/script_hooks_test.go new file mode 100644 index 00000000..8720c463 --- /dev/null +++ b/agent/test/script_hooks_test.go @@ -0,0 +1,244 @@ +package test_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v8 "github.com/yaoapp/gou/runtime/v8" + agenttest "github.com/yaoapp/yao/agent/test" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +const hooksTestAgent = "assistants/tests/hooks-test" + +func TestParseHookRef(t *testing.T) { + tests := []struct { + name string + input string + wantFile string + wantFunc string + expectErr bool + }{ + { + name: "function only", + input: "Before", + wantFile: "", + wantFunc: "Before", + }, + { + name: "with script file", + input: "env_test.Before", + wantFile: "env_test.ts", + wantFunc: "Before", + }, + { + name: "with src prefix", + input: "src/env_test.Before", + wantFile: "env_test.ts", + wantFunc: "Before", + }, + { + name: "nested path", + input: "setup/db_test.Before", + wantFile: "setup/db_test.ts", + wantFunc: "Before", + }, + { + name: "empty string", + input: "", + expectErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ref, err := agenttest.ParseHookRef(tt.input) + if tt.expectErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.wantFile, ref.ScriptFile) + assert.Equal(t, tt.wantFunc, ref.Function) + }) + } +} + +func TestHookExecutorLoadTestScripts(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts using the utility function + scripts := test.LoadAgentTestScripts(t, hooksTestAgent) + + assert.NotEmpty(t, scripts, "Should load at least one test script") + + // Verify the script was loaded into V8 + found := false + for _, scriptID := range scripts { + if _, ok := v8.Scripts[scriptID]; ok { + found = true + t.Logf("Loaded script: %s", scriptID) + break + } + } + assert.True(t, found, "At least one script should be loaded into V8") +} + +func TestHookExecutorExecuteBefore(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts + test.LoadAgentTestScripts(t, hooksTestAgent) + + executor := agenttest.NewHookExecutor(true) + + testCase := &agenttest.Case{ + ID: "TEST001", + Input: "Hello World", + } + + // Execute Before hook + beforeData, err := executor.ExecuteBefore("env_test.Before", testCase, hooksTestAgent) + assert.NoError(t, err) + assert.NotNil(t, beforeData) + + // Verify returned data + dataMap, ok := beforeData.(map[string]interface{}) + assert.True(t, ok, "beforeData should be a map") + assert.Equal(t, "TEST001", dataMap["test_id"]) + assert.NotEmpty(t, dataMap["mock_user_id"]) + assert.NotEmpty(t, dataMap["mock_session_id"]) +} + +func TestHookExecutorExecuteAfter(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts + test.LoadAgentTestScripts(t, hooksTestAgent) + + executor := agenttest.NewHookExecutor(true) + + testCase := &agenttest.Case{ + ID: "TEST002", + Input: "Test input", + } + + result := &agenttest.Result{ + ID: "TEST002", + Status: agenttest.StatusPassed, + DurationMs: 100, + } + + beforeData := map[string]interface{}{ + "test_id": "TEST002", + "mock_user_id": "user_TEST002_12345", + "mock_session_id": "session_12345", + } + + // Execute After hook + err := executor.ExecuteAfter("env_test.After", testCase, result, beforeData, hooksTestAgent) + assert.NoError(t, err) +} + +func TestHookExecutorExecuteBeforeAll(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts + test.LoadAgentTestScripts(t, hooksTestAgent) + + executor := agenttest.NewHookExecutor(true) + + testCases := []*agenttest.Case{ + {ID: "T001", Input: "Test 1"}, + {ID: "T002", Input: "Test 2"}, + {ID: "T003", Input: "Test 3"}, + } + + // Execute BeforeAll hook + globalData, err := executor.ExecuteBeforeAll("env_test.BeforeAll", testCases, hooksTestAgent) + assert.NoError(t, err) + assert.NotNil(t, globalData) + + // Verify returned data + dataMap, ok := globalData.(map[string]interface{}) + assert.True(t, ok, "globalData should be a map") + assert.NotEmpty(t, dataMap["suite_id"]) + assert.Equal(t, float64(3), dataMap["test_count"]) // JSON numbers are float64 +} + +func TestHookExecutorExecuteAfterAll(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts + test.LoadAgentTestScripts(t, hooksTestAgent) + + executor := agenttest.NewHookExecutor(true) + + results := []*agenttest.Result{ + {ID: "T001", Status: agenttest.StatusPassed, DurationMs: 100}, + {ID: "T002", Status: agenttest.StatusFailed, DurationMs: 200, Error: "assertion failed"}, + {ID: "T003", Status: agenttest.StatusPassed, DurationMs: 150}, + } + + globalData := map[string]interface{}{ + "suite_id": "suite_12345", + "test_count": 3, + } + + // Execute AfterAll hook + err := executor.ExecuteAfterAll("env_test.AfterAll", results, globalData, hooksTestAgent) + assert.NoError(t, err) +} + +func TestHookExecutorFunctionNotFound(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts + test.LoadAgentTestScripts(t, hooksTestAgent) + + executor := agenttest.NewHookExecutor(true) + + testCase := &agenttest.Case{ + ID: "TEST001", + Input: "Hello", + } + + // Try to execute non-existent function + _, err := executor.ExecuteBefore("env_test.NonExistent", testCase, hooksTestAgent) + assert.Error(t, err) + assert.Contains(t, err.Error(), "not defined") +} + +func TestHookExecutorScriptNotFound(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts + test.LoadAgentTestScripts(t, hooksTestAgent) + + executor := agenttest.NewHookExecutor(true) + + testCase := &agenttest.Case{ + ID: "TEST001", + Input: "Hello", + } + + // Try to execute from non-existent script + _, err := executor.ExecuteBefore("nonexistent_test.Before", testCase, hooksTestAgent) + assert.Error(t, err) + assert.Contains(t, err.Error(), "not found") +} diff --git a/agent/test/types.go b/agent/test/types.go index b40ab457..cddbeb8a 100644 --- a/agent/test/types.go +++ b/agent/test/types.go @@ -144,6 +144,22 @@ type Options struct { // Only tests matching the pattern will be executed // Example: "TestSystem" matches TestSystemReady, TestSystemError, etc. Run string `json:"run,omitempty"` + + // BeforeAll is the global before script (e.g., "scripts:tests.env.BeforeAll") + // Called once before all test cases + BeforeAll string `json:"before_all,omitempty"` + + // AfterAll is the global after script (e.g., "scripts:tests.env.AfterAll") + // Called once after all test cases + AfterAll string `json:"after_all,omitempty"` + + // DryRun generates test cases without running them + // Useful for previewing agent-generated test cases + DryRun bool `json:"dry_run,omitempty"` + + // Simulator is the default simulator agent ID for dynamic mode + // Can be overridden per test case in JSONL + Simulator string `json:"simulator,omitempty"` } // ContextConfig represents custom context configuration from JSON file @@ -375,6 +391,68 @@ type Case struct { // Timeout overrides the default timeout for this test case // Format: "30s", "1m", "2m30s" Timeout string `json:"timeout,omitempty"` + + // Before script function (e.g., "scripts:tests.env.Before") + // Called before the test case runs, returns data passed to After + Before string `json:"before,omitempty"` + + // After script function (e.g., "scripts:tests.env.After") + // Called after the test case completes (pass or fail) + After string `json:"after,omitempty"` + + // Dynamic Mode Fields + // =============================== + + // Simulator configures the user simulator for dynamic testing + // When set, the test runs in dynamic mode with multi-turn conversation + Simulator *Simulator `json:"simulator,omitempty"` + + // Checkpoints define validation points for dynamic testing + // Each checkpoint is checked after every agent response + Checkpoints []*Checkpoint `json:"checkpoints,omitempty"` + + // MaxTurns is the maximum number of conversation turns (default: 20) + MaxTurns int `json:"max_turns,omitempty"` +} + +// Simulator configures the user simulator for dynamic testing +type Simulator struct { + // Use is the simulator agent ID (no prefix needed) + Use string `json:"use"` + + // Options for the simulator agent + Options *SimulatorOptions `json:"options,omitempty"` +} + +// SimulatorOptions configures simulator behavior +type SimulatorOptions struct { + // Metadata passed to the simulator agent + // Common fields: persona, goal, style + Metadata map[string]interface{} `json:"metadata,omitempty"` + + // Connector overrides the simulator's default connector + Connector string `json:"connector,omitempty"` +} + +// Checkpoint defines a validation point in dynamic testing +type Checkpoint struct { + // ID is the unique identifier for this checkpoint + ID string `json:"id"` + + // Description is a human-readable description + Description string `json:"description,omitempty"` + + // Assert defines the assertion to validate + // Same format as Case.Assert + Assert interface{} `json:"assert"` + + // After specifies checkpoint IDs that must be reached before this one + // Used to enforce ordering (e.g., "ask_type" must come before "confirm") + After []string `json:"after,omitempty"` + + // Required indicates if this checkpoint must be reached (default: true) + // Optional checkpoints don't cause test failure if not reached + Required *bool `json:"required,omitempty"` } // CaseOptions represents per-test-case context options @@ -420,6 +498,7 @@ type Assertion struct { // - "script": run a custom assertion script // - "type": check output type (string, object, array, number, boolean) // - "schema": validate against JSON schema + // - "agent": use an agent to validate the response Type string `json:"type"` // Value is the expected value or pattern (depends on type) @@ -432,6 +511,14 @@ type Assertion struct { // The script receives (output, input, expected) and returns {pass: bool, message: string} Script string `json:"script,omitempty"` + // Use specifies the agent/script for validation + // For agent assertions: "agents:tests.validator-agent" (with prefix) + // For script assertions: "scripts:tests.validate" (with prefix) + Use string `json:"use,omitempty"` + + // Options for agent-driven assertions (aligned with context.Options) + Options *AssertionOptions `json:"options,omitempty"` + // Message is a custom failure message Message string `json:"message,omitempty"` @@ -439,6 +526,15 @@ type Assertion struct { Negate bool `json:"negate,omitempty"` } +// AssertionOptions for agent-driven assertions +type AssertionOptions struct { + // Connector overrides the agent's default connector + Connector string `json:"connector,omitempty"` + + // Metadata contains custom data passed to the validator agent + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + // AssertionResult represents the result of an assertion type AssertionResult struct { // Passed indicates whether the assertion passed diff --git a/cmd/agent/test.go b/cmd/agent/test.go index 7af0ef03..ebe543d1 100644 --- a/cmd/agent/test.go +++ b/cmd/agent/test.go @@ -33,6 +33,10 @@ var ( testParallel int testVerbose bool testFailFast bool + testBefore string // --before flag for global BeforeAll hook + testAfter string // --after flag for global AfterAll hook + testDryRun bool // --dry-run flag for generating tests without running + testSimulator string // --simulator flag for default simulator agent in dynamic mode ) // TestCmd is the agent test command @@ -157,6 +161,10 @@ var TestCmd = &cobra.Command{ Parallel: testParallel, Verbose: testVerbose, FailFast: testFailFast, + BeforeAll: testBefore, + AfterAll: testAfter, + DryRun: testDryRun, + Simulator: testSimulator, } // Merge with defaults @@ -244,6 +252,10 @@ func init() { TestCmd.Flags().IntVar(&testParallel, "parallel", 1, L("Number of parallel test cases")) TestCmd.Flags().BoolVarP(&testVerbose, "verbose", "v", false, L("Verbose output")) TestCmd.Flags().BoolVar(&testFailFast, "fail-fast", false, L("Stop on first failure")) + TestCmd.Flags().StringVar(&testBefore, "before", "", L("Global BeforeAll hook (e.g., env_test.BeforeAll)")) + TestCmd.Flags().StringVar(&testAfter, "after", "", L("Global AfterAll hook (e.g., env_test.AfterAll)")) + TestCmd.Flags().BoolVar(&testDryRun, "dry-run", false, L("Generate test cases without running them")) + TestCmd.Flags().StringVar(&testSimulator, "simulator", "", L("Default simulator agent for dynamic mode (e.g., tests.simulator-agent)")) // Mark input as required TestCmd.MarkFlagRequired("input") diff --git a/test/utils.go b/test/utils.go index 624bdb59..b7d06d5b 100644 --- a/test/utils.go +++ b/test/utils.go @@ -800,3 +800,70 @@ func GuardBearerJWT(c *gin.Context) { claims := helper.JwtValidate(tokenString) c.Set("__sid", claims.SID) } + +// LoadAgentTestScripts loads all *_test.ts/js scripts from an agent's src directory. +// This is useful for testing agent hooks (before/after scripts) and other agent-specific test scripts. +// +// Usage: +// +// test.Prepare(t, config.Conf) +// defer test.Clean() +// scripts := test.LoadAgentTestScripts(t, "assistants/tests/hooks-test") +// +// Parameters: +// - t: testing.T instance +// - agentRelPath: relative path to agent directory from app root (e.g., "assistants/tests/hooks-test") +// +// Returns: +// - []string: list of loaded script IDs (e.g., ["hook.env_test"]) +func LoadAgentTestScripts(t *testing.T, agentRelPath string) []string { + srcDir := filepath.Join(agentRelPath, "src") + + // Check if src directory exists + exists, err := application.App.Exists(srcDir) + if err != nil { + t.Fatalf("Failed to check src directory: %v", err) + } + if !exists { + t.Logf("No src directory found at %s, skipping", srcDir) + return nil + } + + var loadedScripts []string + exts := []string{"*_test.ts", "*_test.js"} + + err = application.App.Walk(srcDir, func(root, file string, isdir bool) error { + if isdir { + return nil + } + + // Only load *_test.ts/js files + base := filepath.Base(file) + if !strings.HasSuffix(base, "_test.ts") && !strings.HasSuffix(base, "_test.js") { + return nil + } + + // Generate script ID: hook.{relative_path_without_ext} + // e.g., assistants/tests/hooks-test/src/env_test.ts -> hook.env_test + relPath := strings.TrimPrefix(file, srcDir+"/") + relPath = strings.TrimPrefix(relPath, "/") + relPath = strings.TrimSuffix(relPath, filepath.Ext(relPath)) + scriptID := "hook." + strings.ReplaceAll(relPath, "/", ".") + + // Load the script + _, err := v8.Load(file, scriptID) + if err != nil { + t.Logf("Warning: Failed to load hook script %s: %v", base, err) + return nil // Continue loading other scripts + } + + loadedScripts = append(loadedScripts, scriptID) + return nil + }, exts...) + + if err != nil { + t.Fatalf("Failed to walk src directory: %v", err) + } + + return loadedScripts +}