diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md index 08b48215..be1a2b9a 100644 --- a/agent/test/DESIGN_V2.md +++ b/agent/test/DESIGN_V2.md @@ -17,8 +17,8 @@ This document describes the design for Agent Test Framework V2, which extends th | JSONL `simulator.use` | No prefix (agent only) | `"use": "workers.test.user-simulator"` | | `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-simulator` | | `t.assert.Agent()` | No prefix (method-bound) | `t.assert.Agent(resp, "workers.test.validator", {...})` | -| JSONL `before/after` | `scripts:` prefix | `"before": "scripts:tests.env.Before"` | -| `--before/--after` | `scripts:` prefix | `--before scripts:tests.env.BeforeAll` | +| JSONL `before/after` | No prefix (in src/) | `"before": "env_test.Before"` | +| `--before/--after` | No prefix (in src/) | `--before env_test.BeforeAll` | ## Design Goals @@ -294,8 +294,8 @@ For coverage testing where conversation flow is unpredictable: | `input` | string \| Message \| Message[] | Yes | Input: text, single message, or message array | | `assertions` | array | No | Assertions to validate response (alias: `assert`) | | `options` | object | No | `context.Options` passed to agent | -| `before` | string | No | Before script (e.g., `scripts:tests.env.Before`) | -| `after` | string | No | After script (e.g., `scripts:tests.env.After`) | +| `before` | string | No | Before script (e.g., `env_test.Before`) | +| `after` | string | No | After script (e.g., `env_test.After`) | **Note**: The `input` field supports three formats: @@ -329,15 +329,17 @@ JSONL test cases can reference `*_test.ts` scripts for environment preparation: ### Script Location -Scripts are located in the agent's `tests/` directory: +Scripts are located in the agent's `src/` directory (as `*_test.ts` files): ``` assistants/expense/ -├── agent.yml ├── package.yao +├── prompts.yml +├── src/ +│ ├── index.ts # Main agent script +│ └── env_test.ts # Before/after functions └── tests/ ├── inputs.jsonl # Test cases - ├── env_test.ts # Before/after functions └── fixtures/ └── receipt.jpg ``` @@ -345,7 +347,7 @@ assistants/expense/ ### Script Interface ```typescript -// tests/env_test.ts +// src/env_test.ts // Before function - called before test case runs // Returns context data that will be passed to After @@ -402,8 +404,8 @@ export function AfterAll(ctx: Context, results: TestResult[], beforeData: any) { { "id": "T001", "name": "Submit expense with user context", - "before": "scripts:tests.env.Before", - "after": "scripts:tests.env.After", + "before": "env_test.Before", + "after": "env_test.After", "input": "Submit a $500 travel expense", "assertions": [ { @@ -419,8 +421,8 @@ export function AfterAll(ctx: Context, results: TestResult[], beforeData: any) { ```bash # Run with global before/after yao agent test -i ./tests/inputs.jsonl \ - --before scripts:tests.env.BeforeAll \ - --after scripts:tests.env.AfterAll + --before env_test.BeforeAll \ + --after env_test.AfterAll ``` ### Execution Order @@ -639,8 +641,8 @@ options := &context.Options{ | `-c` | `--connector` | Override connector for the target agent | | `-v` | `--verbose` | Verbose output | | | `--simulator` | Default simulator agent ID | -| | `--before` | Global before script (e.g., `scripts:tests.env.BeforeAll`) | -| | `--after` | Global after script (e.g., `scripts:tests.env.AfterAll`) | +| | `--before` | Global before script (e.g., `env_test.BeforeAll`) | +| | `--after` | Global after script (e.g., `env_test.AfterAll`) | | | `--timeout` | Timeout per test case (default: 5m) | | | `--parallel` | Number of parallel test cases | | | `--fail-fast` | Stop on first failure | diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md index 8b66e7ff..aa1fb513 100644 --- a/agent/test/TODO_V2.md +++ b/agent/test/TODO_V2.md @@ -11,21 +11,23 @@ | JSONL `simulator.use` | No prefix (agent only) | `"use": "workers.test.user-simulator"` | | `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-simulator` | | `t.assert.Agent()` | No prefix (method-bound) | `t.assert.Agent(resp, "workers.test.val")` | -| JSONL `before/after` | `scripts:` prefix | `"before": "scripts:tests.env.Before"` | -| `--before/--after` | `scripts:` prefix | `--before scripts:tests.env.BeforeAll` | +| JSONL `before/after` | No prefix (in src/) | `"before": "env_test.Before"` | +| `--before/--after` | No prefix (in src/) | `--before env_test.BeforeAll` | -## Phase 1: Before/After Scripts +## Phase 1: Before/After Scripts ✅ **新增文件**: `script_hooks.go` -- [ ] `types.go`: 添加 `Before`, `After` 字段到 `Case` -- [ ] `types.go`: 添加 `BeforeAll`, `AfterAll` 字段到 `Options` -- [ ] `script_hooks.go`: 实现 `HookExecutor` -- [ ] `script_hooks.go`: 解析 `scripts:` 前缀 -- [ ] `runner.go`: 集成 before/after 到 `runSingleTest` -- [ ] `runner.go`: 集成 beforeAll/afterAll 到 `RunTests` -- [ ] `cmd/agent/agent.go`: 添加 `--before`, `--after` flags -- [ ] 创建示例脚本 `tests/env_test.ts` +- [x] `types.go`: 添加 `Before`, `After` 字段到 `Case` +- [x] `types.go`: 添加 `BeforeAll`, `AfterAll` 字段到 `Options` +- [x] `script_hooks.go`: 实现 `HookExecutor` +- [x] `script_hooks.go`: 通过 V8 直接执行 `*_test.ts` 脚本 +- [x] `runner.go`: 集成 before/after 到 `runSingleTest` +- [x] `runner.go`: 集成 beforeAll/afterAll 到 `RunTests` +- [x] `cmd/agent/test.go`: 添加 `--before`, `--after` flags +- [x] `test/utils.go`: 添加 `LoadAgentTestScripts()` 通用函数 +- [x] 创建示例脚本 `assistants/tests/hooks-test/src/env_test.ts` +- [x] 创建单元测试 `script_hooks_test.go` (黑盒测试) ## Phase 2: Agent-Driven Assertions @@ -79,6 +81,7 @@ - [x] `--fail-fast` flag - [x] `-v` verbose mode - [x] Script testing (`*_test.ts`) +- [x] Before/After hooks (Phase 1) ## Open Questions diff --git a/agent/test/runner.go b/agent/test/runner.go index 32658fdb..be0f2d04 100644 --- a/agent/test/runner.go +++ b/agent/test/runner.go @@ -1,7 +1,6 @@ package test import ( - "bufio" stdContext "context" "fmt" "os" @@ -16,19 +15,22 @@ import ( // Executor executes test cases against an agent type Executor struct { - opts *Options - output *OutputWriter - resolver Resolver - loader Loader + opts *Options + output *OutputWriter + resolver Resolver + loader Loader + hookExecutor *HookExecutor + agentPath string // Path to the agent being tested } // NewRunner creates a new test runner func NewRunner(opts *Options) *Executor { return &Executor{ - opts: opts, - output: NewOutputWriter(opts.Verbose), - resolver: NewResolver(), - loader: NewLoader(), + opts: opts, + output: NewOutputWriter(opts.Verbose), + resolver: NewResolver(), + loader: NewLoader(), + hookExecutor: NewHookExecutor(opts.Verbose), } } @@ -161,6 +163,7 @@ func (r *Executor) RunTests() (*Report, error) { } r.output.Info("Agent: %s", agentInfo.ID) + r.agentPath = agentInfo.Path // Store agent path for hook execution if r.opts.Connector != "" { r.output.Info("Connector: %s (override)", r.opts.Connector) } else if agentInfo.Connector != "" { @@ -222,6 +225,27 @@ func (r *Executor) RunTests() (*Report, error) { }, } + // Execute global BeforeAll if specified + var globalBeforeData interface{} + if r.opts.BeforeAll != "" { + r.output.Info("BeforeAll: %s", r.opts.BeforeAll) + var err error + globalBeforeData, err = r.hookExecutor.ExecuteBeforeAll(r.opts.BeforeAll, activeTests, agentInfo.Path) + if err != nil { + return nil, fmt.Errorf("beforeAll script failed: %w", err) + } + } + + // Ensure AfterAll runs even if tests fail + defer func() { + if r.opts.AfterAll != "" { + r.output.Info("AfterAll: %s", r.opts.AfterAll) + if err := r.hookExecutor.ExecuteAfterAll(r.opts.AfterAll, report.Results, globalBeforeData, agentInfo.Path); err != nil { + r.output.Warning("afterAll script failed: %s", err.Error()) + } + } + }() + // Run tests r.output.SubHeader("Running Tests") @@ -322,6 +346,31 @@ func (r *Executor) runSingleTest(ast *assistant.Assistant, tc *Case, agentID str Options: tc.Options, } + // Execute before script if specified + var beforeData interface{} + if tc.Before != "" { + var err error + beforeData, err = r.hookExecutor.ExecuteBefore(tc.Before, tc, r.agentPath) + if err != nil { + result.Status = StatusError + result.Error = fmt.Sprintf("before script failed: %s", err.Error()) + result.DurationMs = time.Since(startTime).Milliseconds() + r.output.TestResult(result.Status, time.Since(startTime)) + r.output.TestError(result.Error) + // Note: after script is NOT called when before fails + return result + } + } + + // Ensure after script runs even if test fails (but only if before succeeded) + defer func() { + if tc.After != "" && (tc.Before == "" || beforeData != nil || result.Status != StatusError || !isBeforeError(result.Error)) { + if err := r.hookExecutor.ExecuteAfter(tc.After, tc, result, beforeData, r.agentPath); err != nil { + r.output.Warning("after script failed: %s", err.Error()) + } + } + }() + // Parse input to messages with file loading support // BaseDir is derived from the input file directory inputOpts := r.getInputOptions() @@ -395,6 +444,19 @@ func (r *Executor) runSingleTest(ast *assistant.Assistant, tc *Case, agentID str return result } +// isBeforeError checks if the error message indicates a before script failure +func isBeforeError(errMsg string) bool { + return len(errMsg) > 0 && errMsg[:min(len(errMsg), 20)] == "before script failed" +} + +// min returns the minimum of two integers +func min(a, b int) int { + if a < b { + return a + } + return b +} + // runStabilityTests runs each test case multiple times for stability analysis func (r *Executor) runStabilityTests(ast *assistant.Assistant, testCases []*Case, agentID string) []*StabilityResult { results := make([]*StabilityResult, 0, len(testCases)) @@ -499,20 +561,6 @@ func (r *Executor) writeOutput(report *Report) error { return reporter.Write(report, file) } -// writeJSONLine writes a JSON line to the writer -func writeJSONLine(writer *bufio.Writer, data interface{}) error { - line, err := jsoniter.Marshal(data) - if err != nil { - return err - } - _, err = writer.Write(line) - if err != nil { - return err - } - _, err = writer.WriteString("\n") - return err -} - // buildContextOptions builds context.Options from test case and runner options // Priority: test case options > runner options > defaults func buildContextOptions(tc *Case, runnerOpts *Options) *context.Options { diff --git a/agent/test/script_hooks.go b/agent/test/script_hooks.go new file mode 100644 index 00000000..ea9349e1 --- /dev/null +++ b/agent/test/script_hooks.go @@ -0,0 +1,592 @@ +package test + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/yaoapp/gou/application" + v8 "github.com/yaoapp/gou/runtime/v8" + "github.com/yaoapp/gou/runtime/v8/bridge" + "github.com/yaoapp/yao/agent/context" + "rogchap.com/v8go" +) + +// HookExecutor executes before/after scripts from *_test.ts files +// Scripts are loaded via V8 and executed directly, not via Process() +type HookExecutor struct { + verbose bool + output *OutputWriter + loadedDirs map[string]bool // Track which directories have been loaded + agentContext *context.Context +} + +// NewHookExecutor creates a new hook executor +func NewHookExecutor(verbose bool) *HookExecutor { + return &HookExecutor{ + verbose: verbose, + output: NewOutputWriter(verbose), + loadedDirs: make(map[string]bool), + } +} + +// SetAgentContext sets the agent context for script execution +func (h *HookExecutor) SetAgentContext(ctx *context.Context) { + h.agentContext = ctx +} + +// HookRef represents a parsed hook reference +// Format: "src/env_test.ts:Before" or just "Before" (uses default test file) +type HookRef struct { + ScriptFile string // e.g., "env_test.ts" + Function string // e.g., "Before" +} + +// ParseHookRef parses a hook reference string +// Formats: +// - "Before" -> uses first *_test.ts file found +// - "env_test.Before" -> uses src/env_test.ts +// - "src/env_test.Before" -> uses src/env_test.ts +func ParseHookRef(ref string) (*HookRef, error) { + if ref == "" { + return nil, fmt.Errorf("empty hook reference") + } + + // Split by last dot to get function name + lastDot := strings.LastIndex(ref, ".") + if lastDot == -1 { + // Just function name, will use default test file + return &HookRef{ + ScriptFile: "", // Will be resolved later + Function: ref, + }, nil + } + + scriptPart := ref[:lastDot] + funcName := ref[lastDot+1:] + + // Normalize script file name + scriptFile := scriptPart + if !strings.HasSuffix(scriptFile, "_test") { + scriptFile += "_test" + } + scriptFile += ".ts" + + // Remove "src/" prefix if present + scriptFile = strings.TrimPrefix(scriptFile, "src/") + + return &HookRef{ + ScriptFile: scriptFile, + Function: funcName, + }, nil +} + +// LoadTestScripts loads all *_test.ts scripts from the agent's src directory +// Returns the script IDs that were loaded +func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) { + srcDir := filepath.Join(agentPath, "src") + + // Check if already loaded + if h.loadedDirs[srcDir] { + return nil, nil + } + + // Check if src directory exists + exists, err := application.App.Exists(srcDir) + if err != nil { + return nil, err + } + if !exists { + return nil, nil // No src directory, not an error + } + + var loadedScripts []string + exts := []string{"*_test.ts", "*_test.js"} + + err = application.App.Walk(srcDir, func(root, file string, isdir bool) error { + if isdir { + return nil + } + + // Only load *_test.ts/js files + base := filepath.Base(file) + if !strings.HasSuffix(base, "_test.ts") && !strings.HasSuffix(base, "_test.js") { + return nil + } + + // Generate script ID + scriptID := generateHookScriptID(file, srcDir) + + // Load the script + _, err := v8.Load(file, scriptID) + if err != nil { + if h.verbose { + h.output.Warning("Failed to load hook script %s: %v", base, err) + } + return nil // Continue loading other scripts + } + + loadedScripts = append(loadedScripts, scriptID) + if h.verbose { + h.output.Verbose("Loaded hook script: %s (id: %s)", base, scriptID) + } + + return nil + }, exts...) + + if err != nil { + return nil, fmt.Errorf("failed to walk src directory: %w", err) + } + + h.loadedDirs[srcDir] = true + return loadedScripts, nil +} + +// generateHookScriptID generates a script ID for hook scripts +// Example: assistants/test/src/env_test.ts -> hook.env_test +func generateHookScriptID(filePath string, srcDir string) string { + filePath = filepath.ToSlash(filePath) + srcDir = filepath.ToSlash(srcDir) + + relPath := strings.TrimPrefix(filePath, srcDir+"/") + relPath = strings.TrimPrefix(relPath, "/") + relPath = strings.TrimSuffix(relPath, filepath.Ext(relPath)) + + return "hook." + strings.ReplaceAll(relPath, "/", ".") +} + +// FindTestScript finds a loaded test script by pattern +// If scriptFile is empty, returns the first *_test script found +func (h *HookExecutor) FindTestScript(scriptFile string) (*v8.Script, string, error) { + if scriptFile != "" { + // Look for specific script + scriptID := "hook." + strings.TrimSuffix(scriptFile, ".ts") + scriptID = strings.TrimSuffix(scriptID, ".js") + + if script, ok := v8.Scripts[scriptID]; ok { + return script, scriptID, nil + } + return nil, "", fmt.Errorf("hook script not found: %s (id: %s)", scriptFile, scriptID) + } + + // Find first *_test script + for id, script := range v8.Scripts { + if strings.HasPrefix(id, "hook.") && strings.Contains(id, "_test") { + return script, id, nil + } + } + + return nil, "", fmt.Errorf("no hook test script found") +} + +// ExecuteBefore executes a Before function from a test script +func (h *HookExecutor) ExecuteBefore(ref string, testCase *Case, agentPath string) (interface{}, error) { + hookRef, err := ParseHookRef(ref) + if err != nil { + return nil, err + } + + // Ensure scripts are loaded + if _, err := h.LoadTestScripts(agentPath); err != nil { + return nil, fmt.Errorf("failed to load test scripts: %w", err) + } + + // Find the script + script, scriptID, err := h.FindTestScript(hookRef.ScriptFile) + if err != nil { + return nil, err + } + + if h.verbose { + h.output.Verbose("Executing %s from %s", hookRef.Function, scriptID) + } + + // Execute the function + return h.executeHookFunction(script, hookRef.Function, testCase, nil, nil) +} + +// ExecuteAfter executes an After function from a test script +func (h *HookExecutor) ExecuteAfter(ref string, testCase *Case, result *Result, beforeData interface{}, agentPath string) error { + hookRef, err := ParseHookRef(ref) + if err != nil { + return err + } + + // Ensure scripts are loaded + if _, err := h.LoadTestScripts(agentPath); err != nil { + return fmt.Errorf("failed to load test scripts: %w", err) + } + + // Find the script + script, scriptID, err := h.FindTestScript(hookRef.ScriptFile) + if err != nil { + return err + } + + if h.verbose { + h.output.Verbose("Executing %s from %s", hookRef.Function, scriptID) + } + + // Execute the function + _, err = h.executeHookFunction(script, hookRef.Function, testCase, result, beforeData) + return err +} + +// ExecuteBeforeAll executes a BeforeAll function +func (h *HookExecutor) ExecuteBeforeAll(ref string, testCases []*Case, agentPath string) (interface{}, error) { + hookRef, err := ParseHookRef(ref) + if err != nil { + return nil, err + } + + // Ensure scripts are loaded + if _, err := h.LoadTestScripts(agentPath); err != nil { + return nil, fmt.Errorf("failed to load test scripts: %w", err) + } + + // Find the script + script, scriptID, err := h.FindTestScript(hookRef.ScriptFile) + if err != nil { + return nil, err + } + + if h.verbose { + h.output.Verbose("Executing %s from %s", hookRef.Function, scriptID) + } + + // Execute with test cases array + return h.executeHookFunctionWithCases(script, hookRef.Function, testCases) +} + +// ExecuteAfterAll executes an AfterAll function +func (h *HookExecutor) ExecuteAfterAll(ref string, results []*Result, beforeData interface{}, agentPath string) error { + hookRef, err := ParseHookRef(ref) + if err != nil { + return err + } + + // Ensure scripts are loaded + if _, err := h.LoadTestScripts(agentPath); err != nil { + return fmt.Errorf("failed to load test scripts: %w", err) + } + + // Find the script + script, scriptID, err := h.FindTestScript(hookRef.ScriptFile) + if err != nil { + return err + } + + if h.verbose { + h.output.Verbose("Executing %s from %s", hookRef.Function, scriptID) + } + + // Execute with results array + _, err = h.executeHookFunctionWithResults(script, hookRef.Function, results, beforeData) + return err +} + +// executeHookFunction executes a hook function with test case context +func (h *HookExecutor) executeHookFunction(script *v8.Script, funcName string, testCase *Case, result *Result, beforeData interface{}) (interface{}, error) { + // Create script context + scriptCtx, err := script.NewContext("", nil) + if err != nil { + return nil, fmt.Errorf("failed to create script context: %w", err) + } + defer scriptCtx.Close() + + v8ctx := scriptCtx.Context + + // Set share data + if err := h.setShareData(v8ctx); err != nil { + return nil, err + } + + // Get the function + global := v8ctx.Global() + fnValue, err := global.Get(funcName) + if err != nil { + return nil, fmt.Errorf("failed to get function %s: %w", funcName, err) + } + + if fnValue.IsUndefined() || fnValue.IsNull() { + return nil, fmt.Errorf("function %s not defined", funcName) + } + + if !fnValue.IsFunction() { + return nil, fmt.Errorf("%s is not a function", funcName) + } + + fn, err := fnValue.AsFunction() + if err != nil { + return nil, fmt.Errorf("failed to convert to function: %w", err) + } + + // Build arguments + args, err := h.buildHookArgs(v8ctx, testCase, result, beforeData) + if err != nil { + return nil, err + } + + // Convert to v8go.Valuer slice for Call + valuerArgs := make([]v8go.Valuer, len(args)) + for i, arg := range args { + valuerArgs[i] = arg + } + + // Call the function + jsResult, err := fn.Call(global, valuerArgs...) + if err != nil { + return nil, fmt.Errorf("hook function %s failed: %w", funcName, err) + } + + // Convert result to Go value + if jsResult == nil || jsResult.IsUndefined() || jsResult.IsNull() { + return nil, nil + } + + goResult, err := bridge.GoValue(jsResult, v8ctx) + if err != nil { + return nil, fmt.Errorf("failed to convert result: %w", err) + } + + // Extract data field if present + if resultMap, ok := goResult.(map[string]interface{}); ok { + if data, exists := resultMap["data"]; exists { + return data, nil + } + } + + return goResult, nil +} + +// executeHookFunctionWithCases executes BeforeAll with test cases array +func (h *HookExecutor) executeHookFunctionWithCases(script *v8.Script, funcName string, testCases []*Case) (interface{}, error) { + scriptCtx, err := script.NewContext("", nil) + if err != nil { + return nil, fmt.Errorf("failed to create script context: %w", err) + } + defer scriptCtx.Close() + + v8ctx := scriptCtx.Context + + if err := h.setShareData(v8ctx); err != nil { + return nil, err + } + + global := v8ctx.Global() + fnValue, err := global.Get(funcName) + if err != nil { + return nil, fmt.Errorf("failed to get function %s: %w", funcName, err) + } + + if fnValue.IsUndefined() || fnValue.IsNull() { + return nil, fmt.Errorf("function %s not defined", funcName) + } + + if !fnValue.IsFunction() { + return nil, fmt.Errorf("%s is not a function", funcName) + } + + fn, err := fnValue.AsFunction() + if err != nil { + return nil, fmt.Errorf("failed to convert to function: %w", err) + } + + // Convert test cases to JS array + casesJS, err := h.testCasesToJS(v8ctx, testCases) + if err != nil { + return nil, err + } + + jsResult, err := fn.Call(global, casesJS) + if err != nil { + return nil, fmt.Errorf("hook function %s failed: %w", funcName, err) + } + + if jsResult == nil || jsResult.IsUndefined() || jsResult.IsNull() { + return nil, nil + } + + goResult, err := bridge.GoValue(jsResult, v8ctx) + if err != nil { + return nil, fmt.Errorf("failed to convert result: %w", err) + } + + if resultMap, ok := goResult.(map[string]interface{}); ok { + if data, exists := resultMap["data"]; exists { + return data, nil + } + } + + return goResult, nil +} + +// executeHookFunctionWithResults executes AfterAll with results array +func (h *HookExecutor) executeHookFunctionWithResults(script *v8.Script, funcName string, results []*Result, beforeData interface{}) (interface{}, error) { + scriptCtx, err := script.NewContext("", nil) + if err != nil { + return nil, fmt.Errorf("failed to create script context: %w", err) + } + defer scriptCtx.Close() + + v8ctx := scriptCtx.Context + + if err := h.setShareData(v8ctx); err != nil { + return nil, err + } + + global := v8ctx.Global() + fnValue, err := global.Get(funcName) + if err != nil { + return nil, fmt.Errorf("failed to get function %s: %w", funcName, err) + } + + if fnValue.IsUndefined() || fnValue.IsNull() { + return nil, fmt.Errorf("function %s not defined", funcName) + } + + if !fnValue.IsFunction() { + return nil, fmt.Errorf("%s is not a function", funcName) + } + + fn, err := fnValue.AsFunction() + if err != nil { + return nil, fmt.Errorf("failed to convert to function: %w", err) + } + + // Convert results to JS array + resultsJS, err := h.resultsToJS(v8ctx, results) + if err != nil { + return nil, err + } + + // Convert beforeData to JS + beforeDataJS, err := bridge.JsValue(v8ctx, beforeData) + if err != nil { + return nil, fmt.Errorf("failed to convert beforeData: %w", err) + } + + jsResult, err := fn.Call(global, resultsJS, beforeDataJS) + if err != nil { + return nil, fmt.Errorf("hook function %s failed: %w", funcName, err) + } + + if jsResult == nil || jsResult.IsUndefined() || jsResult.IsNull() { + return nil, nil + } + + goResult, err := bridge.GoValue(jsResult, v8ctx) + if err != nil { + return nil, fmt.Errorf("failed to convert result: %w", err) + } + + return goResult, nil +} + +// setShareData sets the share data for script execution +func (h *HookExecutor) setShareData(v8ctx *v8go.Context) error { + var authorized map[string]interface{} + if h.agentContext != nil && h.agentContext.Authorized != nil { + authorized = h.agentContext.Authorized.AuthorizedToMap() + } + + return bridge.SetShareData(v8ctx, v8ctx.Global(), &bridge.Share{ + Sid: "", + Root: false, + Global: nil, + Authorized: authorized, + }) +} + +// buildHookArgs builds the arguments for a hook function call +func (h *HookExecutor) buildHookArgs(v8ctx *v8go.Context, testCase *Case, result *Result, beforeData interface{}) ([]*v8go.Value, error) { + var args []*v8go.Value + + // Arg 1: testCase + if testCase != nil { + tcMap := map[string]interface{}{ + "id": testCase.ID, + "input": testCase.Input, + } + if testCase.Metadata != nil { + tcMap["metadata"] = testCase.Metadata + } + if testCase.Assert != nil { + tcMap["assert"] = testCase.Assert + } + + tcJS, err := bridge.JsValue(v8ctx, tcMap) + if err != nil { + return nil, fmt.Errorf("failed to convert testCase: %w", err) + } + args = append(args, tcJS) + } + + // Arg 2: result (for After) + if result != nil { + resultMap := map[string]interface{}{ + "id": result.ID, + "status": string(result.Status), + "duration_ms": result.DurationMs, + } + if result.Output != nil { + resultMap["output"] = result.Output + } + if result.Error != "" { + resultMap["error"] = result.Error + } + + resultJS, err := bridge.JsValue(v8ctx, resultMap) + if err != nil { + return nil, fmt.Errorf("failed to convert result: %w", err) + } + args = append(args, resultJS) + } + + // Arg 3: beforeData (for After) + if beforeData != nil { + beforeDataJS, err := bridge.JsValue(v8ctx, beforeData) + if err != nil { + return nil, fmt.Errorf("failed to convert beforeData: %w", err) + } + args = append(args, beforeDataJS) + } + + return args, nil +} + +// testCasesToJS converts test cases to a JS array +func (h *HookExecutor) testCasesToJS(v8ctx *v8go.Context, testCases []*Case) (*v8go.Value, error) { + cases := make([]map[string]interface{}, len(testCases)) + for i, tc := range testCases { + cases[i] = map[string]interface{}{ + "id": tc.ID, + "input": tc.Input, + } + if tc.Metadata != nil { + cases[i]["metadata"] = tc.Metadata + } + } + + return bridge.JsValue(v8ctx, cases) +} + +// resultsToJS converts results to a JS array +func (h *HookExecutor) resultsToJS(v8ctx *v8go.Context, results []*Result) (*v8go.Value, error) { + resultMaps := make([]map[string]interface{}, len(results)) + for i, r := range results { + resultMaps[i] = map[string]interface{}{ + "id": r.ID, + "status": string(r.Status), + "duration_ms": r.DurationMs, + } + if r.Output != nil { + resultMaps[i]["output"] = r.Output + } + if r.Error != "" { + resultMaps[i]["error"] = r.Error + } + } + + return bridge.JsValue(v8ctx, resultMaps) +} diff --git a/agent/test/script_hooks_test.go b/agent/test/script_hooks_test.go new file mode 100644 index 00000000..8720c463 --- /dev/null +++ b/agent/test/script_hooks_test.go @@ -0,0 +1,244 @@ +package test_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v8 "github.com/yaoapp/gou/runtime/v8" + agenttest "github.com/yaoapp/yao/agent/test" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +const hooksTestAgent = "assistants/tests/hooks-test" + +func TestParseHookRef(t *testing.T) { + tests := []struct { + name string + input string + wantFile string + wantFunc string + expectErr bool + }{ + { + name: "function only", + input: "Before", + wantFile: "", + wantFunc: "Before", + }, + { + name: "with script file", + input: "env_test.Before", + wantFile: "env_test.ts", + wantFunc: "Before", + }, + { + name: "with src prefix", + input: "src/env_test.Before", + wantFile: "env_test.ts", + wantFunc: "Before", + }, + { + name: "nested path", + input: "setup/db_test.Before", + wantFile: "setup/db_test.ts", + wantFunc: "Before", + }, + { + name: "empty string", + input: "", + expectErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ref, err := agenttest.ParseHookRef(tt.input) + if tt.expectErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.wantFile, ref.ScriptFile) + assert.Equal(t, tt.wantFunc, ref.Function) + }) + } +} + +func TestHookExecutorLoadTestScripts(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts using the utility function + scripts := test.LoadAgentTestScripts(t, hooksTestAgent) + + assert.NotEmpty(t, scripts, "Should load at least one test script") + + // Verify the script was loaded into V8 + found := false + for _, scriptID := range scripts { + if _, ok := v8.Scripts[scriptID]; ok { + found = true + t.Logf("Loaded script: %s", scriptID) + break + } + } + assert.True(t, found, "At least one script should be loaded into V8") +} + +func TestHookExecutorExecuteBefore(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts + test.LoadAgentTestScripts(t, hooksTestAgent) + + executor := agenttest.NewHookExecutor(true) + + testCase := &agenttest.Case{ + ID: "TEST001", + Input: "Hello World", + } + + // Execute Before hook + beforeData, err := executor.ExecuteBefore("env_test.Before", testCase, hooksTestAgent) + assert.NoError(t, err) + assert.NotNil(t, beforeData) + + // Verify returned data + dataMap, ok := beforeData.(map[string]interface{}) + assert.True(t, ok, "beforeData should be a map") + assert.Equal(t, "TEST001", dataMap["test_id"]) + assert.NotEmpty(t, dataMap["mock_user_id"]) + assert.NotEmpty(t, dataMap["mock_session_id"]) +} + +func TestHookExecutorExecuteAfter(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts + test.LoadAgentTestScripts(t, hooksTestAgent) + + executor := agenttest.NewHookExecutor(true) + + testCase := &agenttest.Case{ + ID: "TEST002", + Input: "Test input", + } + + result := &agenttest.Result{ + ID: "TEST002", + Status: agenttest.StatusPassed, + DurationMs: 100, + } + + beforeData := map[string]interface{}{ + "test_id": "TEST002", + "mock_user_id": "user_TEST002_12345", + "mock_session_id": "session_12345", + } + + // Execute After hook + err := executor.ExecuteAfter("env_test.After", testCase, result, beforeData, hooksTestAgent) + assert.NoError(t, err) +} + +func TestHookExecutorExecuteBeforeAll(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts + test.LoadAgentTestScripts(t, hooksTestAgent) + + executor := agenttest.NewHookExecutor(true) + + testCases := []*agenttest.Case{ + {ID: "T001", Input: "Test 1"}, + {ID: "T002", Input: "Test 2"}, + {ID: "T003", Input: "Test 3"}, + } + + // Execute BeforeAll hook + globalData, err := executor.ExecuteBeforeAll("env_test.BeforeAll", testCases, hooksTestAgent) + assert.NoError(t, err) + assert.NotNil(t, globalData) + + // Verify returned data + dataMap, ok := globalData.(map[string]interface{}) + assert.True(t, ok, "globalData should be a map") + assert.NotEmpty(t, dataMap["suite_id"]) + assert.Equal(t, float64(3), dataMap["test_count"]) // JSON numbers are float64 +} + +func TestHookExecutorExecuteAfterAll(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts + test.LoadAgentTestScripts(t, hooksTestAgent) + + executor := agenttest.NewHookExecutor(true) + + results := []*agenttest.Result{ + {ID: "T001", Status: agenttest.StatusPassed, DurationMs: 100}, + {ID: "T002", Status: agenttest.StatusFailed, DurationMs: 200, Error: "assertion failed"}, + {ID: "T003", Status: agenttest.StatusPassed, DurationMs: 150}, + } + + globalData := map[string]interface{}{ + "suite_id": "suite_12345", + "test_count": 3, + } + + // Execute AfterAll hook + err := executor.ExecuteAfterAll("env_test.AfterAll", results, globalData, hooksTestAgent) + assert.NoError(t, err) +} + +func TestHookExecutorFunctionNotFound(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts + test.LoadAgentTestScripts(t, hooksTestAgent) + + executor := agenttest.NewHookExecutor(true) + + testCase := &agenttest.Case{ + ID: "TEST001", + Input: "Hello", + } + + // Try to execute non-existent function + _, err := executor.ExecuteBefore("env_test.NonExistent", testCase, hooksTestAgent) + assert.Error(t, err) + assert.Contains(t, err.Error(), "not defined") +} + +func TestHookExecutorScriptNotFound(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts + test.LoadAgentTestScripts(t, hooksTestAgent) + + executor := agenttest.NewHookExecutor(true) + + testCase := &agenttest.Case{ + ID: "TEST001", + Input: "Hello", + } + + // Try to execute from non-existent script + _, err := executor.ExecuteBefore("nonexistent_test.Before", testCase, hooksTestAgent) + assert.Error(t, err) + assert.Contains(t, err.Error(), "not found") +} diff --git a/agent/test/types.go b/agent/test/types.go index b40ab457..f74a0307 100644 --- a/agent/test/types.go +++ b/agent/test/types.go @@ -144,6 +144,14 @@ type Options struct { // Only tests matching the pattern will be executed // Example: "TestSystem" matches TestSystemReady, TestSystemError, etc. Run string `json:"run,omitempty"` + + // BeforeAll is the global before script (e.g., "scripts:tests.env.BeforeAll") + // Called once before all test cases + BeforeAll string `json:"before_all,omitempty"` + + // AfterAll is the global after script (e.g., "scripts:tests.env.AfterAll") + // Called once after all test cases + AfterAll string `json:"after_all,omitempty"` } // ContextConfig represents custom context configuration from JSON file @@ -375,6 +383,14 @@ type Case struct { // Timeout overrides the default timeout for this test case // Format: "30s", "1m", "2m30s" Timeout string `json:"timeout,omitempty"` + + // Before script function (e.g., "scripts:tests.env.Before") + // Called before the test case runs, returns data passed to After + Before string `json:"before,omitempty"` + + // After script function (e.g., "scripts:tests.env.After") + // Called after the test case completes (pass or fail) + After string `json:"after,omitempty"` } // CaseOptions represents per-test-case context options diff --git a/cmd/agent/test.go b/cmd/agent/test.go index 7af0ef03..f48384f0 100644 --- a/cmd/agent/test.go +++ b/cmd/agent/test.go @@ -33,6 +33,8 @@ var ( testParallel int testVerbose bool testFailFast bool + testBefore string // --before flag for global BeforeAll hook + testAfter string // --after flag for global AfterAll hook ) // TestCmd is the agent test command @@ -157,6 +159,8 @@ var TestCmd = &cobra.Command{ Parallel: testParallel, Verbose: testVerbose, FailFast: testFailFast, + BeforeAll: testBefore, + AfterAll: testAfter, } // Merge with defaults @@ -244,6 +248,8 @@ func init() { TestCmd.Flags().IntVar(&testParallel, "parallel", 1, L("Number of parallel test cases")) TestCmd.Flags().BoolVarP(&testVerbose, "verbose", "v", false, L("Verbose output")) TestCmd.Flags().BoolVar(&testFailFast, "fail-fast", false, L("Stop on first failure")) + TestCmd.Flags().StringVar(&testBefore, "before", "", L("Global BeforeAll hook (e.g., env_test.BeforeAll)")) + TestCmd.Flags().StringVar(&testAfter, "after", "", L("Global AfterAll hook (e.g., env_test.AfterAll)")) // Mark input as required TestCmd.MarkFlagRequired("input") diff --git a/test/utils.go b/test/utils.go index 624bdb59..b7d06d5b 100644 --- a/test/utils.go +++ b/test/utils.go @@ -800,3 +800,70 @@ func GuardBearerJWT(c *gin.Context) { claims := helper.JwtValidate(tokenString) c.Set("__sid", claims.SID) } + +// LoadAgentTestScripts loads all *_test.ts/js scripts from an agent's src directory. +// This is useful for testing agent hooks (before/after scripts) and other agent-specific test scripts. +// +// Usage: +// +// test.Prepare(t, config.Conf) +// defer test.Clean() +// scripts := test.LoadAgentTestScripts(t, "assistants/tests/hooks-test") +// +// Parameters: +// - t: testing.T instance +// - agentRelPath: relative path to agent directory from app root (e.g., "assistants/tests/hooks-test") +// +// Returns: +// - []string: list of loaded script IDs (e.g., ["hook.env_test"]) +func LoadAgentTestScripts(t *testing.T, agentRelPath string) []string { + srcDir := filepath.Join(agentRelPath, "src") + + // Check if src directory exists + exists, err := application.App.Exists(srcDir) + if err != nil { + t.Fatalf("Failed to check src directory: %v", err) + } + if !exists { + t.Logf("No src directory found at %s, skipping", srcDir) + return nil + } + + var loadedScripts []string + exts := []string{"*_test.ts", "*_test.js"} + + err = application.App.Walk(srcDir, func(root, file string, isdir bool) error { + if isdir { + return nil + } + + // Only load *_test.ts/js files + base := filepath.Base(file) + if !strings.HasSuffix(base, "_test.ts") && !strings.HasSuffix(base, "_test.js") { + return nil + } + + // Generate script ID: hook.{relative_path_without_ext} + // e.g., assistants/tests/hooks-test/src/env_test.ts -> hook.env_test + relPath := strings.TrimPrefix(file, srcDir+"/") + relPath = strings.TrimPrefix(relPath, "/") + relPath = strings.TrimSuffix(relPath, filepath.Ext(relPath)) + scriptID := "hook." + strings.ReplaceAll(relPath, "/", ".") + + // Load the script + _, err := v8.Load(file, scriptID) + if err != nil { + t.Logf("Warning: Failed to load hook script %s: %v", base, err) + return nil // Continue loading other scripts + } + + loadedScripts = append(loadedScripts, scriptID) + return nil + }, exts...) + + if err != nil { + t.Fatalf("Failed to walk src directory: %v", err) + } + + return loadedScripts +}