From 5fb5035d8c9379d44e3bbf9b04be2a9d9debe6c7 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 27 Dec 2025 12:58:09 +0800 Subject: [PATCH 1/3] Enhance Agent Response Structure and Dynamic Test Functionality - Added support for tool call responses in the agent's response structure, allowing for better handling of tool execution results. - Updated the `TurnResult` and `TurnResponse` types to include full agent responses, including tool call details and next hook data. - Improved dynamic test execution by ensuring consistent chat session state across turns, enhancing the overall testing framework's capabilities. - Enhanced documentation in README.md to reflect changes in response structure and dynamic testing output format. --- agent/assistant/next.go | 2 + agent/context/types.go | 1 + agent/test/README.md | 61 +++++++++++++- agent/test/dynamic_runner.go | 145 +++++++++++++++++++++++++++++--- agent/test/dynamic_types.go | 29 ++++++- agent/test/loader.go | 8 ++ agent/test/runner.go | 110 +++++++++++++++++++++--- agent/test/script_hooks.go | 158 ++++++++++++++++++++++++++++++++--- agent/test/types.go | 14 +++- 9 files changed, 487 insertions(+), 41 deletions(-) diff --git a/agent/assistant/next.go b/agent/assistant/next.go index fc6c7c7a..accacaab 100644 --- a/agent/assistant/next.go +++ b/agent/assistant/next.go @@ -30,6 +30,7 @@ func (ast *Assistant) processNextResponse(npc *NextProcessContext) (*agentContex Create: npc.CreateResponse, Next: npc.NextResponse.Data, // Put custom data in Next field Completion: npc.CompletionResponse, + Tools: npc.ToolCallResponses, }, nil } @@ -72,5 +73,6 @@ func (ast *Assistant) buildStandardResponse(npc *NextProcessContext) *agentConte Create: npc.CreateResponse, Next: npc.NextResponse, Completion: npc.CompletionResponse, + Tools: npc.ToolCallResponses, } } diff --git a/agent/context/types.go b/agent/context/types.go index d567a72f..f5cbb372 100644 --- a/agent/context/types.go +++ b/agent/context/types.go @@ -349,6 +349,7 @@ type Response struct { Create *HookCreateResponse `json:"create,omitempty"` // Create response from the create hook Next interface{} `json:"next,omitempty"` // Next response from the next hook Completion *CompletionResponse `json:"completion,omitempty"` // Completion response from the completion hook + Tools []ToolCallResponse `json:"tools,omitempty"` // Tool call results (if any tools were executed) } // HookCreateResponse the response of the create hook diff --git a/agent/test/README.md b/agent/test/README.md index c476c4a4..2135f018 100644 --- a/agent/test/README.md +++ b/agent/test/README.md @@ -203,6 +203,33 @@ Simulator-driven testing with checkpoint validation. A simulator agent generates | `--fail-fast` | Stop on first failure | false | | `--dry-run` | Generate test cases without running them | false | +## Custom Context File + +Create a JSON file for custom authorization: + +```json +{ + "chat_id": "test-chat-001", + "authorized": { + "user_id": "test-user-123", + "team_id": "test-team-456", + "constraints": { + "owner_only": true, + "extra": { "department": "engineering" } + } + }, + "metadata": { + "mode": "test" + } +} +``` + +Use with `--ctx`: + +```bash +yao agent test -i scripts.expense.setup --ctx tests/context.json -v +``` + ## Input Format (JSONL) Each line is a JSON object. Below are examples organized by scenario. @@ -761,9 +788,12 @@ export function AfterAll(ctx: Context, results: TestResult[], beforeData: any) { ```typescript interface Context { - user_id: string; // Test user ID - team_id: string; // Test team ID locale: string; // Locale (e.g., "en-us") + authorized: { + user_id: string; // Test user ID + team_id: string; // Test team ID + constraints?: object; // Access constraints + }; metadata: object; // Custom metadata from test case } ``` @@ -980,6 +1010,33 @@ For testing complex conversation flows where the path is unpredictable: └─ PASSED (3 turns, 3 checkpoints, 8.5s) ``` +### Dynamic Mode Output Structure + +Each turn in the output includes: + +```typescript +interface TurnResult { + turn: number; // Turn number (1-based) + input: string; // User message + output: any; // Agent response summary (for display) + response: { + // Full agent response (for detailed analysis) + content: string; // LLM text content + tool_calls: [ + { + // Tool calls made + tool: string; // Tool name + arguments: any; // Call arguments + result: any; // Execution result + } + ]; + next: any; // Next hook data + }; + checkpoints_reached: string[]; // Checkpoint IDs reached + duration_ms: number; // Execution time +} +``` + ## Output Formats Determined by `-o` file extension: diff --git a/agent/test/dynamic_runner.go b/agent/test/dynamic_runner.go index 2b6020df..6530c7ad 100644 --- a/agent/test/dynamic_runner.go +++ b/agent/test/dynamic_runner.go @@ -77,7 +77,14 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s // Output dynamic test start if r.opts.Verbose { - r.output.Info("Dynamic test: %s (max %d turns)", tc.ID, maxTurns) + r.output.Verbose("Dynamic test: %s (max %d turns)", tc.ID, maxTurns) + } + + // Use consistent chatID across all turns to preserve session state (ctx.memory.chat) + // Priority: context config > generated ID + chatID := fmt.Sprintf("dynamic-%s", tc.ID) + if r.opts.ContextData != nil && r.opts.ContextData.ChatID != "" { + chatID = r.opts.ContextData.ChatID } // Conversation loop @@ -111,7 +118,7 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s // Check if goal achieved if simOutput.GoalAchieved { if r.opts.Verbose { - r.output.Info(" Turn %d: Simulator signaled goal achieved", turn) + r.output.Verbose("Turn %d: Simulator signaled goal achieved", turn) } // Check if all required checkpoints reached @@ -135,7 +142,7 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s turnResult.Input = simOutput.Message if r.opts.Verbose { - r.output.Info(" Turn %d: User: %s", turn, truncateOutput(simOutput.Message, 50)) + r.output.Verbose("Turn %d: User: %s", turn, truncateOutput(simOutput.Message, 50)) } } else { // Use initial input for first turn @@ -143,14 +150,15 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s lastMsg := messages[len(messages)-1] turnResult.Input = lastMsg.Content if r.opts.Verbose { - r.output.Info(" Turn %d: User: %s", turn, truncateOutput(lastMsg.Content, 50)) + r.output.Verbose("Turn %d: User: %s", turn, truncateOutput(lastMsg.Content, 50)) } } } // Call target agent + // Use consistent chatID across all turns to preserve session state (ctx.memory.chat) ctx := NewTestContextFromOptions( - fmt.Sprintf("dynamic-%s-%d", tc.ID, turn), + chatID, agentID, r.opts, tc, @@ -171,20 +179,20 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s return result } - // Extract output + // Extract output (summary for display and conversation history) output := extractOutput(response) turnResult.Output = output turnResult.DurationMs = time.Since(turnStart).Milliseconds() + // Store full response for reporting + turnResult.Response = buildTurnResponse(response) + if r.opts.Verbose { - r.output.Info(" Turn %d: Agent: %s", turn, truncateOutput(output, 50)) + r.output.Verbose("Turn %d: Agent: %s", turn, truncateOutput(output, 50)) } - // Add assistant response to messages - messages = append(messages, context.Message{ - Role: context.RoleAssistant, - Content: output, - }) + // Add assistant response to messages, including tool calls if any + messages = appendAssistantMessages(messages, response) // Check checkpoints against this response reachedIDs := r.checkCheckpoints(tc.Checkpoints, output, result) @@ -192,7 +200,7 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s if r.opts.Verbose && len(reachedIDs) > 0 { for _, id := range reachedIDs { - r.output.Info(" ✓ checkpoint: %s", id) + r.output.Verbose(" ✓ checkpoint: %s", id) } } @@ -411,3 +419,114 @@ func (r *DynamicRunner) allRequiredCheckpointsReached(result *DynamicResult) boo } return true } + +// buildTurnResponse builds a TurnResponse from the agent response +func buildTurnResponse(response *context.Response) *TurnResponse { + if response == nil { + return nil + } + + tr := &TurnResponse{} + + // Extract completion content + if response.Completion != nil { + tr.Content = response.Completion.Content + + // Extract tool calls from completion + if len(response.Completion.ToolCalls) > 0 { + for _, tc := range response.Completion.ToolCalls { + tr.ToolCalls = append(tr.ToolCalls, ToolCallInfo{ + Tool: tc.Function.Name, + Arguments: tc.Function.Arguments, + }) + } + } + } + + // Add tool results + if len(response.Tools) > 0 { + // If we already have tool calls from completion, match results + if len(tr.ToolCalls) > 0 { + for i, toolResult := range response.Tools { + if i < len(tr.ToolCalls) { + tr.ToolCalls[i].Result = toolResult.Result + } + } + } else { + // Create tool call entries from results + for _, toolResult := range response.Tools { + tr.ToolCalls = append(tr.ToolCalls, ToolCallInfo{ + Tool: toolResult.Tool, + Arguments: toolResult.Arguments, + Result: toolResult.Result, + }) + } + } + } + + // Extract Next hook data + if response.Next != nil && !isEmptyValue(response.Next) { + tr.Next = response.Next + } + + return tr +} + +// appendAssistantMessages appends assistant messages to the conversation history +// including tool calls and tool results if present +func appendAssistantMessages(messages []context.Message, response *context.Response) []context.Message { + if response == nil { + return messages + } + + // Check if there are tool calls in the completion + hasToolCalls := response.Completion != nil && len(response.Completion.ToolCalls) > 0 + + if hasToolCalls { + // Add assistant message with tool calls + assistantMsg := context.Message{ + Role: context.RoleAssistant, + ToolCalls: response.Completion.ToolCalls, + } + // Include content if present + if response.Completion.Content != nil && !isEmptyValue(response.Completion.Content) { + assistantMsg.Content = response.Completion.Content + } + messages = append(messages, assistantMsg) + + // Add tool result messages for each tool call + for i, tc := range response.Completion.ToolCalls { + toolCallID := tc.ID + var resultContent string + + // Get result from response.Tools if available + if i < len(response.Tools) { + resultJSON, err := jsoniter.MarshalToString(response.Tools[i].Result) + if err == nil { + resultContent = resultJSON + } else { + resultContent = fmt.Sprintf("%v", response.Tools[i].Result) + } + } else { + resultContent = "{}" + } + + messages = append(messages, context.Message{ + Role: context.RoleTool, + ToolCallID: &toolCallID, + Content: resultContent, + }) + } + } else { + // No tool calls, just add content if present + content := extractOutput(response) + if content != nil && !isEmptyValue(content) { + messages = append(messages, context.Message{ + Role: context.RoleAssistant, + Content: content, + }) + } + } + + return messages +} diff --git a/agent/test/dynamic_types.go b/agent/test/dynamic_types.go index d75ec2c4..27cb0634 100644 --- a/agent/test/dynamic_types.go +++ b/agent/test/dynamic_types.go @@ -34,9 +34,12 @@ type TurnResult struct { // Input is the user message (from simulator or initial input) Input interface{} `json:"input"` - // Output is the agent's response + // Output is the agent's response (summary for display and conversation history) Output interface{} `json:"output,omitempty"` + // Response is the full agent response including completion and tool results + Response *TurnResponse `json:"response,omitempty"` + // CheckpointsReached lists checkpoint IDs reached in this turn CheckpointsReached []string `json:"checkpoints_reached,omitempty"` @@ -47,6 +50,30 @@ type TurnResult struct { Error string `json:"error,omitempty"` } +// TurnResponse contains the full agent response for a turn +type TurnResponse struct { + // Content is the text content from LLM completion + Content interface{} `json:"content,omitempty"` + + // ToolCalls contains the tool calls made by the agent + ToolCalls []ToolCallInfo `json:"tool_calls,omitempty"` + + // Next is the data returned from Next hook + Next interface{} `json:"next,omitempty"` +} + +// ToolCallInfo contains information about a tool call +type ToolCallInfo struct { + // Tool is the tool name + Tool string `json:"tool"` + + // Arguments are the tool call arguments + Arguments interface{} `json:"arguments,omitempty"` + + // Result is the tool execution result + Result interface{} `json:"result,omitempty"` +} + // CheckpointResult represents the result of a checkpoint validation type CheckpointResult struct { // ID is the checkpoint identifier diff --git a/agent/test/loader.go b/agent/test/loader.go index 52092b68..73fdad39 100644 --- a/agent/test/loader.go +++ b/agent/test/loader.go @@ -4,6 +4,7 @@ import ( "bufio" "fmt" "os" + "regexp" "strings" "time" @@ -145,6 +146,13 @@ func FilterByIDs(cases []*Case, ids []string) []*Case { }) } +// FilterByPattern returns test cases whose ID matches the given regex pattern +func FilterByPattern(cases []*Case, pattern *regexp.Regexp) []*Case { + return FilterTestCases(cases, func(tc *Case) bool { + return pattern.MatchString(tc.ID) + }) +} + // LoadFromAgent generates test cases using a generator agent func (l *JSONLLoader) LoadFromAgent(agentID string, targetInfo *TargetAgentInfo, params map[string]interface{}) ([]*Case, error) { return GenerateTestCases(agentID, targetInfo, params) diff --git a/agent/test/runner.go b/agent/test/runner.go index 3f18e9a8..9159df4d 100644 --- a/agent/test/runner.go +++ b/agent/test/runner.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "reflect" + "regexp" "strings" "sync" "time" @@ -232,6 +233,32 @@ func (r *Executor) RunTests() (*Report, error) { r.output.Warning("Skipped: %d test cases", skippedCount) } + // Filter by --run pattern if specified + if r.opts.Run != "" { + runPattern, err := regexp.Compile(r.opts.Run) + if err != nil { + return nil, fmt.Errorf("invalid --run pattern %q: %w", r.opts.Run, err) + } + activeTests = FilterByPattern(activeTests, runPattern) + if len(activeTests) == 0 { + return nil, fmt.Errorf("no test cases match pattern %q", r.opts.Run) + } + r.output.Info("Filter: %q (%d test cases match)", r.opts.Run, len(activeTests)) + } + + // Load context config if specified + if r.opts.ContextFile != "" { + ctxConfig, err := LoadContextConfig(r.opts.ContextFile) + if err != nil { + return nil, fmt.Errorf("failed to load context file: %w", err) + } + r.opts.ContextData = ctxConfig + r.output.Info("Context: %s", r.opts.ContextFile) + } + + // Set options on hook executor (for context data access in hooks) + r.hookExecutor.SetOptions(r.opts) + // Print test info if r.opts.Runs > 1 { r.output.Info("Runs: %d per test case (stability analysis)", r.opts.Runs) @@ -526,14 +553,12 @@ func (r *Executor) runDynamicTest(ast *assistant.Assistant, tc *Case, agentID st // Convert to standard result result := dynamicResult.ToResult() - // Execute after script if specified - defer func() { - if tc.After != "" && (tc.Before == "" || beforeData != nil || result.Status != StatusError || !isBeforeError(result.Error)) { - if err := r.hookExecutor.ExecuteAfter(tc.After, tc, result, beforeData, r.agentPath); err != nil { - r.output.Warning("after script failed: %s", err.Error()) - } + // Execute after script if specified (before outputting result) + if tc.After != "" && (tc.Before == "" || beforeData != nil || result.Status != StatusError || !isBeforeError(result.Error)) { + if err := r.hookExecutor.ExecuteAfter(tc.After, tc, result, beforeData, r.agentPath); err != nil { + r.output.Warning("after script failed: %s", err.Error()) } - }() + } // Output result duration := time.Duration(result.DurationMs) * time.Millisecond @@ -718,7 +743,7 @@ func buildContextOptions(tc *Case, runnerOpts *Options) *context.Options { } // extractOutput extracts the output from the agent response -// Priority: Next hook data (if non-empty) > Completion content > nil +// Priority: Next hook data > Completion content > Tool results message > nil func extractOutput(response *context.Response) interface{} { if response == nil { return nil @@ -729,11 +754,76 @@ func extractOutput(response *context.Response) interface{} { if response.Next != nil && !isEmptyValue(response.Next) { return response.Next } - // Fall back to raw completion content + + // Fall back to completion response if response.Completion != nil { - return response.Completion.Content + // If content is non-empty, return it + if response.Completion.Content != nil && !isEmptyValue(response.Completion.Content) { + return response.Completion.Content + } } + // If no content but tools were executed, extract message from tool results + // This handles the case where LLM calls tools but doesn't generate text + if len(response.Tools) > 0 { + return extractToolResultMessage(response.Tools) + } + + return nil +} + +// extractToolResultMessage extracts the message field from tool results +// Returns the first non-empty message found, or a summary of tool calls +func extractToolResultMessage(tools []context.ToolCallResponse) interface{} { + if len(tools) == 0 { + return nil + } + + // Try to extract "message" field from tool results first + for _, tool := range tools { + if tool.Result != nil { + // Try to get message from result map + if resultMap, ok := tool.Result.(map[string]interface{}); ok { + if msg, exists := resultMap["message"]; exists && msg != nil { + if msgStr, ok := msg.(string); ok && msgStr != "" { + return msgStr + } + } + } + } + } + + // No message found, generate a summary of tool calls + var summaries []string + for _, tool := range tools { + toolName := tool.Tool + if toolName == "" { + toolName = "unknown" + } + // Extract key info from result if possible + if tool.Result != nil { + if resultMap, ok := tool.Result.(map[string]interface{}); ok { + // Try common result fields + if action, ok := resultMap["action"].(string); ok { + summaries = append(summaries, fmt.Sprintf("[%s: %s]", toolName, action)) + continue + } + if success, ok := resultMap["success"].(bool); ok { + status := "failed" + if success { + status = "success" + } + summaries = append(summaries, fmt.Sprintf("[%s: %s]", toolName, status)) + continue + } + } + } + summaries = append(summaries, fmt.Sprintf("[%s]", toolName)) + } + + if len(summaries) > 0 { + return strings.Join(summaries, " ") + } return nil } diff --git a/agent/test/script_hooks.go b/agent/test/script_hooks.go index ea9349e1..a129b2ba 100644 --- a/agent/test/script_hooks.go +++ b/agent/test/script_hooks.go @@ -19,6 +19,7 @@ type HookExecutor struct { output *OutputWriter loadedDirs map[string]bool // Track which directories have been loaded agentContext *context.Context + opts *Options // Test options (includes ContextData from --ctx) } // NewHookExecutor creates a new hook executor @@ -35,6 +36,11 @@ func (h *HookExecutor) SetAgentContext(ctx *context.Context) { h.agentContext = ctx } +// SetOptions sets the test options for hook execution +func (h *HookExecutor) SetOptions(opts *Options) { + h.opts = opts +} + // HookRef represents a parsed hook reference // Format: "src/env_test.ts:Before" or just "Before" (uses default test file) type HookRef struct { @@ -86,13 +92,23 @@ func ParseHookRef(ref string) (*HookRef, error) { func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) { srcDir := filepath.Join(agentPath, "src") - // Check if already loaded + // Convert to relative path for application.App + // application.App expects paths relative to YAO_ROOT + relSrcDir := srcDir + if application.App != nil { + if rel, err := filepath.Rel(application.App.Root(), srcDir); err == nil { + relSrcDir = rel + } + } + + // Check if already loaded (use absolute path as key) + // No logging for already loaded - this is normal and happens frequently if h.loadedDirs[srcDir] { return nil, nil } // Check if src directory exists - exists, err := application.App.Exists(srcDir) + exists, err := application.App.Exists(relSrcDir) if err != nil { return nil, err } @@ -103,7 +119,7 @@ func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) { var loadedScripts []string exts := []string{"*_test.ts", "*_test.js"} - err = application.App.Walk(srcDir, func(root, file string, isdir bool) error { + err = application.App.Walk(relSrcDir, func(root, file string, isdir bool) error { if isdir { return nil } @@ -114,10 +130,10 @@ func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) { return nil } - // Generate script ID - scriptID := generateHookScriptID(file, srcDir) + // Generate script ID (use relative path for consistency) + scriptID := generateHookScriptID(file, relSrcDir) - // Load the script + // Load the script (file path from Walk is relative to App root) _, err := v8.Load(file, scriptID) if err != nil { if h.verbose { @@ -127,10 +143,6 @@ func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) { } loadedScripts = append(loadedScripts, scriptID) - if h.verbose { - h.output.Verbose("Loaded hook script: %s (id: %s)", base, scriptID) - } - return nil }, exts...) @@ -139,6 +151,12 @@ func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) { } h.loadedDirs[srcDir] = true + + // Log summary only once when scripts are first loaded + if h.verbose && len(loadedScripts) > 0 { + h.output.Verbose("Loaded %d hook scripts from %s", len(loadedScripts), relSrcDir) + } + return loadedScripts, nil } @@ -392,13 +410,19 @@ func (h *HookExecutor) executeHookFunctionWithCases(script *v8.Script, funcName return nil, fmt.Errorf("failed to convert to function: %w", err) } + // Build ctx argument + ctxJS, err := h.buildCtxArg(v8ctx) + if err != nil { + return nil, err + } + // Convert test cases to JS array casesJS, err := h.testCasesToJS(v8ctx, testCases) if err != nil { return nil, err } - jsResult, err := fn.Call(global, casesJS) + jsResult, err := fn.Call(global, ctxJS, casesJS) if err != nil { return nil, fmt.Errorf("hook function %s failed: %w", funcName, err) } @@ -454,6 +478,12 @@ func (h *HookExecutor) executeHookFunctionWithResults(script *v8.Script, funcNam return nil, fmt.Errorf("failed to convert to function: %w", err) } + // Build ctx argument + ctxJS, err := h.buildCtxArg(v8ctx) + if err != nil { + return nil, err + } + // Convert results to JS array resultsJS, err := h.resultsToJS(v8ctx, results) if err != nil { @@ -466,7 +496,7 @@ func (h *HookExecutor) executeHookFunctionWithResults(script *v8.Script, funcNam return nil, fmt.Errorf("failed to convert beforeData: %w", err) } - jsResult, err := fn.Call(global, resultsJS, beforeDataJS) + jsResult, err := fn.Call(global, ctxJS, resultsJS, beforeDataJS) if err != nil { return nil, fmt.Errorf("hook function %s failed: %w", funcName, err) } @@ -498,11 +528,105 @@ func (h *HookExecutor) setShareData(v8ctx *v8go.Context) error { }) } +// buildCtxArg builds the context argument for hook functions +func (h *HookExecutor) buildCtxArg(v8ctx *v8go.Context) (*v8go.Value, error) { + ctxMap := map[string]interface{}{ + "locale": "en", + } + + // Use ContextData from --ctx flag if available + if h.opts != nil && h.opts.ContextData != nil { + cfg := h.opts.ContextData + if cfg.Locale != "" { + ctxMap["locale"] = cfg.Locale + } + if cfg.Authorized != nil { + authorized := map[string]interface{}{} + if cfg.Authorized.UserID != "" { + authorized["user_id"] = cfg.Authorized.UserID + } + if cfg.Authorized.TeamID != "" { + authorized["team_id"] = cfg.Authorized.TeamID + } + if cfg.Authorized.TenantID != "" { + authorized["tenant_id"] = cfg.Authorized.TenantID + } + if cfg.Authorized.Sub != "" { + authorized["sub"] = cfg.Authorized.Sub + } + ctxMap["authorized"] = authorized + } + if cfg.Metadata != nil { + ctxMap["metadata"] = cfg.Metadata + } + } + + return bridge.JsValue(v8ctx, ctxMap) +} + // buildHookArgs builds the arguments for a hook function call +// Arguments order: ctx, testCase, result (for After), beforeData (for After) func (h *HookExecutor) buildHookArgs(v8ctx *v8go.Context, testCase *Case, result *Result, beforeData interface{}) ([]*v8go.Value, error) { var args []*v8go.Value - // Arg 1: testCase + // Arg 1: ctx (context) - build from opts.ContextData if available + ctxMap := map[string]interface{}{ + "locale": "en", + } + + // Use ContextData from --ctx flag if available + if h.opts != nil && h.opts.ContextData != nil { + cfg := h.opts.ContextData + if cfg.Locale != "" { + ctxMap["locale"] = cfg.Locale + } + if cfg.Authorized != nil { + authorized := map[string]interface{}{} + if cfg.Authorized.UserID != "" { + authorized["user_id"] = cfg.Authorized.UserID + } + if cfg.Authorized.TeamID != "" { + authorized["team_id"] = cfg.Authorized.TeamID + } + if cfg.Authorized.TenantID != "" { + authorized["tenant_id"] = cfg.Authorized.TenantID + } + if cfg.Authorized.Sub != "" { + authorized["sub"] = cfg.Authorized.Sub + } + ctxMap["authorized"] = authorized + } + if cfg.Metadata != nil { + ctxMap["metadata"] = cfg.Metadata + } + } else if testCase != nil { + // Fallback to test case fields + if testCase.UserID != "" { + ctxMap["user_id"] = testCase.UserID + } + if testCase.TeamID != "" { + ctxMap["team_id"] = testCase.TeamID + } + // Build authorized info + authorized := map[string]interface{}{} + if testCase.UserID != "" { + authorized["user_id"] = testCase.UserID + } + if testCase.TeamID != "" { + authorized["team_id"] = testCase.TeamID + } + if len(authorized) > 0 { + ctxMap["authorized"] = authorized + } + } + + ctxJS, err := bridge.JsValue(v8ctx, ctxMap) + if err != nil { + return nil, fmt.Errorf("failed to convert ctx: %w", err) + } + args = append(args, ctxJS) + + // Arg 2: testCase if testCase != nil { tcMap := map[string]interface{}{ "id": testCase.ID, @@ -514,12 +638,20 @@ func (h *HookExecutor) buildHookArgs(v8ctx *v8go.Context, testCase *Case, result if testCase.Assert != nil { tcMap["assert"] = testCase.Assert } + // Include simulator options for dynamic tests + if testCase.Simulator != nil { + tcMap["simulator"] = testCase.Simulator + } tcJS, err := bridge.JsValue(v8ctx, tcMap) if err != nil { return nil, fmt.Errorf("failed to convert testCase: %w", err) } args = append(args, tcJS) + } else { + // Pass empty object if no testCase + emptyJS, _ := bridge.JsValue(v8ctx, map[string]interface{}{}) + args = append(args, emptyJS) } // Arg 2: result (for After) diff --git a/agent/test/types.go b/agent/test/types.go index cddbeb8a..6e727b47 100644 --- a/agent/test/types.go +++ b/agent/test/types.go @@ -165,6 +165,10 @@ type Options struct { // ContextConfig represents custom context configuration from JSON file // This allows full customization of the test context including authorized info type ContextConfig struct { + // ChatID is the chat session identifier + // Used to maintain session state across turns in dynamic tests + ChatID string `json:"chat_id,omitempty"` + // Authorized contains custom authorization data Authorized *AuthorizedConfig `json:"authorized,omitempty"` @@ -554,9 +558,15 @@ type AssertionResult struct { } // GetEnvironment returns the effective test environment for this test case -// Priority: command line flags > test case fields > defaults +// Priority: command line flags > context config > test case fields > defaults func (tc *Case) GetEnvironment(opts *Options) *Environment { - env := NewEnvironment("", "") + // Start with context config if available, otherwise use defaults + var env *Environment + if opts != nil && opts.ContextData != nil { + env = NewEnvironmentWithContext("", "", opts.ContextData) + } else { + env = NewEnvironment("", "") + } // Apply test case specific values if tc.UserID != "" { From 3969279f717033d3cf476085f7a693afd5e83dd2 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 27 Dec 2025 13:40:15 +0800 Subject: [PATCH 2/3] Add Detailed Validation for Agent Assertions in Dynamic Testing - Introduced `ValidateWithDetails` method in the Asserter to provide comprehensive results for agent assertions, including failure messages and validation criteria. - Updated `checkCheckpoints` to utilize the new validation method, enhancing the checkpoint validation process with detailed agent validation results. - Enhanced `CheckpointResult` structure to include `AgentValidationResult`, capturing the agent's response and validation details. - Expanded documentation in README.md to explain the new validation process and output structure for agent assertions, ensuring clarity on the validation workflow and expected results. --- agent/test/README.md | 56 ++++++++++++ agent/test/assert.go | 39 +++++++++ agent/test/dynamic_runner.go | 160 +++++++++++++++++++++++++++++++++-- agent/test/dynamic_types.go | 21 +++++ 4 files changed, 268 insertions(+), 8 deletions(-) diff --git a/agent/test/README.md b/agent/test/README.md index 2135f018..f7749f3b 100644 --- a/agent/test/README.md +++ b/agent/test/README.md @@ -607,6 +607,33 @@ For semantic or fuzzy validation using an LLM: The validator agent receives the output and criteria, then returns `{"passed": true/false, "reason": "..."}`. +**How it works:** + +1. The framework builds a validation request with the agent's response (including tool result messages) +2. The validator agent evaluates the response against the criteria +3. The validator returns a JSON response with `passed` and `reason` + +**Output in test report (for checkpoints):** + +```json +{ + "agent_validation": { + "passed": true, + "reason": "Response explicitly confirms setup completion", + "criteria": "Response should be friendly and helpful", + "input": "Hello! How can I help you today?", + "response": { + "passed": true, + "reason": "Response explicitly confirms setup completion" + } + } +} +``` + +- `input`: The content sent to the validator (agent response + tool result messages) +- `response`: The raw JSON response from the validator agent +- `criteria`: The validation criteria from the test case + ### Script Assertions For custom validation logic: @@ -1037,6 +1064,35 @@ interface TurnResult { } ``` +### Checkpoint Result Structure + +Each checkpoint in the output includes: + +```typescript +interface CheckpointResult { + id: string; // Checkpoint identifier + reached: boolean; // Whether checkpoint was reached + reached_at_turn?: number; // Turn number when reached (if reached) + required: boolean; // Whether checkpoint is required + passed: boolean; // Whether assertion passed + message?: string; // Assertion result message + agent_validation?: { + // Agent assertion details (for type: "agent") + passed: boolean; // Validator's determination + reason: string; // Explanation from validator + criteria: string; // Validation criteria checked + input: any; // Content sent to validator + response: { + // Raw validator response + passed: boolean; + reason: string; + }; + }; +} +``` + +**Note**: For agent-based assertions (`type: "agent"`), the `agent_validation` field provides full transparency into the validation process. The `input` field contains the combined output (agent text response + tool result messages) that was validated. + ## Output Formats Determined by `-o` file extension: diff --git a/agent/test/assert.go b/agent/test/assert.go index 505f0e62..ad0c4864 100644 --- a/agent/test/assert.go +++ b/agent/test/assert.go @@ -42,6 +42,45 @@ func (a *Asserter) Validate(tc *Case, output interface{}) (bool, string) { return true, "" } +// ValidateWithDetails validates the output and returns detailed results +// This is useful for agent assertions where we want to capture the validator's response +func (a *Asserter) ValidateWithDetails(tc *Case, output interface{}) *AssertionResult { + if tc.Assert == nil { + return &AssertionResult{Passed: true} + } + + assertions := a.parseAssertions(tc.Assert) + if len(assertions) == 0 { + return &AssertionResult{Passed: true} + } + + // For single assertion, return its full result + if len(assertions) == 1 { + return a.evaluateAssertion(assertions[0], output, tc.Input) + } + + // For multiple assertions, combine results + var failures []string + for _, assertion := range assertions { + result := a.evaluateAssertion(assertion, output, tc.Input) + if !result.Passed { + msg := result.Message + if assertion.Message != "" { + msg = assertion.Message + } + failures = append(failures, msg) + } + } + + if len(failures) > 0 { + return &AssertionResult{ + Passed: false, + Message: strings.Join(failures, "; "), + } + } + return &AssertionResult{Passed: true} +} + // validateAssertions validates output against assertion rules func (a *Asserter) validateAssertions(tc *Case, output interface{}) (bool, string) { assertions := a.parseAssertions(tc.Assert) diff --git a/agent/test/dynamic_runner.go b/agent/test/dynamic_runner.go index 6530c7ad..6dc66e33 100644 --- a/agent/test/dynamic_runner.go +++ b/agent/test/dynamic_runner.go @@ -194,8 +194,8 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s // Add assistant response to messages, including tool calls if any messages = appendAssistantMessages(messages, response) - // Check checkpoints against this response - reachedIDs := r.checkCheckpoints(tc.Checkpoints, output, result) + // Check checkpoints against this response (including tool results) + reachedIDs := r.checkCheckpoints(tc.Checkpoints, response, result) turnResult.CheckpointsReached = reachedIDs if r.opts.Verbose && len(reachedIDs) > 0 { @@ -368,10 +368,15 @@ func (r *DynamicRunner) parseSimulatorResponse(response *context.Response) (*Sim return output, nil } -// checkCheckpoints validates checkpoints against current output -func (r *DynamicRunner) checkCheckpoints(checkpoints []*Checkpoint, output interface{}, result *DynamicResult) []string { +// checkCheckpoints validates checkpoints against current response +// It checks both the completion content and tool results for comprehensive validation +func (r *DynamicRunner) checkCheckpoints(checkpoints []*Checkpoint, response *context.Response, result *DynamicResult) []string { reachedIDs := make([]string, 0) + // Build combined output for checkpoint validation + // This includes both content and tool result messages + combinedOutput := buildCombinedOutput(response) + for _, cp := range checkpoints { cpResult := result.Checkpoints[cp.ID] if cpResult.Reached { @@ -394,22 +399,161 @@ func (r *DynamicRunner) checkCheckpoints(checkpoints []*Checkpoint, output inter } } - // Validate using asserter + // Validate using asserter against combined output with full details tempCase := &Case{Assert: cp.Assert} - passed, msg := r.asserter.Validate(tempCase, output) + assertResult := r.asserter.ValidateWithDetails(tempCase, combinedOutput) - if passed { + if assertResult.Passed { cpResult.Reached = true cpResult.Passed = true cpResult.ReachedAtTurn = len(result.Turns) + 1 - cpResult.Message = msg + cpResult.Message = assertResult.Message reachedIDs = append(reachedIDs, cp.ID) } + + // Store agent validation details if this is an agent assertion + if isAgentAssertion(cp.Assert) { + // Extract criteria from assertion value + var criteria string + if assertMap, ok := cp.Assert.(map[string]interface{}); ok { + if c, ok := assertMap["value"].(string); ok { + criteria = c + } + } + + cpResult.AgentValidation = &AgentValidationResult{ + Passed: assertResult.Passed, + Criteria: criteria, + Input: combinedOutput, // Content sent to validator for checking + } + + // Extract reason and store full response from validator + if assertResult.Expected != nil { + if validatorResponse, ok := assertResult.Expected.(map[string]interface{}); ok { + if reason, ok := validatorResponse["reason"].(string); ok { + cpResult.AgentValidation.Reason = reason + } + // Store the full validator response + cpResult.AgentValidation.Response = validatorResponse + } + } + } } return reachedIDs } +// isAgentAssertion checks if the assertion is an agent-based assertion +func isAgentAssertion(assert interface{}) bool { + if assertMap, ok := assert.(map[string]interface{}); ok { + if assertType, ok := assertMap["type"].(string); ok { + return assertType == "agent" + } + } + return false +} + +// truncateForReport truncates content for report output +func truncateForReport(content interface{}, maxLen int) interface{} { + if content == nil { + return nil + } + + str, ok := content.(string) + if !ok { + return content + } + + if len(str) <= maxLen { + return str + } + + return str[:maxLen] + "... (truncated)" +} + +// buildCombinedOutput builds a combined output string from response +// that includes both completion content and tool result messages +func buildCombinedOutput(response *context.Response) string { + if response == nil { + return "" + } + + var parts []string + + // Add completion content + if response.Completion != nil && response.Completion.Content != nil { + if content := extractContentString(response.Completion.Content); content != "" { + parts = append(parts, content) + } + } + + // Add tool result messages + if len(response.Tools) > 0 { + for _, tool := range response.Tools { + if tool.Result != nil { + // Try to extract message from result + if resultMap, ok := tool.Result.(map[string]interface{}); ok { + if msg, exists := resultMap["message"]; exists && msg != nil { + if msgStr, ok := msg.(string); ok && msgStr != "" { + parts = append(parts, msgStr) + } + } + } + } + } + } + + // Join all parts with newline for comprehensive matching + return joinNonEmpty(parts, "\n") +} + +// extractContentString extracts string content from various types +func extractContentString(content interface{}) string { + if content == nil { + return "" + } + + switch v := content.(type) { + case string: + return v + case []interface{}: + // Handle array content (e.g., multimodal content) + var texts []string + for _, item := range v { + if m, ok := item.(map[string]interface{}); ok { + if text, ok := m["text"].(string); ok { + texts = append(texts, text) + } + } + } + return joinNonEmpty(texts, "\n") + default: + // Try to marshal to string + if data, err := jsoniter.MarshalToString(content); err == nil { + return data + } + return "" + } +} + +// joinNonEmpty joins non-empty strings with separator +func joinNonEmpty(parts []string, sep string) string { + var nonEmpty []string + for _, p := range parts { + if p != "" { + nonEmpty = append(nonEmpty, p) + } + } + if len(nonEmpty) == 0 { + return "" + } + result := nonEmpty[0] + for i := 1; i < len(nonEmpty); i++ { + result += sep + nonEmpty[i] + } + return result +} + // allRequiredCheckpointsReached checks if all required checkpoints are reached func (r *DynamicRunner) allRequiredCheckpointsReached(result *DynamicResult) bool { for _, cp := range result.Checkpoints { diff --git a/agent/test/dynamic_types.go b/agent/test/dynamic_types.go index 27cb0634..ea215779 100644 --- a/agent/test/dynamic_types.go +++ b/agent/test/dynamic_types.go @@ -93,6 +93,27 @@ type CheckpointResult struct { // Message contains assertion result message Message string `json:"message,omitempty"` + + // AgentValidation contains the agent validator's response (for agent assertions) + AgentValidation *AgentValidationResult `json:"agent_validation,omitempty"` +} + +// AgentValidationResult contains the result from an agent-based assertion +type AgentValidationResult struct { + // Passed indicates if the agent validator determined the assertion passed + Passed bool `json:"passed"` + + // Reason is the explanation from the agent validator + Reason string `json:"reason,omitempty"` + + // Criteria is the validation criteria that was checked + Criteria string `json:"criteria,omitempty"` + + // Input is the content that was sent to validator for checking + Input interface{} `json:"input,omitempty"` + + // Response is the raw response from the validator agent + Response interface{} `json:"response,omitempty"` } // SimulatorInput is the input sent to the simulator agent From 5a63d9280be9e9752adaeb501ffa579c08038de2 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 27 Dec 2025 14:23:15 +0800 Subject: [PATCH 3/3] Implement Tool Call Assertions in Asserter for Enhanced Validation - Added support for `tool_called` and `tool_result` assertions in the Asserter, allowing for validation of tool execution and arguments. - Introduced methods to check if specific tools were called and to validate their results against expected patterns. - Updated the `README.md` to include detailed documentation on new assertion types, including usage examples and value formats. - Enhanced the dynamic runner to set the response for tool-related assertions, improving the overall testing framework's capabilities. --- agent/assistant/agent.go | 8 +- agent/test/README.md | 72 +++++ agent/test/assert.go | 277 +++++++++++++++++- agent/test/assert_test.go | 551 +++++++++++++++++++++++++++++++++++ agent/test/dynamic_runner.go | 8 + 5 files changed, 914 insertions(+), 2 deletions(-) diff --git a/agent/assistant/agent.go b/agent/assistant/agent.go index 89807d54..546617d3 100644 --- a/agent/assistant/agent.go +++ b/agent/assistant/agent.go @@ -264,6 +264,12 @@ func (ast *Assistant) Stream(ctx *context.Context, inputMessages []context.Messa // Execute all tool calls toolResults, hasErrors := ast.executeToolCalls(ctx, currentResponse.ToolCalls, attempt) + // Build a map of tool call ID to arguments for quick lookup + toolCallArgsMap := make(map[string]interface{}) + for _, tc := range currentResponse.ToolCalls { + toolCallArgsMap[tc.ID] = tc.Function.Arguments + } + // Convert toolResults to toolCallResponses toolCallResponses = make([]context.ToolCallResponse, len(toolResults)) for i, result := range toolResults { @@ -272,7 +278,7 @@ func (ast *Assistant) Stream(ctx *context.Context, inputMessages []context.Messa ToolCallID: result.ToolCallID, Server: result.Server(), Tool: result.Tool(), - Arguments: nil, + Arguments: toolCallArgsMap[result.ToolCallID], Result: parsedContent, Error: "", } diff --git a/agent/test/README.md b/agent/test/README.md index f7749f3b..f3f7039c 100644 --- a/agent/test/README.md +++ b/agent/test/README.md @@ -575,6 +575,8 @@ Use `assert` for flexible validation. If `assert` is defined, it takes precedenc | `json_path` | Extract JSON path and compare | `{"type": "json_path", "path": "$.field", "value": true}` | | `regex` | Match regex pattern | `{"type": "regex", "value": "\\d+"}` | | `type` | Check output type | `{"type": "type", "value": "object"}` | +| `tool_called` | Check if a tool was called | `{"type": "tool_called", "value": "setup"}` | +| `tool_result` | Check tool execution result | `{"type": "tool_result", "value": {"tool": "setup", "result": {"success": true}}}` | ### Assertion Fields @@ -634,6 +636,76 @@ The validator agent receives the output and criteria, then returns `{"passed": t - `response`: The raw JSON response from the validator agent - `criteria`: The validation criteria from the test case +### Tool Assertions + +For validating that specific tools were called and their results: + +#### tool_called + +Check if a specific tool was called: + +```jsonl +{ + "id": "T001", + "input": "Set up my expense system", + "assert": { + "type": "tool_called", + "value": "setup" + } +} +``` + +**Value formats:** + +- **String**: Tool name (supports suffix matching, e.g., `"setup"` matches `"agents_expense_tools__setup"`) +- **Array**: Any of the specified tools must be called +- **Object**: Match tool name and optionally arguments + +```jsonl +// Match any of these tools +{"type": "tool_called", "value": ["setup", "init"]} + +// Match tool with specific arguments +{"type": "tool_called", "value": {"name": "setup", "arguments": {"action": "init"}}} +``` + +#### tool_result + +Check the result of a tool execution: + +```jsonl +{ + "id": "T001", + "input": "Set up my expense system", + "assert": { + "type": "tool_result", + "value": { + "tool": "setup", + "result": { + "success": true + } + } + } +} +``` + +**Result matching:** + +- If `result` is omitted, only checks that the tool executed without error +- Supports partial matching (only specified fields are checked) +- Supports regex patterns with `regex:` prefix for string values + +```jsonl +// Just check tool executed without error +{"type": "tool_result", "value": {"tool": "setup"}} + +// Check specific result fields +{"type": "tool_result", "value": {"tool": "setup", "result": {"success": true}}} + +// Use regex for message matching +{"type": "tool_result", "value": {"tool": "setup", "result": {"message": "regex:(?i)setup.*complete"}}} +``` + ### Script Assertions For custom validation logic: diff --git a/agent/test/assert.go b/agent/test/assert.go index ad0c4864..ac16e3fd 100644 --- a/agent/test/assert.go +++ b/agent/test/assert.go @@ -15,13 +15,22 @@ import ( ) // Asserter handles test assertions -type Asserter struct{} +type Asserter struct { + // response holds the current response for tool-related assertions + response *context.Response +} // NewAsserter creates a new asserter func NewAsserter() *Asserter { return &Asserter{} } +// WithResponse sets the response for tool-related assertions +func (a *Asserter) WithResponse(response *context.Response) *Asserter { + a.response = response + return a +} + // Validate validates the output against the test case's assertions // Returns (passed, error message) func (a *Asserter) Validate(tc *Case, output interface{}) (bool, string) { @@ -205,6 +214,10 @@ func (a *Asserter) evaluateAssertion(assertion *Assertion, output, input interfa result = a.assertScript(assertion, output, input) case "agent": result = a.assertAgent(assertion, output, input) + case "tool_called": + result = a.assertToolCalled(assertion) + case "tool_result": + result = a.assertToolResult(assertion) default: result.Passed = false result.Message = fmt.Sprintf("unknown assertion type: %s", assertion.Type) @@ -747,6 +760,268 @@ func (a *Asserter) assertScript(assertion *Assertion, output, input interface{}) return result } +// assertToolCalled checks if a specific tool was called +// value can be: +// - string: exact tool name to match +// - []string: any of the tool names +// - map with "name" and optional "arguments" for more specific matching +func (a *Asserter) assertToolCalled(assertion *Assertion) *AssertionResult { + result := &AssertionResult{ + Assertion: assertion, + Expected: assertion.Value, + } + + if a.response == nil { + result.Passed = false + result.Message = "no response available for tool_called assertion" + return result + } + + if len(a.response.Tools) == 0 { + result.Passed = false + result.Message = "no tools were called" + return result + } + + // Get tool names that were called + calledTools := make([]string, 0, len(a.response.Tools)) + for _, tool := range a.response.Tools { + calledTools = append(calledTools, tool.Tool) + } + result.Actual = calledTools + + switch v := assertion.Value.(type) { + case string: + // Simple case: check if tool name matches (supports prefix matching) + for _, tool := range a.response.Tools { + if matchToolName(tool.Tool, v) { + result.Passed = true + result.Message = fmt.Sprintf("tool '%s' was called", v) + return result + } + } + result.Passed = false + result.Message = fmt.Sprintf("tool '%s' was not called, called: %v", v, calledTools) + + case []interface{}: + // Check if any of the specified tools were called + for _, expected := range v { + if expectedStr, ok := expected.(string); ok { + for _, tool := range a.response.Tools { + if matchToolName(tool.Tool, expectedStr) { + result.Passed = true + result.Message = fmt.Sprintf("tool '%s' was called", expectedStr) + return result + } + } + } + } + result.Passed = false + result.Message = fmt.Sprintf("none of the expected tools were called, called: %v", calledTools) + + case map[string]interface{}: + // Advanced case: match name and optionally arguments + expectedName, _ := v["name"].(string) + expectedArgs := v["arguments"] + + for _, tool := range a.response.Tools { + if matchToolName(tool.Tool, expectedName) { + // If arguments specified, check them too + if expectedArgs != nil { + if matchArguments(tool.Arguments, expectedArgs) { + result.Passed = true + result.Message = fmt.Sprintf("tool '%s' was called with matching arguments", expectedName) + return result + } + } else { + result.Passed = true + result.Message = fmt.Sprintf("tool '%s' was called", expectedName) + return result + } + } + } + result.Passed = false + if expectedArgs != nil { + result.Message = fmt.Sprintf("tool '%s' was not called with expected arguments", expectedName) + } else { + result.Message = fmt.Sprintf("tool '%s' was not called, called: %v", expectedName, calledTools) + } + + default: + result.Passed = false + result.Message = fmt.Sprintf("invalid tool_called value type: %T", assertion.Value) + } + + return result +} + +// assertToolResult checks the result of a tool call +// value should be a map with "tool" (name) and "result" (expected result pattern) +func (a *Asserter) assertToolResult(assertion *Assertion) *AssertionResult { + result := &AssertionResult{ + Assertion: assertion, + Expected: assertion.Value, + } + + if a.response == nil { + result.Passed = false + result.Message = "no response available for tool_result assertion" + return result + } + + if len(a.response.Tools) == 0 { + result.Passed = false + result.Message = "no tools were called" + return result + } + + spec, ok := assertion.Value.(map[string]interface{}) + if !ok { + result.Passed = false + result.Message = "tool_result assertion requires a map with 'tool' and 'result' fields" + return result + } + + toolName, _ := spec["tool"].(string) + expectedResult := spec["result"] + + if toolName == "" { + result.Passed = false + result.Message = "tool_result assertion requires 'tool' field" + return result + } + + // Find the tool call + for _, tool := range a.response.Tools { + if matchToolName(tool.Tool, toolName) { + result.Actual = tool.Result + + // Check if there was an error + if tool.Error != "" { + result.Passed = false + result.Message = fmt.Sprintf("tool '%s' returned error: %s", toolName, tool.Error) + return result + } + + // If no expected result specified, just check success (no error) + if expectedResult == nil { + result.Passed = true + result.Message = fmt.Sprintf("tool '%s' executed successfully", toolName) + return result + } + + // Match result + if matchResult(tool.Result, expectedResult) { + result.Passed = true + result.Message = fmt.Sprintf("tool '%s' result matches expected", toolName) + return result + } + + result.Passed = false + result.Message = fmt.Sprintf("tool '%s' result does not match expected", toolName) + return result + } + } + + result.Passed = false + result.Message = fmt.Sprintf("tool '%s' was not called", toolName) + return result +} + +// matchToolName checks if a tool name matches the expected pattern +// Supports exact match and suffix match (e.g., "setup" matches "agents_expense_tools__setup") +func matchToolName(actual, expected string) bool { + if actual == expected { + return true + } + // Support suffix matching (tool name without namespace prefix) + if strings.HasSuffix(actual, "__"+expected) || strings.HasSuffix(actual, "."+expected) { + return true + } + // Support contains matching for partial names + if strings.Contains(actual, expected) { + return true + } + return false +} + +// matchArguments checks if tool arguments match expected pattern +func matchArguments(actual, expected interface{}) bool { + expectedMap, ok := expected.(map[string]interface{}) + if !ok { + return false + } + + actualMap, ok := actual.(map[string]interface{}) + if !ok { + // Try parsing as JSON string + if actualStr, ok := actual.(string); ok { + var parsed map[string]interface{} + if err := jsoniter.UnmarshalFromString(actualStr, &parsed); err == nil { + actualMap = parsed + } else { + return false + } + } else { + return false + } + } + + // Check that all expected keys exist and match + for key, expectedVal := range expectedMap { + actualVal, exists := actualMap[key] + if !exists { + return false + } + if !validateOutput(actualVal, expectedVal) { + return false + } + } + return true +} + +// matchResult checks if tool result matches expected pattern +func matchResult(actual, expected interface{}) bool { + switch exp := expected.(type) { + case map[string]interface{}: + actualMap, ok := actual.(map[string]interface{}) + if !ok { + return false + } + // Check that all expected keys exist and match + for key, expectedVal := range exp { + actualVal, exists := actualMap[key] + if !exists { + return false + } + if !matchResult(actualVal, expectedVal) { + return false + } + } + return true + + case string: + // Support regex pattern matching for strings + if strings.HasPrefix(exp, "regex:") { + pattern := strings.TrimPrefix(exp, "regex:") + re, err := regexp.Compile(pattern) + if err != nil { + return false + } + actualStr := fmt.Sprintf("%v", actual) + return re.MatchString(actualStr) + } + return fmt.Sprintf("%v", actual) == exp + + case bool: + actualBool, ok := actual.(bool) + return ok && actualBool == exp + + default: + return validateOutput(actual, expected) + } +} + // toString converts a value to string for comparison func (a *Asserter) toString(v interface{}) string { if v == nil { diff --git a/agent/test/assert_test.go b/agent/test/assert_test.go index 05c9d850..6f262d63 100644 --- a/agent/test/assert_test.go +++ b/agent/test/assert_test.go @@ -2,6 +2,8 @@ package test import ( "testing" + + "github.com/yaoapp/yao/agent/context" ) func TestAsserter_JSONPath_ArrayEquality(t *testing.T) { @@ -303,3 +305,552 @@ func TestAsserter_Regex(t *testing.T) { }) } } + +func TestAsserter_ToolCalled(t *testing.T) { + tests := []struct { + name string + tc *Case + response *context.Response + expected bool + errMsg string + }{ + { + name: "tool called - exact match", + tc: &Case{ + Assert: map[string]interface{}{ + "type": "tool_called", + "value": "agents_expense_tools__setup", + }, + }, + response: &context.Response{ + Tools: []context.ToolCallResponse{ + {Tool: "agents_expense_tools__setup", Result: map[string]interface{}{"success": true}}, + }, + }, + expected: true, + }, + { + name: "tool called - suffix match", + tc: &Case{ + Assert: map[string]interface{}{ + "type": "tool_called", + "value": "setup", + }, + }, + response: &context.Response{ + Tools: []context.ToolCallResponse{ + {Tool: "agents_expense_tools__setup", Result: map[string]interface{}{"success": true}}, + }, + }, + expected: true, + }, + { + name: "tool not called", + tc: &Case{ + Assert: map[string]interface{}{ + "type": "tool_called", + "value": "setup", + }, + }, + response: &context.Response{ + Tools: []context.ToolCallResponse{}, + }, + expected: false, + }, + { + name: "tool called - wrong tool", + tc: &Case{ + Assert: map[string]interface{}{ + "type": "tool_called", + "value": "setup", + }, + }, + response: &context.Response{ + Tools: []context.ToolCallResponse{ + {Tool: "agents_expense_tools__submit", Result: map[string]interface{}{"success": true}}, + }, + }, + expected: false, + }, + { + name: "tool called - any of multiple", + tc: &Case{ + Assert: map[string]interface{}{ + "type": "tool_called", + "value": []interface{}{"setup", "init"}, + }, + }, + response: &context.Response{ + Tools: []context.ToolCallResponse{ + {Tool: "agents_expense_tools__init", Result: map[string]interface{}{"success": true}}, + }, + }, + expected: true, + }, + { + name: "tool called - with arguments (map)", + tc: &Case{ + Assert: map[string]interface{}{ + "type": "tool_called", + "value": map[string]interface{}{ + "name": "setup", + "arguments": map[string]interface{}{ + "action": "init", + }, + }, + }, + }, + response: &context.Response{ + Tools: []context.ToolCallResponse{ + { + Tool: "agents_expense_tools__setup", + Arguments: map[string]interface{}{"action": "init", "config": map[string]interface{}{}}, + Result: map[string]interface{}{"success": true}, + }, + }, + }, + expected: true, + }, + { + name: "tool called - with arguments (JSON string)", + tc: &Case{ + Assert: map[string]interface{}{ + "type": "tool_called", + "value": map[string]interface{}{ + "name": "setup", + "arguments": map[string]interface{}{ + "action": "init", + }, + }, + }, + }, + response: &context.Response{ + Tools: []context.ToolCallResponse{ + { + Tool: "agents_expense_tools__setup", + Arguments: `{"action":"init","config":{"default_currency":"USD"}}`, + Result: map[string]interface{}{"success": true}, + }, + }, + }, + expected: true, + }, + { + name: "tool called - wrong arguments", + tc: &Case{ + Assert: map[string]interface{}{ + "type": "tool_called", + "value": map[string]interface{}{ + "name": "setup", + "arguments": map[string]interface{}{ + "action": "update", + }, + }, + }, + }, + response: &context.Response{ + Tools: []context.ToolCallResponse{ + { + Tool: "agents_expense_tools__setup", + Arguments: map[string]interface{}{"action": "init"}, + Result: map[string]interface{}{"success": true}, + }, + }, + }, + expected: false, + }, + { + name: "no response", + tc: &Case{ + Assert: map[string]interface{}{ + "type": "tool_called", + "value": "setup", + }, + }, + response: nil, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + asserter := NewAsserter().WithResponse(tt.response) + passed, errMsg := asserter.Validate(tt.tc, nil) + if passed != tt.expected { + t.Errorf("Expected passed=%v, got passed=%v, error: %s", tt.expected, passed, errMsg) + } + }) + } +} + +func TestAsserter_ToolResult(t *testing.T) { + tests := []struct { + name string + tc *Case + response *context.Response + expected bool + }{ + { + name: "tool result - success check", + tc: &Case{ + Assert: map[string]interface{}{ + "type": "tool_result", + "value": map[string]interface{}{ + "tool": "setup", + "result": map[string]interface{}{ + "success": true, + }, + }, + }, + }, + response: &context.Response{ + Tools: []context.ToolCallResponse{ + { + Tool: "agents_expense_tools__setup", + Result: map[string]interface{}{"success": true, "message": "Setup complete"}, + }, + }, + }, + expected: true, + }, + { + name: "tool result - message check with regex", + tc: &Case{ + Assert: map[string]interface{}{ + "type": "tool_result", + "value": map[string]interface{}{ + "tool": "setup", + "result": map[string]interface{}{ + "message": "regex:(?i)setup.*complete", + }, + }, + }, + }, + response: &context.Response{ + Tools: []context.ToolCallResponse{ + { + Tool: "agents_expense_tools__setup", + Result: map[string]interface{}{"success": true, "message": "Setup complete!"}, + }, + }, + }, + expected: true, + }, + { + name: "tool result - no expected result (just check no error)", + tc: &Case{ + Assert: map[string]interface{}{ + "type": "tool_result", + "value": map[string]interface{}{ + "tool": "setup", + }, + }, + }, + response: &context.Response{ + Tools: []context.ToolCallResponse{ + { + Tool: "agents_expense_tools__setup", + Result: map[string]interface{}{"success": true}, + }, + }, + }, + expected: true, + }, + { + name: "tool result - tool has error", + tc: &Case{ + Assert: map[string]interface{}{ + "type": "tool_result", + "value": map[string]interface{}{ + "tool": "setup", + }, + }, + }, + response: &context.Response{ + Tools: []context.ToolCallResponse{ + { + Tool: "agents_expense_tools__setup", + Error: "permission denied", + }, + }, + }, + expected: false, + }, + { + name: "tool result - result mismatch", + tc: &Case{ + Assert: map[string]interface{}{ + "type": "tool_result", + "value": map[string]interface{}{ + "tool": "setup", + "result": map[string]interface{}{ + "success": true, + }, + }, + }, + }, + response: &context.Response{ + Tools: []context.ToolCallResponse{ + { + Tool: "agents_expense_tools__setup", + Result: map[string]interface{}{"success": false}, + }, + }, + }, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + asserter := NewAsserter().WithResponse(tt.response) + passed, errMsg := asserter.Validate(tt.tc, nil) + if passed != tt.expected { + t.Errorf("Expected passed=%v, got passed=%v, error: %s", tt.expected, passed, errMsg) + } + }) + } +} + +func TestAsserter_MultipleToolAssertions(t *testing.T) { + // This tests the exact scenario from setup-006: tool_called + tool_result + response := &context.Response{ + Tools: []context.ToolCallResponse{ + { + Tool: "agents_expense_tools__setup", + Arguments: `{"action":"init","config":{"default_currency":"USD","categories":[{"id":"meals","name":"Meals","daily_limit":100}]}}`, + Result: map[string]interface{}{ + "success": true, + "action": "init", + "config": map[string]interface{}{ + "default_currency": "USD", + }, + "message": "Setup complete!", + }, + }, + }, + } + + asserter := NewAsserter().WithResponse(response) + + tc := &Case{ + Assert: []interface{}{ + map[string]interface{}{ + "type": "tool_called", + "value": map[string]interface{}{ + "name": "setup", + "arguments": map[string]interface{}{ + "action": "init", + }, + }, + }, + map[string]interface{}{ + "type": "tool_result", + "value": map[string]interface{}{ + "tool": "setup", + "result": map[string]interface{}{ + "success": true, + }, + }, + }, + }, + } + + result := asserter.ValidateWithDetails(tc, nil) + if !result.Passed { + t.Errorf("Expected multiple tool assertions to pass, got: %s", result.Message) + } +} + +func TestAsserter_SharedAsserterWithResponse(t *testing.T) { + // Test that a shared asserter correctly uses WithResponse + asserter := NewAsserter() + + // First call without response - should fail + tc := &Case{ + Assert: map[string]interface{}{ + "type": "tool_called", + "value": "setup", + }, + } + result := asserter.ValidateWithDetails(tc, nil) + if result.Passed { + t.Error("Expected tool_called to fail without response") + } + if result.Message != "no response available for tool_called assertion" { + t.Errorf("Unexpected message: %s", result.Message) + } + + // Now set response + response := &context.Response{ + Tools: []context.ToolCallResponse{ + { + Tool: "agents_expense_tools__setup", + Result: map[string]interface{}{"success": true}, + }, + }, + } + asserter.WithResponse(response) + + // Should pass now + result = asserter.ValidateWithDetails(tc, nil) + if !result.Passed { + t.Errorf("Expected tool_called to pass with response, got: %s", result.Message) + } +} + +func TestAsserter_Setup006Scenario(t *testing.T) { + // Exact reproduction of setup-006 scenario + // Turn 2: tool was called with action: init, result has success: true + response := &context.Response{ + Tools: []context.ToolCallResponse{ + { + Tool: "agents_expense_tools__setup", + Arguments: `{"action":"init","config":{"default_currency":"USD","categories":[{"id":"meals","name":"Meals","daily_limit":100},{"id":"travel","name":"Travel","daily_limit":500}]}}`, + Result: map[string]interface{}{ + "config": map[string]interface{}{ + "categories": []interface{}{ + map[string]interface{}{"daily_limit": float64(100), "id": "meals", "name": "Meals"}, + map[string]interface{}{"daily_limit": float64(500), "id": "travel", "name": "Travel"}, + }, + "default_currency": "USD", + }, + "message": "Setup complete! The expense system has been initialized successfully with the configured settings. You can now start submitting expenses.", + "success": true, + "action": "init", + }, + }, + }, + } + + // This is the exact assert from setup-006's quick_complete checkpoint + assertDef := []interface{}{ + map[string]interface{}{ + "type": "tool_called", + "value": map[string]interface{}{ + "name": "setup", + "arguments": map[string]interface{}{ + "action": "init", + }, + }, + }, + map[string]interface{}{ + "type": "tool_result", + "value": map[string]interface{}{ + "tool": "setup", + "result": map[string]interface{}{ + "success": true, + }, + }, + }, + } + + asserter := NewAsserter().WithResponse(response) + tc := &Case{Assert: assertDef} + + result := asserter.ValidateWithDetails(tc, nil) + if !result.Passed { + t.Errorf("Expected setup-006 scenario to pass, got: %s", result.Message) + } + + // Also test individual assertions + t.Run("tool_called only", func(t *testing.T) { + tc2 := &Case{Assert: assertDef[0]} + result2 := asserter.ValidateWithDetails(tc2, nil) + if !result2.Passed { + t.Errorf("Expected tool_called to pass, got: %s", result2.Message) + } + }) + + t.Run("tool_result only", func(t *testing.T) { + tc3 := &Case{Assert: assertDef[1]} + result3 := asserter.ValidateWithDetails(tc3, nil) + if !result3.Passed { + t.Errorf("Expected tool_result to pass, got: %s", result3.Message) + } + }) +} + +func TestAsserter_Setup003Scenario(t *testing.T) { + // Exact reproduction of setup-003 scenario + // Turn 3: tool was called with action: update + response := &context.Response{ + Tools: []context.ToolCallResponse{ + { + Tool: "agents_expense_tools__setup", + Arguments: `{"action":"update","config":{"categories":[{"daily_limit":500,"id":"meals","name":"Business Meals"}]}}`, + Result: map[string]interface{}{ + "action": "update", + "config": map[string]interface{}{ + "categories": []interface{}{ + map[string]interface{}{"daily_limit": float64(500), "id": "meals", "name": "Business Meals"}, + }, + }, + "message": "Configuration updated successfully! Your changes have been saved.", + "success": true, + }, + }, + }, + } + + // This is the exact assert from setup-003's update_complete checkpoint + assertDef := []interface{}{ + map[string]interface{}{ + "type": "tool_called", + "value": map[string]interface{}{ + "name": "setup", + "arguments": map[string]interface{}{ + "action": "update", + }, + }, + }, + map[string]interface{}{ + "type": "tool_result", + "value": map[string]interface{}{ + "tool": "setup", + "result": map[string]interface{}{ + "success": true, + }, + }, + }, + } + + asserter := NewAsserter().WithResponse(response) + tc := &Case{Assert: assertDef} + + result := asserter.ValidateWithDetails(tc, nil) + if !result.Passed { + t.Errorf("Expected setup-003 scenario to pass, got: %s", result.Message) + } + + // Test individual assertions + t.Run("tool_called with action:update", func(t *testing.T) { + tc2 := &Case{Assert: assertDef[0]} + result2 := asserter.ValidateWithDetails(tc2, nil) + if !result2.Passed { + t.Errorf("Expected tool_called to pass, got: %s", result2.Message) + } + }) +} + +func TestMatchToolName(t *testing.T) { + tests := []struct { + actual string + expected string + match bool + }{ + {"agents_expense_tools__setup", "agents_expense_tools__setup", true}, + {"agents_expense_tools__setup", "setup", true}, + {"agents.expense.tools.setup", "setup", true}, + {"agents_expense_tools__setup", "init", false}, + {"setup", "setup", true}, + } + + for _, tt := range tests { + t.Run(tt.actual+"_"+tt.expected, func(t *testing.T) { + if matchToolName(tt.actual, tt.expected) != tt.match { + t.Errorf("matchToolName(%q, %q) = %v, want %v", tt.actual, tt.expected, !tt.match, tt.match) + } + }) + } +} diff --git a/agent/test/dynamic_runner.go b/agent/test/dynamic_runner.go index 6dc66e33..2f52266c 100644 --- a/agent/test/dynamic_runner.go +++ b/agent/test/dynamic_runner.go @@ -377,6 +377,9 @@ func (r *DynamicRunner) checkCheckpoints(checkpoints []*Checkpoint, response *co // This includes both content and tool result messages combinedOutput := buildCombinedOutput(response) + // Set response on asserter for tool-related assertions + r.asserter.WithResponse(response) + for _, cp := range checkpoints { cpResult := result.Checkpoints[cp.ID] if cpResult.Reached { @@ -409,6 +412,11 @@ func (r *DynamicRunner) checkCheckpoints(checkpoints []*Checkpoint, response *co cpResult.ReachedAtTurn = len(result.Turns) + 1 cpResult.Message = assertResult.Message reachedIDs = append(reachedIDs, cp.ID) + } else { + // Store failure message for debugging (but don't mark as failed yet - it might pass in a later turn) + if cpResult.Message == "" { + cpResult.Message = assertResult.Message + } } // Store agent validation details if this is an agent assertion