diff --git a/agent/context/log.go b/agent/context/log.go index 1f555bc6..16658e42 100644 --- a/agent/context/log.go +++ b/agent/context/log.go @@ -378,6 +378,10 @@ func (l *RequestLogger) LLMStart(connector, model string, messageCount int) { // LLMComplete logs the completion of an LLM call func (l *RequestLogger) LLMComplete(tokens int, hasToolCalls bool) { + if l.noop { + return + } + elapsed := time.Since(l.startTime).Round(time.Millisecond) status := "streaming" if hasToolCalls { @@ -397,6 +401,10 @@ func (l *RequestLogger) LLMComplete(tokens int, hasToolCalls bool) { // ToolStart logs the start of tool execution func (l *RequestLogger) ToolStart(toolName string) { + if l.noop { + return + } + if config.IsDevelopment() { fmt.Printf("%s ๐Ÿ”ง Tool: %s%s\n", colorYellow, toolName, colorReset) } else { @@ -406,6 +414,10 @@ func (l *RequestLogger) ToolStart(toolName string) { // ToolComplete logs the completion of tool execution func (l *RequestLogger) ToolComplete(toolName string, success bool) { + if l.noop { + return + } + if config.IsDevelopment() { if success { fmt.Printf("%s โœ“ %s completed%s\n", colorGreen, toolName, colorReset) @@ -423,6 +435,10 @@ func (l *RequestLogger) ToolComplete(toolName string, success bool) { // HookStart logs the start of a hook execution func (l *RequestLogger) HookStart(hookName string) { + if l.noop { + return + } + elapsed := time.Since(l.startTime).Round(time.Millisecond) if config.IsDevelopment() { @@ -434,6 +450,10 @@ func (l *RequestLogger) HookStart(hookName string) { // HookComplete logs the completion of a hook func (l *RequestLogger) HookComplete(hookName string) { + if l.noop { + return + } + if config.IsDevelopment() { fmt.Printf("%s โœ“ %s done%s\n", colorGreen, hookName, colorReset) } else { @@ -456,6 +476,10 @@ func (l *RequestLogger) Cleanup(resource string) { // HistoryLoad logs history loading func (l *RequestLogger) HistoryLoad(count, maxSize int) { + if l.noop { + return + } + if config.IsDevelopment() { fmt.Printf("%s Loaded %d/%d history messages%s\n", colorGray, count, maxSize, colorReset) } else { @@ -465,6 +489,10 @@ func (l *RequestLogger) HistoryLoad(count, maxSize int) { // HistoryOverlap logs overlap detection func (l *RequestLogger) HistoryOverlap(overlapCount int) { + if l.noop { + return + } + if overlapCount > 0 { if config.IsDevelopment() { fmt.Printf("%s Removed %d overlapping messages%s\n", colorYellow, overlapCount, colorReset) diff --git a/agent/test/DESIGN.md b/agent/test/DESIGN.md new file mode 100644 index 00000000..cd072d94 --- /dev/null +++ b/agent/test/DESIGN.md @@ -0,0 +1,657 @@ +# Agent Test Package Design + +## Overview + +Agent Test Package provides a framework for testing AI agents with structured test cases. +It supports batch testing, report generation, stability analysis, and CI integration. + +### Quick Start + +```bash +# Quick test with a single message (auto-detect agent from current directory) +cd assistants/keyword +yao agent test -i "hello world" + +# Or specify agent explicitly +yao agent test -i "hello world" -n keyword.agent + +# Run tests from JSONL file (auto-detect agent from path) +yao agent test -i assistants/keyword/tests/inputs.jsonl + +# Run with stability analysis (5 runs per test case) +yao agent test -i assistants/keyword/tests/inputs.jsonl --runs 5 + +# Generate HTML report +yao agent test -i assistants/keyword/tests/inputs.jsonl -r report.html -o report.html +``` + +## Usage + +```bash +# Basic usage - auto-detect agent, output to same directory as input +# Output: tests/output-20241217100000.jsonl +yao agent test -i tests/inputs.jsonl + +# Override connector +yao agent test -i tests/inputs.jsonl -c openai.gpt4 + +# Specify agent explicitly +yao agent test -i tests/inputs.jsonl -n my.agent + +# Specify test environment (user and team) +yao agent test -i tests/inputs.jsonl -u test-user -t test-team + +# Run multiple times for stability analysis +yao agent test -i tests/inputs.jsonl --runs 5 + +# Custom timeout per test case (default: 5m) +yao agent test -i tests/inputs.jsonl --timeout 10m + +# Run tests in parallel (4 concurrent test cases) +yao agent test -i tests/inputs.jsonl --parallel 4 + +# Combine parallel and timeout for faster execution +yao agent test -i tests/inputs.jsonl --parallel 8 --timeout 2m + +# Custom output file path +yao agent test -i tests/inputs.jsonl -o /path/to/results.jsonl + +# Use custom reporter agent for personalized report (HTML) +yao agent test -i tests/inputs.jsonl -r report.html -o report.html + +# Use custom reporter agent for personalized report (Markdown) +yao agent test -i tests/inputs.jsonl -r report.markdown -o report.md + +# Full example with all options +yao agent test -i tests/inputs.jsonl \ + -n keyword.agent \ + -c deepseek.v3 \ + -u test-user \ + -t test-team \ + --runs 3 \ + --timeout 10m \ + --parallel 4 \ + -r report.html \ + -o report.html +``` + +### Input Modes + +The `-i` flag supports two input modes: + +**1. JSONL File Mode** - Load test cases from a file: + +```bash +yao agent test -i tests/inputs.jsonl +``` + +**2. Direct Message Mode** - Test with a single message: + +```bash +# Auto-detect agent from current working directory +cd assistants/keyword +yao agent test -i "Extract keywords from this text" + +# Or specify agent explicitly +yao agent test -i "Extract keywords from this text" -n keyword.agent +yao agent test -i "ไฝ ๅฅฝไธ–็•Œ" -n keyword.agent -c deepseek.v3 +``` + +When using direct message mode: + +- Agent is resolved from current working directory (looks for `package.yao` upward) +- If not found, use `-n` flag to specify agent explicitly +- Output is printed to stdout (or saved to `-o` if specified) +- Useful for quick testing and debugging + +### Default Output Path + +When `-o` is not specified and using JSONL file mode, the output file is automatically generated in the same directory as the input file: + +``` +{input_directory}/output-{timestamp}.jsonl +``` + +Example: + +- Input: `/app/assistants/keyword/tests/inputs.jsonl` +- Output: `/app/assistants/keyword/tests/output-20241217100000.jsonl` + +The timestamp format is `YYYYMMDDHHMMSS` (e.g., `20241217100000` for 2024-12-17 10:00:00). + +When using direct message mode without `-o`, output is printed to stdout. + +## Command Line Options + +| Flag | Long Flag | Description | Default | Example | +| ---- | ------------- | ------------------------------------- | -------------------------- | --------------------------------------- | +| `-i` | `--input` | Input: JSONL file path or message | - | `-i tests/inputs.jsonl` or `-i "hello"` | +| `-o` | `--output` | Path to output file (format by ext) | `output-{timestamp}.jsonl` | `-o report.html` | +| `-n` | `--name` | Explicit agent ID | auto-detect | `-n keyword.agent` | +| `-c` | `--connector` | Override connector | agent default | `-c openai.gpt4` | +| `-u` | `--user` | Test user ID (global override) | "test-user" | `-u admin` | +| `-t` | `--team` | Test team ID (global override) | "test-team" | `-t ops-team` | +| `-r` | `--reporter` | Custom reporter agent ID | - (use built-in) | `-r report.beautiful` | +| | `--runs` | Number of runs for stability analysis | 1 | `--runs 5` | +| | `--timeout` | Default timeout per test case | 5m | `--timeout 10m` | +| | `--parallel` | Number of parallel test cases | 1 | `--parallel 4` | +| `-v` | `--verbose` | Verbose output | false | `-v` | +| | `--fail-fast` | Stop on first failure | false | `--fail-fast` | + +**Notes**: + +- Without `-o` flag, output is saved to `{input_dir}/output-{timestamp}.jsonl` +- Output format is determined by `-o` file extension: `.jsonl`, `.json`, `.md`, `.html` +- Use `-r` to specify a custom reporter agent for personalized report generation + +## Agent Resolution + +The agent is resolved in the following order: + +1. **Explicit specification** (`-n` flag): Use the specified agent ID +2. **Path-based detection**: Traverse up from `tests/inputs.jsonl` to find `package.yao` + +### Path-based Detection Example + +``` +/app/assistants/workers/system/keyword/ +โ”œโ”€โ”€ package.yao <- Agent definition +โ”œโ”€โ”€ prompts.yml +โ”œโ”€โ”€ src/ +โ”‚ โ””โ”€โ”€ index.ts +โ””โ”€โ”€ tests/ + โ””โ”€โ”€ inputs.jsonl <- Test input file +``` + +Given input path `/app/assistants/workers/system/keyword/tests/inputs.jsonl`: + +1. Check `/app/assistants/workers/system/keyword/tests/package.yao` - not found +2. Check `/app/assistants/workers/system/keyword/package.yao` - **found!** +3. Load agent from `/app/assistants/workers/system/keyword/` + +## Test Environment + +Agent calls require a `Context` with user and tenant information. The test framework creates a test context with configurable environment: + +```go +// TestEnvironment configures the test execution context +type TestEnvironment struct { + UserID string // User ID for authorized info (-u flag) + TeamID string // Team ID for authorized info (-t flag) + Locale string // Locale (default: "en-us") + ClientType string // Client type (default: "test") + ClientIP string // Client IP (default: "127.0.0.1") + Referer string // Request referer (default: "test") + Accept string // Accept format (default: "standard") +} +``` + +Example context creation (similar to `agent_next_test.go`): + +```go +func newTestContext(env *TestEnvironment, chatID, assistantID string) *context.Context { + authorized := &types.AuthorizedInfo{ + Subject: env.UserID, + UserID: env.UserID, + TeamID: env.TeamID, + } + ctx := context.New(stdContext.Background(), authorized, chatID) + ctx.AssistantID = assistantID + ctx.Locale = env.Locale + ctx.Client = context.Client{ + Type: env.ClientType, + IP: env.ClientIP, + } + ctx.Referer = env.Referer + ctx.Accept = env.Accept + return ctx +} +``` + +## Stability Analysis (Multiple Runs) + +When `--runs N` is specified (N > 1), the framework runs each test case N times and collects stability metrics: + +### Stability Metrics + +| Metric | Description | +| ------------------ | ------------------------------------------ | +| `pass_rate` | Percentage of runs that passed (0-100%) | +| `consistency` | How consistent the outputs are across runs | +| `avg_duration_ms` | Average execution time | +| `min_duration_ms` | Minimum execution time | +| `max_duration_ms` | Maximum execution time | +| `std_deviation_ms` | Standard deviation of execution time | + +### Stability Report Structure + +```json +{ + "summary": { + "total_cases": 42, + "total_runs": 126, + "runs_per_case": 3, + "overall_pass_rate": 95.2, + "stable_cases": 38, + "unstable_cases": 4, + "duration_ms": 45678 + }, + "results": [ + { + "id": "T001", + "runs": 3, + "passed": 3, + "failed": 0, + "pass_rate": 100.0, + "consistency": 1.0, + "stable": true, + "avg_duration_ms": 234, + "min_duration_ms": 210, + "max_duration_ms": 256, + "std_deviation_ms": 18.5, + "run_details": [ + {"run": 1, "status": "passed", "duration_ms": 234, "output": {...}}, + {"run": 2, "status": "passed", "duration_ms": 210, "output": {...}}, + {"run": 3, "status": "passed", "duration_ms": 256, "output": {...}} + ] + }, + { + "id": "T002", + "runs": 3, + "passed": 2, + "failed": 1, + "pass_rate": 66.7, + "consistency": 0.67, + "stable": false, + "run_details": [...] + } + ] +} +``` + +### Stability Classification + +| Pass Rate | Classification | +| --------- | --------------- | +| 100% | Stable | +| 80-99% | Mostly Stable | +| 50-79% | Unstable | +| < 50% | Highly Unstable | + +## Custom Reporter Agent + +By default, the framework outputs JSONL format. You can specify a reporter agent (`-r` flag) for personalized report generation: + +### Reporter Agent Interface + +The reporter agent receives the test results and generates a custom report: + +```json +// Input to reporter agent +{ + "report": { + "summary": {...}, + "results": [...], + "metadata": {...} + }, + "format": "html", // or "markdown", "text" + "options": { + "verbose": true, + "include_outputs": true + } +} +``` + +### Built-in Reporter Agents + +| Agent ID | Description | +| ----------------- | -------------------------------------- | +| `report.json` | JSON format (default, no agent needed) | +| `report.markdown` | Markdown format with tables | +| `report.html` | Interactive HTML report | +| `report.summary` | Brief text summary | + +### Custom Reporter Example + +Create a custom reporter agent at `assistants/reporters/my-reporter/`: + +```yaml +# prompts.yml +- role: system + content: | + You are a test report generator. Generate a beautiful report from test results. + + Output format: HTML with embedded CSS + + Requirements: + - Show summary statistics prominently + - Use color coding (green=pass, red=fail) + - Include charts for stability metrics + - Make it printable +``` + +## Input Format (JSONL) + +Each line in the input file is a JSON object with the following structure: + +```jsonl +{"id": "T001", "input": "Simple text input"} +{"id": "T002", "input": {"role": "user", "content": "Message with role"}} +{"id": "T003", "input": {"role": "user", "content": [{"type": "text", "text": "ContentPart array"}]}} +{"id": "T004", "input": [{"role": "user", "content": "First message"}, {"role": "assistant", "content": "Response"}, {"role": "user", "content": "Follow-up"}]} +{"id": "T005", "input": "Text input", "expected": {"keywords": ["keyword1", "keyword2"]}} +{"id": "T006", "input": "Test with specific user", "user": "admin", "team": "ops-team"} +``` + +### Input Types + +| Type | Description | Example | +| ----------- | ------------------------ | ----------------------------------------------------- | +| `string` | Simple text input | `"Hello world"` | +| `Message` | Single message with role | `{"role": "user", "content": "..."}` | +| `[]Message` | Conversation history | `[{"role": "user", ...}, {"role": "assistant", ...}]` | + +### Fields + +| Field | Type | Required | Description | +| ---------- | ------------------------------ | -------- | ---------------------------------------------------- | +| `id` | string | Yes | Unique test case identifier (e.g., "T001") | +| `input` | string \| Message \| []Message | Yes | Test input | +| `expected` | any | No | Expected output for validation | +| `user` | string | No | User ID for this test case (overridden by `-u` flag) | +| `team` | string | No | Team ID for this test case (overridden by `-t` flag) | +| `metadata` | map | No | Additional metadata for the test case | +| `skip` | bool | No | Skip this test case | +| `timeout` | string | No | Override timeout (e.g., "30s", "1m") | + +### Environment Override Priority + +The test environment (user/team) is determined by the following priority (highest first): + +1. **Command line flags** (`-u`, `-t`): Global override for all test cases +2. **Test case fields** (`user`, `team`): Per-test case configuration +3. **Default values**: "test-user", "test-team" + +Example: + +```bash +# All tests run as "admin" user in "prod-team", regardless of test case settings +yao agent test -i tests/inputs.jsonl -u admin -t prod-team -o report.json +``` + +```jsonl +# T001 uses default user/team +{"id": "T001", "input": "Hello"} + +# T002 uses specific user/team (unless overridden by -u/-t flags) +{"id": "T002", "input": "Admin action", "user": "admin", "team": "admin-team"} + +# T003 uses specific user only, team uses default +{"id": "T003", "input": "User specific test", "user": "special-user"} +``` + +## Output Format + +### Default: JSONL (without `-r` flag) + +By default (without `-r` flag), the output is JSONL format - one JSON object per line, suitable for streaming and CI integration: + +```jsonl +{"type": "start", "timestamp": "2024-12-17T10:00:00Z", "agent_id": "keyword", "total_cases": 42} +{"type": "result", "id": "T001", "status": "passed", "duration_ms": 234, "output": {"keywords": ["AI", "ML"]}} +{"type": "result", "id": "T002", "status": "passed", "duration_ms": 189, "output": {"keywords": ["cloud"]}} +{"type": "result", "id": "T003", "status": "failed", "duration_ms": 0, "error": "timeout after 30s"} +{"type": "summary", "total": 42, "passed": 40, "failed": 2, "duration_ms": 12345} +``` + +This format is: + +- **Streamable**: Results are output as they complete +- **Parseable**: Each line is valid JSON, easy to process with `jq` or scripts +- **CI-friendly**: Exit code indicates pass/fail status + +### Custom Report (with `-r` flag) + +```json +{ + "summary": { + "total": 42, + "passed": 40, + "failed": 2, + "skipped": 0, + "duration_ms": 12345, + "agent_id": "keyword", + "connector": "deepseek.v3", + "runs_per_case": 1, + "overall_pass_rate": 95.2 + }, + "environment": { + "user_id": "test-user", + "team_id": "test-team", + "locale": "en-us" + }, + "results": [ + { + "id": "T001", + "status": "passed", + "input": "...", + "output": { "keywords": ["AI", "machine learning"] }, + "expected": null, + "duration_ms": 234, + "error": null + } + ], + "metadata": { + "started_at": "2024-12-17T10:00:00Z", + "completed_at": "2024-12-17T10:00:12Z", + "version": "0.10.5" + } +} +``` + +### HTML Report + +Beautiful, interactive HTML report with: + +- Summary statistics (pass/fail/skip counts, duration) +- Stability charts (when runs > 1) +- Filterable test results table +- Expandable input/output details +- Error highlighting +- Export options + +### Markdown Report + +```markdown +# Agent Test Report + +## Summary + +| Metric | Value | +| --------- | ----------- | +| Agent | keyword | +| Connector | deepseek.v3 | +| Total | 42 | +| Passed | 40 | +| Failed | 2 | +| Pass Rate | 95.2% | +| Duration | 12.3s | + +## Environment + +| Setting | Value | +| ------- | --------- | +| User | test-user | +| Team | test-team | +| Locale | en-us | + +## Results + +### โœ… T001 - Passed (234ms) + +... +``` + +## Architecture + +``` +agent/test/ +โ”œโ”€โ”€ DESIGN.md # This file +โ”œโ”€โ”€ types.go # Core types and interfaces +โ”œโ”€โ”€ interfaces.go # Runner and Reporter interfaces +โ”œโ”€โ”€ runner.go # Test runner implementation +โ”œโ”€โ”€ loader.go # Test case loader +โ”œโ”€โ”€ resolver.go # Agent resolver +โ”œโ”€โ”€ environment.go # Test environment setup +โ”œโ”€โ”€ stability.go # Stability analysis +โ””โ”€โ”€ reporter/ + โ”œโ”€โ”€ json.go # JSON reporter + โ”œโ”€โ”€ html.go # HTML reporter + โ”œโ”€โ”€ markdown.go # Markdown reporter + โ””โ”€โ”€ agent.go # Agent-based custom reporter +``` + +## Core Components + +### 1. TestCase + +Represents a single test case loaded from JSONL. + +### 2. TestResult + +Represents the result of running a single test case. + +### 3. TestReport + +Represents the complete test report with summary and results. + +### 4. Runner + +Executes test cases against an agent: + +- Loads test cases from JSONL +- Resolves agent from path or explicit ID +- Creates test context with environment +- Executes each test case (optionally multiple runs) +- Collects results and stability metrics + +### 5. Reporter + +Generates reports in various formats. The format is determined by the `-o` file extension: + +| Extension | Format | Description | +| --------- | -------- | -------------------------- | +| `.jsonl` | JSONL | Streaming, line-by-line | +| `.json` | JSON | Full structured report | +| `.md` | Markdown | Human-readable with tables | +| `.html` | HTML | Interactive web report | + +#### Custom Reporter Agent (`-r` flag) + +When `-r ` is specified, the framework calls the specified agent to generate the report: + +1. Test execution completes, `TestReport` is generated +2. Framework calls the reporter agent with input: + ```json + { + "report": { + /* TestReport object */ + }, + "format": "html", + "options": { "verbose": true } + } + ``` +3. Agent processes the report and returns formatted content +4. Framework writes the returned content to the output file + +Example usage: + +```bash +# Use custom reporter agent to generate a beautiful HTML report +yao agent test -i tests/inputs.jsonl -r report.beautiful -o report.html + +# Use custom reporter agent to generate Slack-formatted summary +yao agent test -i tests/inputs.jsonl -r report.slack -o summary.txt +``` + +This allows for fully customizable report generation using AI agents + +## Configuration + +### Test Options + +```go +type Options struct { + // Input/Output + InputFile string // Path to inputs.jsonl + OutputFile string // Path to output report + + // Agent Selection + AgentID string // Explicit agent ID (optional) + Connector string // Override connector (optional) + + // Test Environment + UserID string // Test user ID (-u flag) + TeamID string // Test team ID (-t flag) + Locale string // Locale (default: "en-us") + + // Execution + Timeout time.Duration // Default timeout per test + Parallel int // Number of parallel tests (default: 1) + Runs int // Number of runs per test case (default: 1) + + // Reporting + ReporterID string // Reporter agent ID for custom report + + // Behavior + Verbose bool // Verbose output + FailFast bool // Stop on first failure +} +``` + +## Exit Codes + +| Code | Description | +| ---- | ------------------- | +| 0 | All tests passed | +| 1 | Some tests failed | +| 2 | Configuration error | +| 3 | Runtime error | + +## CI Integration + +### GitHub Actions Example + +```yaml +- name: Run Agent Tests + run: | + yao agent test -i assistants/keyword/tests/inputs.jsonl \ + -u ci-user -t ci-team \ + --runs 3 \ + -o report.json + +- name: Check Stability + run: | + # Fail if any test has pass rate below 80% + jq -e '.results | all(.pass_rate >= 80)' report.json + +- name: Upload Test Report + uses: actions/upload-artifact@v3 + with: + name: agent-test-report + path: report.json +``` + +### Exit Code Handling + +The command exits with code 1 if any tests fail, making it easy to integrate with CI pipelines. + +## Future Enhancements + +1. **Snapshot Testing**: Compare outputs against saved snapshots +2. **Fuzzing**: Generate random inputs for robustness testing +3. **Coverage**: Track which agent code paths are exercised +4. **Benchmarking**: Performance metrics and regression detection +5. **Diff Reports**: Compare results between runs +6. **Flaky Test Detection**: Automatic identification of unstable tests +7. **Test Prioritization**: Run most important/failing tests first diff --git a/agent/test/context.go b/agent/test/context.go new file mode 100644 index 00000000..244e7bd5 --- /dev/null +++ b/agent/test/context.go @@ -0,0 +1,63 @@ +package test + +import ( + stdContext "context" + + "github.com/yaoapp/yao/agent/context" + "github.com/yaoapp/yao/agent/output/message" + "github.com/yaoapp/yao/openapi/oauth/types" +) + +// NewTestContext creates a new context for testing +// This is similar to newAgentNextTestContext in agent_next_test.go +// but configurable via Environment +func NewTestContext(chatID, assistantID string, env *Environment) *context.Context { + // Build authorized info from environment + authorized := &types.AuthorizedInfo{ + Subject: env.UserID, + UserID: env.UserID, + TenantID: env.TeamID, + } + + // Create context with standard initialization + ctx := context.New(stdContext.Background(), authorized, chatID) + ctx.ID = chatID + ctx.AssistantID = assistantID + ctx.Locale = env.Locale + ctx.Client = context.Client{ + Type: env.ClientType, + UserAgent: "yao-agent-test/1.0", + IP: env.ClientIP, + } + ctx.Referer = env.Referer + ctx.Accept = context.AcceptStandard + ctx.IDGenerator = message.NewIDGenerator() + ctx.Metadata = make(map[string]interface{}) + + // Initialize interrupt controller + ctx.Interrupt = context.NewInterruptController() + + // Close the default logger created by context.New() and use noop logger + // to suppress LLM debug output during tests + if ctx.Logger != nil { + ctx.Logger.Close() + } + ctx.Logger = context.Noop() + + return ctx +} + +// NewTestContextFromOptions creates a test context from test options and test case +func NewTestContextFromOptions(chatID, assistantID string, opts *Options, tc *Case) *context.Context { + // Get environment from test case (with options override) + env := tc.GetEnvironment(opts) + return NewTestContext(chatID, assistantID, env) +} + +// GenerateChatID generates a unique chat ID for testing +func GenerateChatID(testID string, runNumber int) string { + if runNumber > 1 { + return "test-" + testID + "-run" + string(rune('0'+runNumber)) + } + return "test-" + testID +} diff --git a/agent/test/input.go b/agent/test/input.go new file mode 100644 index 00000000..32df9768 --- /dev/null +++ b/agent/test/input.go @@ -0,0 +1,230 @@ +package test + +import ( + "fmt" + + jsoniter "github.com/json-iterator/go" + "github.com/yaoapp/yao/agent/context" +) + +// ParseInput converts various input formats to []context.Message +// Supported formats: +// - string: converted to single user message +// - map (Message): single message with role and content +// - []interface{} ([]Message): array of messages (conversation history) +func ParseInput(input interface{}) ([]context.Message, error) { + if input == nil { + return nil, fmt.Errorf("input is nil") + } + + switch v := input.(type) { + case string: + // Simple string input -> single user message + return []context.Message{ + { + Role: context.RoleUser, + Content: v, + }, + }, nil + + case map[string]interface{}: + // Single message object + msg, err := parseMessageMap(v) + if err != nil { + return nil, fmt.Errorf("failed to parse message: %w", err) + } + return []context.Message{*msg}, nil + + case []interface{}: + // Array of messages (conversation history) + messages := make([]context.Message, 0, len(v)) + for i, item := range v { + switch m := item.(type) { + case map[string]interface{}: + msg, err := parseMessageMap(m) + if err != nil { + return nil, fmt.Errorf("failed to parse message at index %d: %w", i, err) + } + messages = append(messages, *msg) + default: + return nil, fmt.Errorf("invalid message type at index %d: expected object, got %T", i, item) + } + } + return messages, nil + + default: + return nil, fmt.Errorf("unsupported input type: %T", input) + } +} + +// parseMessageMap converts a map to context.Message +func parseMessageMap(m map[string]interface{}) (*context.Message, error) { + msg := &context.Message{} + + // Parse role (required) + if role, ok := m["role"].(string); ok { + msg.Role = context.MessageRole(role) + } else { + // Default to user role if not specified + msg.Role = context.RoleUser + } + + // Parse content (required) + if content, ok := m["content"]; ok { + msg.Content = content + } else { + return nil, fmt.Errorf("message missing 'content' field") + } + + // Parse optional name + if name, ok := m["name"].(string); ok { + msg.Name = &name + } + + // Parse optional tool_call_id (for tool messages) + if toolCallID, ok := m["tool_call_id"].(string); ok { + msg.ToolCallID = &toolCallID + } + + // Parse optional tool_calls (for assistant messages) + if toolCalls, ok := m["tool_calls"].([]interface{}); ok { + msg.ToolCalls = make([]context.ToolCall, 0, len(toolCalls)) + for _, tc := range toolCalls { + if tcMap, ok := tc.(map[string]interface{}); ok { + toolCall, err := parseToolCall(tcMap) + if err != nil { + return nil, fmt.Errorf("failed to parse tool_call: %w", err) + } + msg.ToolCalls = append(msg.ToolCalls, *toolCall) + } + } + } + + // Parse optional refusal (for assistant messages) + if refusal, ok := m["refusal"].(string); ok { + msg.Refusal = &refusal + } + + return msg, nil +} + +// parseToolCall converts a map to context.ToolCall +func parseToolCall(m map[string]interface{}) (*context.ToolCall, error) { + tc := &context.ToolCall{} + + if id, ok := m["id"].(string); ok { + tc.ID = id + } + + if typ, ok := m["type"].(string); ok { + tc.Type = context.ToolCallType(typ) + } else { + tc.Type = context.ToolTypeFunction + } + + if fn, ok := m["function"].(map[string]interface{}); ok { + if name, ok := fn["name"].(string); ok { + tc.Function.Name = name + } + if args, ok := fn["arguments"].(string); ok { + tc.Function.Arguments = args + } else if args, ok := fn["arguments"].(map[string]interface{}); ok { + // Convert map to JSON string + argsBytes, err := jsoniter.Marshal(args) + if err != nil { + return nil, fmt.Errorf("failed to marshal arguments: %w", err) + } + tc.Function.Arguments = string(argsBytes) + } + } + + return tc, nil +} + +// ExtractTextContent extracts text content from various content formats +// Used for display in reports +func ExtractTextContent(content interface{}) string { + if content == nil { + return "" + } + + switch v := content.(type) { + case string: + return v + + case []interface{}: + // ContentPart array + var texts []string + for _, part := range v { + if partMap, ok := part.(map[string]interface{}); ok { + if partMap["type"] == "text" { + if text, ok := partMap["text"].(string); ok { + texts = append(texts, text) + } + } + } + } + if len(texts) > 0 { + result := texts[0] + for i := 1; i < len(texts); i++ { + result += "\n" + texts[i] + } + return result + } + return fmt.Sprintf("[%d content parts]", len(v)) + + case map[string]interface{}: + // Single ContentPart or Message + if v["type"] == "text" { + if text, ok := v["text"].(string); ok { + return text + } + } + if content, ok := v["content"]; ok { + return ExtractTextContent(content) + } + return fmt.Sprintf("%v", v) + + default: + return fmt.Sprintf("%v", v) + } +} + +// SummarizeInput creates a short summary of the input for display +func SummarizeInput(input interface{}, maxLen int) string { + text := "" + + switch v := input.(type) { + case string: + text = v + + case map[string]interface{}: + if content, ok := v["content"]; ok { + text = ExtractTextContent(content) + } + + case []interface{}: + // Get the last user message for summary + for i := len(v) - 1; i >= 0; i-- { + if msg, ok := v[i].(map[string]interface{}); ok { + if msg["role"] == "user" { + if content, ok := msg["content"]; ok { + text = ExtractTextContent(content) + break + } + } + } + } + if text == "" && len(v) > 0 { + text = fmt.Sprintf("[%d messages]", len(v)) + } + + default: + text = fmt.Sprintf("%v", v) + } + + if maxLen > 0 && len(text) > maxLen { + return text[:maxLen-3] + "..." + } + return text +} diff --git a/agent/test/interfaces.go b/agent/test/interfaces.go new file mode 100644 index 00000000..a5996010 --- /dev/null +++ b/agent/test/interfaces.go @@ -0,0 +1,125 @@ +package test + +import ( + "context" + "io" +) + +// Runner is the interface for test execution +type Runner interface { + // Run executes all test cases and returns the report + Run(ctx context.Context) (*Report, error) + + // RunCase executes a single test case + RunCase(ctx context.Context, tc *Case) (*Result, error) + + // GetAgentInfo returns information about the agent being tested + GetAgentInfo() *AgentInfo + + // SetProgressCallback sets a callback for progress updates + SetProgressCallback(callback ProgressCallback) +} + +// ProgressCallback is called during test execution to report progress +// Parameters: +// - current: current test index (1-based) +// - total: total number of tests +// - result: result of the current test (nil if not yet completed) +type ProgressCallback func(current, total int, result *Result) + +// Reporter is the interface for generating test reports +type Reporter interface { + // Generate generates a report from the test results + Generate(report *Report) error + + // Write writes the report to the given writer + Write(report *Report, w io.Writer) error +} + +// Loader is the interface for loading test cases +type Loader interface { + // Load loads test cases from the input source + Load() ([]*Case, error) + + // LoadFile loads test cases from a JSONL file + LoadFile(path string) ([]*Case, error) +} + +// Resolver is the interface for resolving agent information +type Resolver interface { + // Resolve resolves the agent from options + // Priority: explicit AgentID > path-based detection + Resolve(opts *Options) (*AgentInfo, error) + + // ResolveFromPath resolves the agent by traversing up from the input file path + ResolveFromPath(inputPath string) (*AgentInfo, error) +} + +// Validator is the interface for validating test outputs +type Validator interface { + // Validate compares actual output against expected output + // Returns nil if validation passes, error otherwise + Validate(actual, expected interface{}) error + + // ValidateJSON validates JSON outputs with flexible comparison + ValidateJSON(actual, expected interface{}) error +} + +// OutputAdapter adapts agent output to a comparable format +type OutputAdapter interface { + // Adapt transforms the raw agent output to a normalized format + Adapt(output interface{}) (interface{}, error) +} + +// RunnerFactory creates Runner instances +type RunnerFactory interface { + // Create creates a new Runner with the given options + Create(opts *Options) (Runner, error) +} + +// ReporterFactory creates Reporter instances +type ReporterFactory interface { + // Create creates a new Reporter for the given format + Create(format OutputFormat) (Reporter, error) + + // CreateFromPath creates a Reporter based on output file extension + CreateFromPath(outputPath string) (Reporter, error) +} + +// Hook allows customization of test execution +type Hook interface { + // BeforeAll is called before any tests run + BeforeAll(ctx context.Context, cases []*Case) error + + // BeforeEach is called before each test case + BeforeEach(ctx context.Context, tc *Case) error + + // AfterEach is called after each test case + AfterEach(ctx context.Context, tc *Case, result *Result) error + + // AfterAll is called after all tests complete + AfterAll(ctx context.Context, report *Report) error +} + +// DefaultHook provides a no-op implementation of Hook +type DefaultHook struct{} + +// BeforeAll implements Hook +func (h *DefaultHook) BeforeAll(ctx context.Context, cases []*Case) error { + return nil +} + +// BeforeEach implements Hook +func (h *DefaultHook) BeforeEach(ctx context.Context, tc *Case) error { + return nil +} + +// AfterEach implements Hook +func (h *DefaultHook) AfterEach(ctx context.Context, tc *Case, result *Result) error { + return nil +} + +// AfterAll implements Hook +func (h *DefaultHook) AfterAll(ctx context.Context, report *Report) error { + return nil +} diff --git a/agent/test/loader.go b/agent/test/loader.go new file mode 100644 index 00000000..c648a617 --- /dev/null +++ b/agent/test/loader.go @@ -0,0 +1,141 @@ +package test + +import ( + "bufio" + "fmt" + "os" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" +) + +// JSONLLoader loads test cases from JSONL files +type JSONLLoader struct{} + +// NewLoader creates a new JSONL loader +func NewLoader() Loader { + return &JSONLLoader{} +} + +// Load loads test cases from the default input source +// This is a placeholder - actual implementation would use configured path +func (l *JSONLLoader) Load() ([]*Case, error) { + return nil, fmt.Errorf("Load() requires explicit path, use LoadFile() instead") +} + +// LoadFile loads test cases from a JSONL file +func (l *JSONLLoader) LoadFile(path string) ([]*Case, error) { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("failed to open file %s: %w", path, err) + } + defer file.Close() + + var cases []*Case + scanner := bufio.NewScanner(file) + lineNum := 0 + + // Increase buffer size for long lines + const maxCapacity = 1024 * 1024 // 1MB + buf := make([]byte, maxCapacity) + scanner.Buffer(buf, maxCapacity) + + for scanner.Scan() { + lineNum++ + line := strings.TrimSpace(scanner.Text()) + + // Skip empty lines + if line == "" { + continue + } + + // Skip comments (lines starting with //) + if strings.HasPrefix(line, "//") { + continue + } + + var tc Case + if err := jsoniter.UnmarshalFromString(line, &tc); err != nil { + return nil, fmt.Errorf("failed to parse line %d: %w", lineNum, err) + } + + // Validate required fields + if tc.ID == "" { + return nil, fmt.Errorf("line %d: missing required field 'id'", lineNum) + } + if tc.Input == nil { + return nil, fmt.Errorf("line %d (id=%s): missing required field 'input'", lineNum, tc.ID) + } + + cases = append(cases, &tc) + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error reading file: %w", err) + } + + if len(cases) == 0 { + return nil, fmt.Errorf("no test cases found in %s", path) + } + + return cases, nil +} + +// ValidateTestCases validates a slice of test cases +func ValidateTestCases(cases []*Case) error { + ids := make(map[string]bool) + + for i, tc := range cases { + // Check for duplicate IDs + if ids[tc.ID] { + return fmt.Errorf("duplicate test case ID: %s", tc.ID) + } + ids[tc.ID] = true + + // Validate input can be parsed + if _, err := tc.GetMessages(); err != nil { + return fmt.Errorf("test case %s (index %d): invalid input: %w", tc.ID, i, err) + } + + // Validate timeout format if specified + if tc.Timeout != "" { + // GetTimeout returns a duration, parsing error would return default + // We validate by checking if the string is parseable + if _, err := time.ParseDuration(tc.Timeout); err != nil { + return fmt.Errorf("test case %s: invalid timeout format: %s", tc.ID, tc.Timeout) + } + } + } + + return nil +} + +// FilterTestCases filters test cases based on criteria +func FilterTestCases(cases []*Case, filter func(*Case) bool) []*Case { + var result []*Case + for _, tc := range cases { + if filter(tc) { + result = append(result, tc) + } + } + return result +} + +// FilterSkipped returns test cases that are not skipped +func FilterSkipped(cases []*Case) []*Case { + return FilterTestCases(cases, func(tc *Case) bool { + return !tc.Skip + }) +} + +// FilterByIDs returns test cases matching the given IDs +func FilterByIDs(cases []*Case, ids []string) []*Case { + idSet := make(map[string]bool) + for _, id := range ids { + idSet[id] = true + } + return FilterTestCases(cases, func(tc *Case) bool { + return idSet[tc.ID] + }) +} diff --git a/agent/test/output.go b/agent/test/output.go new file mode 100644 index 00000000..1b3fc51b --- /dev/null +++ b/agent/test/output.go @@ -0,0 +1,316 @@ +package test + +import ( + "fmt" + "strings" + "time" + + "github.com/fatih/color" + jsoniter "github.com/json-iterator/go" +) + +// OutputWriter handles colored console output for test execution +type OutputWriter struct { + verbose bool +} + +// NewOutputWriter creates a new output writer +func NewOutputWriter(verbose bool) *OutputWriter { + return &OutputWriter{verbose: verbose} +} + +// Header prints a header section +func (w *OutputWriter) Header(title string) { + fmt.Println() + color.New(color.FgCyan, color.Bold).Println("โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•") + color.New(color.FgCyan, color.Bold).Printf(" %s\n", title) + color.New(color.FgCyan, color.Bold).Println("โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•") +} + +// SubHeader prints a sub-header +func (w *OutputWriter) SubHeader(title string) { + fmt.Println() + color.New(color.FgWhite, color.Bold).Println("โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€") + color.New(color.FgWhite, color.Bold).Printf(" %s\n", title) + color.New(color.FgWhite, color.Bold).Println("โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€") +} + +// Info prints an info message +func (w *OutputWriter) Info(format string, args ...interface{}) { + color.New(color.FgBlue).Printf("โ„น ") + fmt.Printf(format+"\n", args...) +} + +// Success prints a success message +func (w *OutputWriter) Success(format string, args ...interface{}) { + color.New(color.FgGreen).Printf("โœ“ ") + fmt.Printf(format+"\n", args...) +} + +// Error prints an error message +func (w *OutputWriter) Error(format string, args ...interface{}) { + color.New(color.FgRed).Printf("โœ— ") + fmt.Printf(format+"\n", args...) +} + +// Warning prints a warning message +func (w *OutputWriter) Warning(format string, args ...interface{}) { + color.New(color.FgYellow).Printf("โš  ") + fmt.Printf(format+"\n", args...) +} + +// Skip prints a skip message +func (w *OutputWriter) Skip(format string, args ...interface{}) { + color.New(color.FgYellow).Printf("โ—‹ ") + fmt.Printf(format+"\n", args...) +} + +// Verbose prints a verbose message (only if verbose mode is enabled) +func (w *OutputWriter) Verbose(format string, args ...interface{}) { + if w.verbose { + color.New(color.FgHiBlack).Printf(" โ”‚ ") + fmt.Printf(format+"\n", args...) + } +} + +// TestStart prints test case start +func (w *OutputWriter) TestStart(id string, input string, runNum int) { + inputPreview := truncateString(input, 50) + if runNum > 1 { + color.New(color.FgWhite).Printf("โ–บ [%s] Run %d: ", id, runNum) + } else { + color.New(color.FgWhite).Printf("โ–บ [%s] ", id) + } + color.New(color.FgHiBlack).Printf("%s", inputPreview) + fmt.Print(" ") +} + +// TestResult prints test case result +func (w *OutputWriter) TestResult(status Status, duration time.Duration) { + switch status { + case StatusPassed: + color.New(color.FgGreen, color.Bold).Printf("PASSED") + case StatusFailed: + color.New(color.FgRed, color.Bold).Printf("FAILED") + case StatusSkipped: + color.New(color.FgYellow).Printf("SKIPPED") + case StatusError: + color.New(color.FgRed, color.Bold).Printf("ERROR") + case StatusTimeout: + color.New(color.FgRed).Printf("TIMEOUT") + } + color.New(color.FgHiBlack).Printf(" (%s)\n", formatDuration(duration)) +} + +// TestError prints test error details +func (w *OutputWriter) TestError(err string) { + color.New(color.FgRed).Printf(" โ””โ”€ %s\n", err) +} + +// TestOutput prints test output (verbose mode) +func (w *OutputWriter) TestOutput(output string) { + if w.verbose && output != "" { + outputPreview := truncateString(output, 100) + color.New(color.FgHiBlack).Printf(" โ””โ”€ Output: %s\n", outputPreview) + } +} + +// Progress prints progress information +func (w *OutputWriter) Progress(current, total int) { + percentage := float64(current) / float64(total) * 100 + color.New(color.FgHiBlack).Printf("\r Progress: %d/%d (%.0f%%)", current, total, percentage) +} + +// Summary prints the test summary +func (w *OutputWriter) Summary(summary *Summary, duration time.Duration) { + w.SubHeader("Summary") + + // Agent info + color.New(color.FgWhite).Printf(" Agent: ") + color.New(color.FgCyan).Printf("%s\n", summary.AgentID) + + if summary.Connector != "" { + color.New(color.FgWhite).Printf(" Connector: ") + color.New(color.FgCyan).Printf("%s\n", summary.Connector) + } + + // Results + color.New(color.FgWhite).Printf(" Total: ") + fmt.Printf("%d\n", summary.Total) + + color.New(color.FgWhite).Printf(" Passed: ") + if summary.Passed > 0 { + color.New(color.FgGreen).Printf("%d\n", summary.Passed) + } else { + fmt.Printf("%d\n", summary.Passed) + } + + color.New(color.FgWhite).Printf(" Failed: ") + if summary.Failed > 0 { + color.New(color.FgRed).Printf("%d\n", summary.Failed) + } else { + fmt.Printf("%d\n", summary.Failed) + } + + if summary.Skipped > 0 { + color.New(color.FgWhite).Printf(" Skipped: ") + color.New(color.FgYellow).Printf("%d\n", summary.Skipped) + } + + if summary.Errors > 0 { + color.New(color.FgWhite).Printf(" Errors: ") + color.New(color.FgRed).Printf("%d\n", summary.Errors) + } + + if summary.Timeouts > 0 { + color.New(color.FgWhite).Printf(" Timeouts: ") + color.New(color.FgRed).Printf("%d\n", summary.Timeouts) + } + + // Pass rate + passRate := float64(0) + if summary.Total > 0 { + passRate = float64(summary.Passed) / float64(summary.Total) * 100 + } + color.New(color.FgWhite).Printf(" Pass Rate: ") + if passRate == 100 { + color.New(color.FgGreen, color.Bold).Printf("%.1f%%\n", passRate) + } else if passRate >= 80 { + color.New(color.FgYellow).Printf("%.1f%%\n", passRate) + } else { + color.New(color.FgRed).Printf("%.1f%%\n", passRate) + } + + // Duration + color.New(color.FgWhite).Printf(" Duration: ") + fmt.Printf("%s\n", formatDuration(duration)) + + // Stability info (if runs > 1) + if summary.RunsPerCase > 1 { + fmt.Println() + color.New(color.FgWhite, color.Bold).Println(" Stability Analysis:") + color.New(color.FgWhite).Printf(" Runs/Case: %d\n", summary.RunsPerCase) + color.New(color.FgWhite).Printf(" Total Runs: %d\n", summary.TotalRuns) + color.New(color.FgWhite).Printf(" Stable Cases: ") + if summary.StableCases == summary.Total { + color.New(color.FgGreen).Printf("%d\n", summary.StableCases) + } else { + color.New(color.FgYellow).Printf("%d\n", summary.StableCases) + } + color.New(color.FgWhite).Printf(" Unstable: ") + if summary.UnstableCases > 0 { + color.New(color.FgRed).Printf("%d\n", summary.UnstableCases) + } else { + fmt.Printf("%d\n", summary.UnstableCases) + } + } +} + +// OutputFile prints the output file path +func (w *OutputWriter) OutputFile(path string) { + fmt.Println() + color.New(color.FgWhite).Printf(" Output: ") + color.New(color.FgCyan).Printf("%s\n", path) +} + +// FinalResult prints the final result banner +func (w *OutputWriter) FinalResult(passed bool) { + fmt.Println() + if passed { + color.New(color.FgGreen, color.Bold).Println("โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•") + color.New(color.FgGreen, color.Bold).Println(" โœจ ALL TESTS PASSED โœจ") + color.New(color.FgGreen, color.Bold).Println("โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•") + } else { + color.New(color.FgRed, color.Bold).Println("โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•") + color.New(color.FgRed, color.Bold).Println(" โŒ TESTS FAILED") + color.New(color.FgRed, color.Bold).Println("โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•") + } + fmt.Println() +} + +// DirectOutput prints the agent output directly (for development mode) +func (w *OutputWriter) DirectOutput(output interface{}) { + if output == nil { + return + } + + // Try to format as JSON if it's a complex type + switch v := output.(type) { + case string: + fmt.Println(v) + case map[string]interface{}, []interface{}: + // Pretty print JSON + jsonBytes, err := jsoniter.MarshalIndent(v, "", " ") + if err != nil { + fmt.Printf("%v\n", output) + } else { + fmt.Println(string(jsonBytes)) + } + default: + // Try to marshal as JSON + jsonBytes, err := jsoniter.MarshalIndent(output, "", " ") + if err != nil { + fmt.Printf("%v\n", output) + } else { + fmt.Println(string(jsonBytes)) + } + } +} + +// StabilityResult prints stability analysis result for a test case +func (w *OutputWriter) StabilityResult(sr *StabilityResult) { + color.New(color.FgWhite).Printf(" [%s] ", sr.ID) + + // Pass rate + if sr.PassRate == 100 { + color.New(color.FgGreen).Printf("%.0f%%", sr.PassRate) + } else if sr.PassRate >= 80 { + color.New(color.FgYellow).Printf("%.0f%%", sr.PassRate) + } else { + color.New(color.FgRed).Printf("%.0f%%", sr.PassRate) + } + + // Classification + color.New(color.FgHiBlack).Printf(" (%d/%d) ", sr.Passed, sr.Runs) + + switch sr.StabilityClass { + case StabilityStable: + color.New(color.FgGreen).Printf("Stable") + case StabilityMostlyStable: + color.New(color.FgYellow).Printf("Mostly Stable") + case StabilityUnstable: + color.New(color.FgRed).Printf("Unstable") + case StabilityHighlyUnstable: + color.New(color.FgRed, color.Bold).Printf("Highly Unstable") + } + + // Timing + color.New(color.FgHiBlack).Printf(" avg:%.0fms\n", sr.AvgDurationMs) +} + +// Helper functions + +func truncateString(s string, maxLen int) string { + // Remove newlines and extra spaces + s = strings.ReplaceAll(s, "\n", " ") + s = strings.ReplaceAll(s, "\r", "") + s = strings.Join(strings.Fields(s), " ") + + if len(s) <= maxLen { + return s + } + return s[:maxLen-3] + "..." +} + +func formatDuration(d time.Duration) string { + if d < time.Millisecond { + return fmt.Sprintf("%dยตs", d.Microseconds()) + } + if d < time.Second { + return fmt.Sprintf("%dms", d.Milliseconds()) + } + if d < time.Minute { + return fmt.Sprintf("%.1fs", d.Seconds()) + } + return fmt.Sprintf("%.1fm", d.Minutes()) +} diff --git a/agent/test/reporter.go b/agent/test/reporter.go new file mode 100644 index 00000000..7e486249 --- /dev/null +++ b/agent/test/reporter.go @@ -0,0 +1,713 @@ +package test + +import ( + "bufio" + "fmt" + "html/template" + "io" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + "github.com/yaoapp/yao/agent/caller" + "github.com/yaoapp/yao/agent/context" +) + +// JSONLReporter generates JSONL format reports (default) +type JSONLReporter struct{} + +// NewJSONLReporter creates a new JSONL reporter +func NewJSONLReporter() *JSONLReporter { + return &JSONLReporter{} +} + +// Generate generates a JSONL report (writes to stdout or file) +func (r *JSONLReporter) Generate(report *Report) error { + return nil // JSONL is written during test execution +} + +// Write writes the report in JSONL format +func (r *JSONLReporter) Write(report *Report, w io.Writer) error { + writer := bufio.NewWriter(w) + defer writer.Flush() + + // Start event + startEvent := map[string]interface{}{ + "type": "start", + "timestamp": report.Metadata.StartedAt.Format(time.RFC3339), + "agent_id": report.Summary.AgentID, + "total_cases": report.Summary.Total, + } + if err := writeJSONLineToWriter(writer, startEvent); err != nil { + return err + } + + // Result events + if report.Results != nil { + for _, result := range report.Results { + resultEvent := map[string]interface{}{ + "type": "result", + "id": result.ID, + "status": result.Status, + "duration_ms": result.DurationMs, + } + if result.Output != nil { + resultEvent["output"] = result.Output + } + if result.Error != "" { + resultEvent["error"] = result.Error + } + if err := writeJSONLineToWriter(writer, resultEvent); err != nil { + return err + } + } + } + + // Stability results + if report.StabilityResults != nil { + for _, sr := range report.StabilityResults { + stabilityEvent := map[string]interface{}{ + "type": "stability", + "id": sr.ID, + "runs": sr.Runs, + "passed": sr.Passed, + "failed": sr.Failed, + "pass_rate": sr.PassRate, + "stable": sr.Stable, + "stability_class": sr.StabilityClass, + "avg_duration_ms": sr.AvgDurationMs, + } + if err := writeJSONLineToWriter(writer, stabilityEvent); err != nil { + return err + } + } + } + + // Summary event + summaryEvent := map[string]interface{}{ + "type": "summary", + "total": report.Summary.Total, + "passed": report.Summary.Passed, + "failed": report.Summary.Failed, + "skipped": report.Summary.Skipped, + "errors": report.Summary.Errors, + "timeouts": report.Summary.Timeouts, + "duration_ms": report.Summary.DurationMs, + } + if report.Summary.RunsPerCase > 1 { + summaryEvent["runs_per_case"] = report.Summary.RunsPerCase + summaryEvent["total_runs"] = report.Summary.TotalRuns + summaryEvent["overall_pass_rate"] = report.Summary.OverallPassRate + summaryEvent["stable_cases"] = report.Summary.StableCases + summaryEvent["unstable_cases"] = report.Summary.UnstableCases + } + return writeJSONLineToWriter(writer, summaryEvent) +} + +// writeJSONLineToWriter writes a JSON line to the writer +func writeJSONLineToWriter(writer *bufio.Writer, data interface{}) error { + line, err := jsoniter.Marshal(data) + if err != nil { + return err + } + _, err = writer.Write(line) + if err != nil { + return err + } + _, err = writer.WriteString("\n") + return err +} + +// JSONReporter generates full JSON format reports +type JSONReporter struct{} + +// NewJSONReporter creates a new JSON reporter +func NewJSONReporter() *JSONReporter { + return &JSONReporter{} +} + +// Generate generates a JSON report +func (r *JSONReporter) Generate(report *Report) error { + return nil +} + +// Write writes the report in JSON format +func (r *JSONReporter) Write(report *Report, w io.Writer) error { + encoder := jsoniter.NewEncoder(w) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} + +// MarkdownReporter generates Markdown format reports +type MarkdownReporter struct{} + +// NewMarkdownReporter creates a new Markdown reporter +func NewMarkdownReporter() *MarkdownReporter { + return &MarkdownReporter{} +} + +// Generate generates a Markdown report +func (r *MarkdownReporter) Generate(report *Report) error { + return nil +} + +// Write writes the report in Markdown format +func (r *MarkdownReporter) Write(report *Report, w io.Writer) error { + var sb strings.Builder + + // Header + sb.WriteString("# Agent Test Report\n\n") + + // Summary + sb.WriteString("## Summary\n\n") + sb.WriteString("| Metric | Value |\n") + sb.WriteString("| ------ | ----- |\n") + sb.WriteString(fmt.Sprintf("| Agent | %s |\n", report.Summary.AgentID)) + if report.Summary.Connector != "" { + sb.WriteString(fmt.Sprintf("| Connector | %s |\n", report.Summary.Connector)) + } + sb.WriteString(fmt.Sprintf("| Total | %d |\n", report.Summary.Total)) + sb.WriteString(fmt.Sprintf("| Passed | %d |\n", report.Summary.Passed)) + sb.WriteString(fmt.Sprintf("| Failed | %d |\n", report.Summary.Failed)) + if report.Summary.Skipped > 0 { + sb.WriteString(fmt.Sprintf("| Skipped | %d |\n", report.Summary.Skipped)) + } + if report.Summary.Errors > 0 { + sb.WriteString(fmt.Sprintf("| Errors | %d |\n", report.Summary.Errors)) + } + if report.Summary.Timeouts > 0 { + sb.WriteString(fmt.Sprintf("| Timeouts | %d |\n", report.Summary.Timeouts)) + } + + passRate := float64(0) + if report.Summary.Total > 0 { + passRate = float64(report.Summary.Passed) / float64(report.Summary.Total) * 100 + } + sb.WriteString(fmt.Sprintf("| Pass Rate | %.1f%% |\n", passRate)) + sb.WriteString(fmt.Sprintf("| Duration | %dms |\n", report.Summary.DurationMs)) + sb.WriteString("\n") + + // Environment + if report.Environment != nil { + sb.WriteString("## Environment\n\n") + sb.WriteString("| Setting | Value |\n") + sb.WriteString("| ------- | ----- |\n") + sb.WriteString(fmt.Sprintf("| User | %s |\n", report.Environment.UserID)) + sb.WriteString(fmt.Sprintf("| Team | %s |\n", report.Environment.TeamID)) + sb.WriteString(fmt.Sprintf("| Locale | %s |\n", report.Environment.Locale)) + sb.WriteString("\n") + } + + // Results + sb.WriteString("## Results\n\n") + + if report.Results != nil { + for _, result := range report.Results { + statusIcon := "โœ…" + switch result.Status { + case StatusFailed: + statusIcon = "โŒ" + case StatusError: + statusIcon = "๐Ÿ’ฅ" + case StatusTimeout: + statusIcon = "โฑ๏ธ" + case StatusSkipped: + statusIcon = "โญ๏ธ" + } + + sb.WriteString(fmt.Sprintf("### %s %s - %s (%dms)\n\n", statusIcon, result.ID, result.Status, result.DurationMs)) + + if result.Error != "" { + sb.WriteString(fmt.Sprintf("**Error:** %s\n\n", result.Error)) + } + } + } + + // Stability results + if report.StabilityResults != nil { + sb.WriteString("## Stability Analysis\n\n") + sb.WriteString("| ID | Pass Rate | Runs | Status | Avg Duration |\n") + sb.WriteString("| -- | --------- | ---- | ------ | ------------ |\n") + + for _, sr := range report.StabilityResults { + status := string(sr.StabilityClass) + sb.WriteString(fmt.Sprintf("| %s | %.0f%% | %d/%d | %s | %.0fms |\n", + sr.ID, sr.PassRate, sr.Passed, sr.Runs, status, sr.AvgDurationMs)) + } + sb.WriteString("\n") + } + + // Metadata + sb.WriteString("## Metadata\n\n") + sb.WriteString(fmt.Sprintf("- **Started:** %s\n", report.Metadata.StartedAt.Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("- **Completed:** %s\n", report.Metadata.CompletedAt.Format(time.RFC3339))) + if report.Metadata.InputFile != "" { + sb.WriteString(fmt.Sprintf("- **Input File:** %s\n", report.Metadata.InputFile)) + } + if report.Metadata.OutputFile != "" { + sb.WriteString(fmt.Sprintf("- **Output File:** %s\n", report.Metadata.OutputFile)) + } + + _, err := w.Write([]byte(sb.String())) + return err +} + +// HTMLReporter generates HTML format reports +type HTMLReporter struct{} + +// NewHTMLReporter creates a new HTML reporter +func NewHTMLReporter() *HTMLReporter { + return &HTMLReporter{} +} + +// Generate generates an HTML report +func (r *HTMLReporter) Generate(report *Report) error { + return nil +} + +// Write writes the report in HTML format +func (r *HTMLReporter) Write(report *Report, w io.Writer) error { + tmpl, err := template.New("report").Parse(htmlTemplate) + if err != nil { + return fmt.Errorf("failed to parse HTML template: %w", err) + } + + // Calculate pass rate + passRate := float64(0) + if report.Summary.Total > 0 { + passRate = float64(report.Summary.Passed) / float64(report.Summary.Total) * 100 + } + + data := map[string]interface{}{ + "Report": report, + "PassRate": passRate, + } + + return tmpl.Execute(w, data) +} + +// HTML template for reports +const htmlTemplate = ` + + + + + Agent Test Report - {{.Report.Summary.AgentID}} + + + +
+

Agent Test Report

+

{{.Report.Summary.AgentID}} {{if .Report.Summary.Connector}}โ€ข {{.Report.Summary.Connector}}{{end}}

+ +
+
+
{{.Report.Summary.Total}}
+
Total Tests
+
+
+
{{.Report.Summary.Passed}}
+
Passed
+
+
+
{{.Report.Summary.Failed}}
+
Failed
+
+
+
{{printf "%.1f" .PassRate}}%
+
Pass Rate
+
+
+
{{.Report.Summary.DurationMs}}ms
+
Duration
+
+
+ +

Test Results

+ + + + + + + + + + + {{range .Report.Results}} + + + + + + + {{end}} + {{range .Report.StabilityResults}} + + + + + + + {{end}} + +
IDStatusDurationDetails
{{.ID}}{{.Status}}{{.DurationMs}}ms + {{if .Error}}
{{.Error}}
{{end}} +
{{.ID}}{{.StabilityClass}}{{printf "%.0f" .AvgDurationMs}}ms avg{{.Passed}}/{{.Runs}} passed ({{printf "%.0f" .PassRate}}%)
+ +

Metadata

+
+
Started:
{{.Report.Metadata.StartedAt}}
+
Completed:
{{.Report.Metadata.CompletedAt}}
+ {{if .Report.Metadata.InputFile}}
Input:
{{.Report.Metadata.InputFile}}
{{end}} + {{if .Report.Metadata.OutputFile}}
Output:
{{.Report.Metadata.OutputFile}}
{{end}} +
+
+ +` + +// AgentReporter uses a custom agent to generate reports +type AgentReporter struct { + agentID string + format string + verbose bool + ctx *context.Context // Test context for agent call +} + +// NewAgentReporter creates a new agent-based reporter +func NewAgentReporter(agentID, format string, verbose bool) *AgentReporter { + return &AgentReporter{ + agentID: agentID, + format: format, + verbose: verbose, + } +} + +// SetContext sets the context for agent calls +func (r *AgentReporter) SetContext(ctx *context.Context) { + r.ctx = ctx +} + +// Generate generates a report using the agent +func (r *AgentReporter) Generate(report *Report) error { + return nil +} + +// Write writes the report using the agent +func (r *AgentReporter) Write(report *Report, w io.Writer) error { + // Check if AgentGetterFunc is initialized + if caller.AgentGetterFunc == nil { + return fmt.Errorf("AgentGetterFunc not initialized, cannot call reporter agent") + } + + // Get the reporter agent + agent, err := caller.AgentGetterFunc(r.agentID) + if err != nil { + return fmt.Errorf("failed to get reporter agent %s: %w", r.agentID, err) + } + + // Build input for the reporter agent + input := &ReporterInput{ + Report: report, + Format: r.format, + Options: &ReporterOptions{ + Verbose: r.verbose, + IncludeOutputs: r.verbose, + IncludeInputs: r.verbose, + }, + } + + // Convert input to JSON for the agent + inputJSON, err := jsoniter.Marshal(input) + if err != nil { + return fmt.Errorf("failed to marshal reporter input: %w", err) + } + + // Create message for the agent + messages := []context.Message{ + { + Role: context.RoleUser, + Content: string(inputJSON), + }, + } + + // Create context if not provided + ctx := r.ctx + if ctx == nil { + // Create a minimal context for the reporter agent call + ctx = NewTestContext("reporter", r.agentID, NewEnvironment("", "")) + defer ctx.Release() + } + + // Call the agent with skip options (no history, no output) + options := &context.Options{ + Skip: &context.Skip{ + History: true, + Output: true, + }, + } + + result, err := agent.Stream(ctx, messages, options) + if err != nil { + return fmt.Errorf("reporter agent call failed: %w", err) + } + + // Extract content from result + content, err := r.extractContent(result) + if err != nil { + return fmt.Errorf("failed to extract report content: %w", err) + } + + // Write the content to output + _, err = w.Write([]byte(content)) + if err != nil { + return fmt.Errorf("failed to write report: %w", err) + } + + return nil +} + +// extractContent extracts the report content from the agent's response +func (r *AgentReporter) extractContent(result interface{}) (string, error) { + if result == nil { + return "", fmt.Errorf("agent returned nil result") + } + + // Try to convert to map first (context.Response) + switch v := result.(type) { + case string: + return v, nil + + case *context.Response: + // Extract from completion content + if v.Completion != nil && v.Completion.Content != nil { + return r.contentToString(v.Completion.Content) + } + // Try next field + if v.Next != nil { + return r.contentToString(v.Next) + } + return "", fmt.Errorf("no content in response") + + case map[string]interface{}: + // Check for completion.content + if completion, ok := v["completion"].(map[string]interface{}); ok { + if content, ok := completion["content"]; ok { + return r.contentToString(content) + } + } + // Check for next + if next, ok := v["next"]; ok { + return r.contentToString(next) + } + // Check for content directly + if content, ok := v["content"]; ok { + return r.contentToString(content) + } + // Marshal the whole thing + jsonBytes, _ := jsoniter.Marshal(v) + return string(jsonBytes), nil + + default: + // Try to marshal as JSON + jsonBytes, err := jsoniter.Marshal(result) + if err != nil { + return fmt.Sprintf("%v", result), nil + } + return string(jsonBytes), nil + } +} + +// contentToString converts various content types to string +func (r *AgentReporter) contentToString(content interface{}) (string, error) { + switch v := content.(type) { + case string: + return v, nil + case []byte: + return string(v), nil + default: + jsonBytes, err := jsoniter.Marshal(content) + if err != nil { + return fmt.Sprintf("%v", content), nil + } + return string(jsonBytes), nil + } +} + +// GetReporter returns a reporter based on output format +func GetReporter(format OutputFormat) Reporter { + switch format { + case FormatJSON: + return NewJSONReporter() + case FormatHTML: + return NewHTMLReporter() + case FormatMarkdown: + return NewMarkdownReporter() + default: + return NewJSONLReporter() + } +} + +// GetReporterFromPath returns a reporter based on file extension +func GetReporterFromPath(outputPath string) Reporter { + format := GetOutputFormat(outputPath) + return GetReporter(format) +} + +// GetReporterWithAgent returns an agent-based reporter if agentID is specified, +// otherwise returns a built-in reporter based on output format +func GetReporterWithAgent(agentID, outputPath string, verbose bool) Reporter { + if agentID != "" { + format := GetOutputFormat(outputPath) + return NewAgentReporter(agentID, string(format), verbose) + } + return GetReporterFromPath(outputPath) +} diff --git a/agent/test/resolver.go b/agent/test/resolver.go new file mode 100644 index 00000000..9ccf31ee --- /dev/null +++ b/agent/test/resolver.go @@ -0,0 +1,316 @@ +package test + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" +) + +// PathResolver resolves agent information from file paths +type PathResolver struct{} + +// NewResolver creates a new path resolver +func NewResolver() Resolver { + return &PathResolver{} +} + +// Resolve resolves the agent from options +// Priority: explicit AgentID > path-based detection (from input file or cwd) +func (r *PathResolver) Resolve(opts *Options) (*AgentInfo, error) { + // If explicit agent ID is provided, use it + if opts.AgentID != "" { + return r.ResolveByID(opts.AgentID) + } + + // For file mode, resolve from input file path + if opts.InputMode == InputModeFile { + if opts.Input == "" { + return nil, fmt.Errorf("no agent ID or input file specified") + } + return r.ResolveFromPath(opts.Input) + } + + // For message mode, try to resolve from current working directory + return r.ResolveFromCwd() +} + +// ResolveFromCwd resolves the agent from the current working directory +// It looks for package.yao in the current directory or parent directories +func (r *PathResolver) ResolveFromCwd() (*AgentInfo, error) { + cwd, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("failed to get current working directory: %w", err) + } + + info, err := r.ResolveFromPath(cwd) + if err != nil { + return nil, fmt.Errorf("no agent found in current directory. Use -n to specify agent explicitly") + } + return info, nil +} + +// ResolveFromPath resolves the agent by traversing up from the input file path +// It looks for package.yao in parent directories +func (r *PathResolver) ResolveFromPath(inputPath string) (*AgentInfo, error) { + // Get absolute path + absPath, err := filepath.Abs(inputPath) + if err != nil { + return nil, fmt.Errorf("failed to get absolute path: %w", err) + } + + // Start from the directory containing the input file + dir := filepath.Dir(absPath) + + // Traverse up to find package.yao + for { + packagePath := filepath.Join(dir, "package.yao") + if _, err := os.Stat(packagePath); err == nil { + // Found package.yao + return r.loadAgentFromPath(dir, packagePath) + } + + // Move to parent directory + parent := filepath.Dir(dir) + if parent == dir { + // Reached root, no package.yao found + break + } + dir = parent + } + + return nil, fmt.Errorf("no package.yao found in path hierarchy of %s", inputPath) +} + +// ResolveByID resolves an agent by its ID +// This would integrate with the assistant loading system +func (r *PathResolver) ResolveByID(agentID string) (*AgentInfo, error) { + // This is a placeholder - actual implementation would use assistant.Get() + // For now, return basic info + return &AgentInfo{ + ID: agentID, + Name: agentID, + }, nil +} + +// loadAgentFromPath loads agent information from a package.yao file +func (r *PathResolver) loadAgentFromPath(agentDir, packagePath string) (*AgentInfo, error) { + // Read package.yao + data, err := os.ReadFile(packagePath) + if err != nil { + return nil, fmt.Errorf("failed to read package.yao: %w", err) + } + + // Parse package.yao + var pkg PackageYao + if err := jsoniter.Unmarshal(data, &pkg); err != nil { + return nil, fmt.Errorf("failed to parse package.yao: %w", err) + } + + // Derive agent ID from directory path + agentID := deriveAgentID(agentDir) + + return &AgentInfo{ + ID: agentID, + Name: pkg.Name, + Description: pkg.Description, + Path: agentDir, + Connector: pkg.Connector, + Type: pkg.Type, + }, nil +} + +// PackageYao represents the structure of package.yao +type PackageYao struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Connector string `json:"connector,omitempty"` + Type string `json:"type,omitempty"` + Uses map[string]interface{} `json:"uses,omitempty"` + Options map[string]interface{} `json:"options,omitempty"` +} + +// deriveAgentID derives an agent ID from the directory path +// e.g., /app/assistants/workers/system/keyword -> workers.system.keyword +func deriveAgentID(dir string) string { + // Find "assistants" in path and use everything after it + parts := strings.Split(filepath.ToSlash(dir), "/") + + // Look for "assistants" marker + startIdx := -1 + for i, part := range parts { + if part == "assistants" { + startIdx = i + 1 + break + } + } + + if startIdx == -1 || startIdx >= len(parts) { + // No "assistants" found, use the last directory name + return filepath.Base(dir) + } + + // Join remaining parts with dots + return strings.Join(parts[startIdx:], ".") +} + +// GetOutputFormat determines the output format from file extension +func GetOutputFormat(outputPath string) OutputFormat { + ext := strings.ToLower(filepath.Ext(outputPath)) + switch ext { + case ".json": + return FormatJSON + case ".html", ".htm": + return FormatHTML + case ".md", ".markdown": + return FormatMarkdown + default: + return FormatJSON // Default to JSON + } +} + +// ValidateOptions validates test options +func ValidateOptions(opts *Options) error { + if opts.Input == "" { + return fmt.Errorf("input is required (-i flag)") + } + + // For file mode, check input file exists + if opts.InputMode == InputModeFile { + if _, err := os.Stat(opts.Input); os.IsNotExist(err) { + return fmt.Errorf("input file not found: %s", opts.Input) + } + } + + // Note: For message mode, agent can be resolved from cwd, so no validation here + // The resolver will return an error if agent cannot be found + + // Validate timeout + if opts.Timeout < 0 { + return fmt.Errorf("timeout cannot be negative") + } + + // Validate parallel + if opts.Parallel < 0 { + return fmt.Errorf("parallel cannot be negative") + } + + return nil +} + +// DefaultOptions returns options with default values +func DefaultOptions() *Options { + return &Options{ + Timeout: 5 * time.Minute, // 5 minutes default timeout + Parallel: 1, + Runs: 1, + Verbose: false, + FailFast: false, + } +} + +// DetectInputMode detects the input mode from the input string +// Returns InputModeFile if input looks like a file path, InputModeMessage otherwise +func DetectInputMode(input string) InputMode { + // If input ends with .jsonl or .json, treat as file + if strings.HasSuffix(input, ".jsonl") || strings.HasSuffix(input, ".json") { + return InputModeFile + } + + // If input contains path separator, check if file exists + if strings.Contains(input, string(filepath.Separator)) || strings.Contains(input, "/") { + if _, err := os.Stat(input); err == nil { + return InputModeFile + } + } + + // Otherwise treat as direct message + return InputModeMessage +} + +// MergeOptions merges user options with defaults +func MergeOptions(opts *Options, defaults *Options) *Options { + result := *defaults + + if opts.Input != "" { + result.Input = opts.Input + result.InputMode = DetectInputMode(opts.Input) + } + if opts.OutputFile != "" { + result.OutputFile = opts.OutputFile + } + if opts.AgentID != "" { + result.AgentID = opts.AgentID + } + if opts.Connector != "" { + result.Connector = opts.Connector + } + if opts.UserID != "" { + result.UserID = opts.UserID + } + if opts.TeamID != "" { + result.TeamID = opts.TeamID + } + if opts.Locale != "" { + result.Locale = opts.Locale + } + if opts.Timeout > 0 { + result.Timeout = opts.Timeout + } + if opts.Parallel > 0 { + result.Parallel = opts.Parallel + } + if opts.Runs > 0 { + result.Runs = opts.Runs + } + if opts.ReporterID != "" { + result.ReporterID = opts.ReporterID + } + if opts.Verbose { + result.Verbose = opts.Verbose + } + if opts.FailFast { + result.FailFast = opts.FailFast + } + + return &result +} + +// GenerateDefaultOutputPath generates the default output path based on input file +// Format: {input_directory}/output-{timestamp}.jsonl +// Timestamp format: YYYYMMDDHHMMSS +func GenerateDefaultOutputPath(inputPath string) string { + dir := filepath.Dir(inputPath) + timestamp := time.Now().Format("20060102150405") + filename := fmt.Sprintf("output-%s.jsonl", timestamp) + return filepath.Join(dir, filename) +} + +// ResolveOutputPath resolves the output path based on input mode +// - File mode: generate default path in same directory as input +// - Message mode: return empty string (output to stdout) +// If outputPath is explicitly specified, always use it +func ResolveOutputPath(opts *Options) string { + if opts.OutputFile != "" { + return opts.OutputFile + } + + // For file mode, generate default output path + if opts.InputMode == InputModeFile { + return GenerateDefaultOutputPath(opts.Input) + } + + // For message mode, output to stdout (empty string) + return "" +} + +// CreateTestCaseFromMessage creates a single test case from a direct message +func CreateTestCaseFromMessage(message string) *Case { + return &Case{ + ID: "T001", + Input: message, + } +} diff --git a/agent/test/runner.go b/agent/test/runner.go new file mode 100644 index 00000000..39140a0c --- /dev/null +++ b/agent/test/runner.go @@ -0,0 +1,505 @@ +package test + +import ( + "bufio" + stdContext "context" + "fmt" + "os" + "sync" + "time" + + jsoniter "github.com/json-iterator/go" + "github.com/yaoapp/yao/agent/assistant" + "github.com/yaoapp/yao/agent/context" +) + +// Executor executes test cases against an agent +type Executor struct { + opts *Options + output *OutputWriter + resolver Resolver + loader Loader +} + +// NewRunner creates a new test runner +func NewRunner(opts *Options) *Executor { + return &Executor{ + opts: opts, + output: NewOutputWriter(opts.Verbose), + resolver: NewResolver(), + loader: NewLoader(), + } +} + +// Run executes all test cases and returns a report +func (r *Executor) Run() (*Report, error) { + // For direct message mode, use simplified output (development mode) + if r.opts.InputMode == InputModeMessage { + return r.RunDirect() + } + + return r.RunTests() +} + +// RunDirect executes a single direct message and outputs the result directly +// This is optimized for development/debugging scenarios +func (r *Executor) RunDirect() (*Report, error) { + // Resolve agent + agentInfo, err := r.resolver.Resolve(r.opts) + if err != nil { + return nil, fmt.Errorf("failed to resolve agent: %w", err) + } + + // Get assistant + ast, err := assistant.Get(agentInfo.ID) + if err != nil { + return nil, fmt.Errorf("failed to get assistant: %w", err) + } + + // Create test case from message + tc := CreateTestCaseFromMessage(r.opts.Input) + + // Create context + chatID := GenerateChatID(tc.ID, 1) + ctx := NewTestContextFromOptions(chatID, agentInfo.ID, r.opts, tc) + defer ctx.Release() + + // Set connector override if specified + opts := &context.Options{} + if r.opts.Connector != "" { + opts.Connector = r.opts.Connector + } + + // Create timeout context + timeout := tc.GetTimeout(r.opts.Timeout) + timeoutCtx, cancel := stdContext.WithTimeout(ctx.Context, timeout) + defer cancel() + ctx.Context = timeoutCtx + + // Parse input to messages + messages, err := tc.GetMessages() + if err != nil { + return nil, fmt.Errorf("failed to parse input: %w", err) + } + + // Run the agent + response, err := ast.Stream(ctx, messages, opts) + + // Check for timeout + if timeoutCtx.Err() != nil { + return nil, fmt.Errorf("timeout after %s", timeout) + } + + // Check for error + if err != nil { + return nil, err + } + + // Extract and print output directly + output := extractOutput(response) + r.output.DirectOutput(output) + + // Return minimal report (for exit code handling) + return &Report{ + Summary: &Summary{ + Total: 1, + Passed: 1, + AgentID: agentInfo.ID, + }, + }, nil +} + +// RunTests executes test cases from file and generates a report +func (r *Executor) RunTests() (*Report, error) { + startTime := time.Now() + + // Print header + r.output.Header("Agent Test") + + // Resolve agent + agentInfo, err := r.resolver.Resolve(r.opts) + if err != nil { + return nil, fmt.Errorf("failed to resolve agent: %w", err) + } + + r.output.Info("Agent: %s", agentInfo.ID) + if r.opts.Connector != "" { + r.output.Info("Connector: %s (override)", r.opts.Connector) + } else if agentInfo.Connector != "" { + r.output.Info("Connector: %s", agentInfo.Connector) + } + + // Load test cases + var testCases []*Case + + // File mode - load from JSONL + testCases, err = r.loader.LoadFile(r.opts.Input) + if err != nil { + return nil, fmt.Errorf("failed to load test cases: %w", err) + } + r.output.Info("Input: %s (%d test cases)", r.opts.Input, len(testCases)) + + // Filter skipped tests + activeTests := FilterSkipped(testCases) + skippedCount := len(testCases) - len(activeTests) + if skippedCount > 0 { + r.output.Warning("Skipped: %d test cases", skippedCount) + } + + // Print test info + if r.opts.Runs > 1 { + r.output.Info("Runs: %d per test case (stability analysis)", r.opts.Runs) + } + r.output.Info("Timeout: %s", r.opts.Timeout) + if r.opts.Parallel > 1 { + r.output.Info("Parallel: %d", r.opts.Parallel) + } + + // Get assistant + ast, err := assistant.Get(agentInfo.ID) + if err != nil { + return nil, fmt.Errorf("failed to get assistant: %w", err) + } + + // Create report + report := &Report{ + Summary: &Summary{ + Total: len(testCases), + AgentID: agentInfo.ID, + AgentPath: agentInfo.Path, + Connector: r.opts.Connector, + RunsPerCase: r.opts.Runs, + }, + Environment: NewEnvironment(r.opts.UserID, r.opts.TeamID), + Metadata: &ReportMetadata{ + StartedAt: startTime, + InputFile: r.opts.Input, + Options: r.opts, + }, + } + + // Run tests + r.output.SubHeader("Running Tests") + + if r.opts.Runs > 1 { + // Stability testing mode + report.StabilityResults = r.runStabilityTests(ast, activeTests, agentInfo.ID) + r.calculateStabilitySummary(report) + } else { + // Single run mode + report.Results = r.runSingleTests(ast, activeTests, agentInfo.ID) + r.calculateSingleSummary(report) + } + + // Add skipped count + report.Summary.Skipped = skippedCount + + // Complete report + report.Summary.DurationMs = time.Since(startTime).Milliseconds() + report.Metadata.CompletedAt = time.Now() + + // Print summary + r.output.Summary(report.Summary, time.Since(startTime)) + + // Write output + if r.opts.OutputFile != "" { + err = r.writeOutput(report) + if err != nil { + r.output.Error("Failed to write output: %s", err.Error()) + } else { + r.output.OutputFile(r.opts.OutputFile) + } + } + + // Print final result + r.output.FinalResult(!report.HasFailures()) + + return report, nil +} + +// runSingleTests runs each test case once +func (r *Executor) runSingleTests(ast *assistant.Assistant, testCases []*Case, agentID string) []*Result { + results := make([]*Result, 0, len(testCases)) + + if r.opts.Parallel > 1 { + // Parallel execution + results = r.runParallel(ast, testCases, agentID) + } else { + // Sequential execution + for i, tc := range testCases { + result := r.runSingleTest(ast, tc, agentID, 1) + results = append(results, result) + + // Check fail-fast + if r.opts.FailFast && result.Status != StatusPassed && result.Status != StatusSkipped { + r.output.Warning("Stopping due to --fail-fast (failed at test %d/%d)", i+1, len(testCases)) + break + } + } + } + + return results +} + +// runParallel runs tests in parallel +func (r *Executor) runParallel(ast *assistant.Assistant, testCases []*Case, agentID string) []*Result { + results := make([]*Result, len(testCases)) + var wg sync.WaitGroup + sem := make(chan struct{}, r.opts.Parallel) + + for i, tc := range testCases { + wg.Add(1) + go func(idx int, testCase *Case) { + defer wg.Done() + sem <- struct{}{} // Acquire + defer func() { <-sem }() // Release + + results[idx] = r.runSingleTest(ast, testCase, agentID, 1) + }(i, tc) + } + + wg.Wait() + return results +} + +// runSingleTest runs a single test case +func (r *Executor) runSingleTest(ast *assistant.Assistant, tc *Case, agentID string, runNum int) *Result { + // Get input summary for display + inputSummary := SummarizeInput(tc.Input, 50) + r.output.TestStart(tc.ID, inputSummary, runNum) + + startTime := time.Now() + + // Create result + result := &Result{ + ID: tc.ID, + Input: tc.Input, + Expected: tc.Expected, + } + + // Parse input to messages + messages, err := tc.GetMessages() + if err != nil { + result.Status = StatusError + result.Error = fmt.Sprintf("failed to parse input: %s", err.Error()) + result.DurationMs = time.Since(startTime).Milliseconds() + r.output.TestResult(result.Status, time.Since(startTime)) + r.output.TestError(result.Error) + return result + } + + // Create context + chatID := GenerateChatID(tc.ID, runNum) + ctx := NewTestContextFromOptions(chatID, agentID, r.opts, tc) + defer ctx.Release() + + // Set connector override if specified + opts := &context.Options{} + if r.opts.Connector != "" { + opts.Connector = r.opts.Connector + } + + // Create timeout context + timeout := tc.GetTimeout(r.opts.Timeout) + timeoutCtx, cancel := stdContext.WithTimeout(ctx.Context, timeout) + defer cancel() + ctx.Context = timeoutCtx + + // Run the test + response, err := ast.Stream(ctx, messages, opts) + + duration := time.Since(startTime) + result.DurationMs = duration.Milliseconds() + + // Check for timeout + if timeoutCtx.Err() != nil { + result.Status = StatusTimeout + result.Error = fmt.Sprintf("timeout after %s", timeout) + r.output.TestResult(result.Status, duration) + r.output.TestError(result.Error) + return result + } + + // Check for error + if err != nil { + result.Status = StatusError + result.Error = err.Error() + r.output.TestResult(result.Status, duration) + r.output.TestError(result.Error) + return result + } + + // Extract output + result.Output = extractOutput(response) + + // Validate result if expected is set + if tc.Expected != nil { + if validateOutput(result.Output, tc.Expected) { + result.Status = StatusPassed + } else { + result.Status = StatusFailed + result.Error = "output does not match expected" + } + } else { + // No expected value - pass if no error + result.Status = StatusPassed + } + + r.output.TestResult(result.Status, duration) + if result.Status == StatusFailed { + r.output.TestError(result.Error) + } + r.output.TestOutput(fmt.Sprintf("%v", result.Output)) + + return result +} + +// runStabilityTests runs each test case multiple times for stability analysis +func (r *Executor) runStabilityTests(ast *assistant.Assistant, testCases []*Case, agentID string) []*StabilityResult { + results := make([]*StabilityResult, 0, len(testCases)) + + for _, tc := range testCases { + sr := &StabilityResult{ + ID: tc.ID, + Input: tc.Input, + Expected: tc.Expected, + RunDetails: make([]*RunDetail, 0, r.opts.Runs), + } + + // Run multiple times + for run := 1; run <= r.opts.Runs; run++ { + result := r.runSingleTest(ast, tc, agentID, run) + + rd := &RunDetail{ + Run: run, + Status: result.Status, + DurationMs: result.DurationMs, + Output: result.Output, + Error: result.Error, + } + sr.RunDetails = append(sr.RunDetails, rd) + } + + // Calculate stability metrics + sr.CalculateStability() + + // Print stability result + r.output.StabilityResult(sr) + + results = append(results, sr) + + // Check fail-fast + if r.opts.FailFast && !sr.Stable { + r.output.Warning("Stopping due to --fail-fast (unstable test: %s)", tc.ID) + break + } + } + + return results +} + +// calculateSingleSummary calculates summary for single run mode +func (r *Executor) calculateSingleSummary(report *Report) { + for _, result := range report.Results { + switch result.Status { + case StatusPassed: + report.Summary.Passed++ + case StatusFailed: + report.Summary.Failed++ + case StatusError: + report.Summary.Errors++ + case StatusTimeout: + report.Summary.Timeouts++ + } + } +} + +// calculateStabilitySummary calculates summary for stability mode +func (r *Executor) calculateStabilitySummary(report *Report) { + report.Summary.TotalRuns = len(report.StabilityResults) * r.opts.Runs + + var totalPassRate float64 + for _, sr := range report.StabilityResults { + if sr.Stable { + report.Summary.StableCases++ + report.Summary.Passed++ + } else { + report.Summary.UnstableCases++ + report.Summary.Failed++ + } + totalPassRate += sr.PassRate + } + + if len(report.StabilityResults) > 0 { + report.Summary.OverallPassRate = totalPassRate / float64(len(report.StabilityResults)) + } +} + +// writeOutput writes the test report to the output file +func (r *Executor) writeOutput(report *Report) error { + file, err := os.Create(r.opts.OutputFile) + if err != nil { + return fmt.Errorf("failed to create output file: %w", err) + } + defer file.Close() + + // Get reporter based on -r flag or file extension + reporter := GetReporterWithAgent(r.opts.ReporterID, r.opts.OutputFile, r.opts.Verbose) + + // If using agent reporter, set context + if agentReporter, ok := reporter.(*AgentReporter); ok { + // Create a context for the reporter agent call + ctx := NewTestContext("reporter", r.opts.ReporterID, report.Environment) + defer ctx.Release() + agentReporter.SetContext(ctx) + } + + // Write report using the reporter + return reporter.Write(report, file) +} + +// writeJSONLine writes a JSON line to the writer +func writeJSONLine(writer *bufio.Writer, data interface{}) error { + line, err := jsoniter.Marshal(data) + if err != nil { + return err + } + _, err = writer.Write(line) + if err != nil { + return err + } + _, err = writer.WriteString("\n") + return err +} + +// extractOutput extracts the output from the agent response +func extractOutput(response interface{}) interface{} { + if response == nil { + return nil + } + + // Try to get completion content from context.Response + if resp, ok := response.(*context.Response); ok { + if resp.Completion != nil { + return resp.Completion.Content + } + if resp.Next != nil { + return resp.Next + } + } + + return response +} + +// validateOutput validates the actual output against expected +func validateOutput(actual, expected interface{}) bool { + // Simple JSON comparison + actualJSON, err1 := jsoniter.Marshal(actual) + expectedJSON, err2 := jsoniter.Marshal(expected) + + if err1 != nil || err2 != nil { + return false + } + + return string(actualJSON) == string(expectedJSON) +} diff --git a/agent/test/types.go b/agent/test/types.go new file mode 100644 index 00000000..77b3f991 --- /dev/null +++ b/agent/test/types.go @@ -0,0 +1,575 @@ +package test + +import ( + "math" + "time" + + "github.com/yaoapp/yao/agent/context" +) + +// Status represents the status of a test case execution +type Status string + +const ( + // StatusPassed indicates the test passed + StatusPassed Status = "passed" + // StatusFailed indicates the test failed + StatusFailed Status = "failed" + // StatusSkipped indicates the test was skipped + StatusSkipped Status = "skipped" + // StatusError indicates a runtime error occurred + StatusError Status = "error" + // StatusTimeout indicates the test timed out + StatusTimeout Status = "timeout" +) + +// OutputFormat represents the output format for test reports +type OutputFormat string + +const ( + // FormatJSON outputs JSON format (for CI integration) + FormatJSON OutputFormat = "json" + // FormatHTML outputs HTML format (for human review) + FormatHTML OutputFormat = "html" + // FormatMarkdown outputs Markdown format (for documentation) + FormatMarkdown OutputFormat = "markdown" +) + +// StabilityClass represents the stability classification of a test case +type StabilityClass string + +const ( + // StabilityStable indicates 100% pass rate + StabilityStable StabilityClass = "stable" + // StabilityMostlyStable indicates 80-99% pass rate + StabilityMostlyStable StabilityClass = "mostly_stable" + // StabilityUnstable indicates 50-79% pass rate + StabilityUnstable StabilityClass = "unstable" + // StabilityHighlyUnstable indicates < 50% pass rate + StabilityHighlyUnstable StabilityClass = "highly_unstable" +) + +// InputMode represents the input mode for test cases +type InputMode string + +const ( + // InputModeFile indicates input from a JSONL file + InputModeFile InputMode = "file" + // InputModeMessage indicates input from a direct message string + InputModeMessage InputMode = "message" +) + +// Options represents the configuration options for running tests +type Options struct { + // Input/Output + // =============================== + + // Input is the input source: either a file path or a direct message + Input string `json:"input"` + + // InputMode is the input mode (auto-detected from Input) + InputMode InputMode `json:"input_mode"` + + // OutputFile is the path to write the test report + // Format is determined by file extension (.json, .html, .md) + OutputFile string `json:"output_file"` + + // Agent Selection + // =============================== + + // AgentID is the explicit agent ID to test (optional) + // If not set, agent is resolved from InputFile path + AgentID string `json:"agent_id,omitempty"` + + // Connector overrides the agent's default connector (optional) + Connector string `json:"connector,omitempty"` + + // Test Environment + // =============================== + + // UserID is the test user ID (-u flag) + UserID string `json:"user_id,omitempty"` + + // TeamID is the test team ID (-t flag) + TeamID string `json:"team_id,omitempty"` + + // Locale is the locale for the test context (default: "en-us") + Locale string `json:"locale,omitempty"` + + // Execution + // =============================== + + // Timeout is the default timeout for each test case + // Can be overridden per test case + Timeout time.Duration `json:"timeout,omitempty"` + + // Parallel is the number of tests to run in parallel + // Default is 1 (sequential execution) + Parallel int `json:"parallel,omitempty"` + + // Runs is the number of times to run each test case + // Default is 1. When > 1, stability metrics are collected + Runs int `json:"runs,omitempty"` + + // Reporting + // =============================== + + // ReporterID is the reporter agent ID for custom report generation + // If not set, default JSONL format is used + ReporterID string `json:"reporter_id,omitempty"` + + // Behavior + // =============================== + + // Verbose enables verbose output during test execution + Verbose bool `json:"verbose,omitempty"` + + // FailFast stops execution on first failure + FailFast bool `json:"fail_fast,omitempty"` +} + +// Environment configures the test execution context +type Environment struct { + // UserID is the user ID for authorized info (-u flag) + UserID string `json:"user_id"` + + // TeamID is the team ID for authorized info (-t flag) + TeamID string `json:"team_id"` + + // Locale is the locale (default: "en-us") + Locale string `json:"locale"` + + // ClientType is the client type (default: "test") + ClientType string `json:"client_type"` + + // ClientIP is the client IP (default: "127.0.0.1") + ClientIP string `json:"client_ip"` + + // Referer is the request referer (default: "test") + Referer string `json:"referer"` + + // Accept is the accept format (default: "standard") + Accept string `json:"accept"` +} + +// NewEnvironment creates a new test environment with defaults +func NewEnvironment(userID, teamID string) *Environment { + env := &Environment{ + UserID: userID, + TeamID: teamID, + Locale: "en-us", + ClientType: "test", + ClientIP: "127.0.0.1", + Referer: "test", + Accept: "standard", + } + + // Apply defaults if not set + if env.UserID == "" { + env.UserID = "test-user" + } + if env.TeamID == "" { + env.TeamID = "test-team" + } + + return env +} + +// Case represents a single test case loaded from JSONL +type Case struct { + // ID is the unique identifier for this test case (e.g., "T001") + ID string `json:"id"` + + // Input is the test input, can be: + // - string: simple text input + // - map (Message): single message with role and content + // - []map ([]Message): conversation history + Input interface{} `json:"input"` + + // Expected is the expected output for validation (optional) + // If set, the actual output will be compared against this + Expected interface{} `json:"expected,omitempty"` + + // Environment (per-test case, can be overridden by command line flags) + // =============================== + + // UserID is the user ID for this test case (overridden by -u flag) + UserID string `json:"user,omitempty"` + + // TeamID is the team ID for this test case (overridden by -t flag) + TeamID string `json:"team,omitempty"` + + // Metadata contains additional metadata for the test case + Metadata map[string]interface{} `json:"metadata,omitempty"` + + // Skip indicates whether to skip this test case + Skip bool `json:"skip,omitempty"` + + // Timeout overrides the default timeout for this test case + // Format: "30s", "1m", "2m30s" + Timeout string `json:"timeout,omitempty"` +} + +// GetEnvironment returns the effective test environment for this test case +// Priority: command line flags > test case fields > defaults +func (tc *Case) GetEnvironment(opts *Options) *Environment { + env := NewEnvironment("", "") + + // Apply test case specific values + if tc.UserID != "" { + env.UserID = tc.UserID + } + if tc.TeamID != "" { + env.TeamID = tc.TeamID + } + + // Apply command line overrides (highest priority) + if opts != nil { + if opts.UserID != "" { + env.UserID = opts.UserID + } + if opts.TeamID != "" { + env.TeamID = opts.TeamID + } + if opts.Locale != "" { + env.Locale = opts.Locale + } + } + + return env +} + +// GetMessages converts the Input to a slice of context.Message +// This handles all input formats: string, Message, []Message +func (tc *Case) GetMessages() ([]context.Message, error) { + return ParseInput(tc.Input) +} + +// GetTimeout returns the timeout duration for this test case +// Returns the override timeout if set, otherwise returns the default +func (tc *Case) GetTimeout(defaultTimeout time.Duration) time.Duration { + if tc.Timeout == "" { + return defaultTimeout + } + d, err := time.ParseDuration(tc.Timeout) + if err != nil { + return defaultTimeout + } + return d +} + +// Result represents the result of running a single test case +type Result struct { + // ID is the test case identifier + ID string `json:"id"` + + // Status is the test execution status + Status Status `json:"status"` + + // Input is the original test input (for reference in reports) + Input interface{} `json:"input"` + + // Output is the actual output from the agent + Output interface{} `json:"output,omitempty"` + + // Expected is the expected output (if specified in test case) + Expected interface{} `json:"expected,omitempty"` + + // DurationMs is the execution duration in milliseconds + DurationMs int64 `json:"duration_ms"` + + // Error contains the error message if status is failed/error/timeout + Error string `json:"error,omitempty"` + + // Metadata contains additional result metadata + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + +// RunDetail represents the result of a single run in stability testing +type RunDetail struct { + // Run is the run number (1-based) + Run int `json:"run"` + + // Status is the execution status for this run + Status Status `json:"status"` + + // DurationMs is the execution duration in milliseconds + DurationMs int64 `json:"duration_ms"` + + // Output is the output from this run + Output interface{} `json:"output,omitempty"` + + // Error contains the error message if this run failed + Error string `json:"error,omitempty"` +} + +// StabilityResult represents the stability analysis result for a test case +type StabilityResult struct { + // ID is the test case identifier + ID string `json:"id"` + + // Input is the original test input + Input interface{} `json:"input"` + + // Expected is the expected output (if specified) + Expected interface{} `json:"expected,omitempty"` + + // Runs is the total number of runs + Runs int `json:"runs"` + + // Passed is the number of runs that passed + Passed int `json:"passed"` + + // Failed is the number of runs that failed + Failed int `json:"failed"` + + // PassRate is the pass rate percentage (0-100) + PassRate float64 `json:"pass_rate"` + + // Consistency is a measure of output consistency (0-1) + // 1.0 means all outputs are identical, lower values indicate variation + Consistency float64 `json:"consistency"` + + // Stable indicates whether the test is considered stable + Stable bool `json:"stable"` + + // StabilityClass is the stability classification + StabilityClass StabilityClass `json:"stability_class"` + + // Timing statistics + AvgDurationMs float64 `json:"avg_duration_ms"` + MinDurationMs int64 `json:"min_duration_ms"` + MaxDurationMs int64 `json:"max_duration_ms"` + StdDeviationMs float64 `json:"std_deviation_ms"` + + // RunDetails contains details for each run + RunDetails []*RunDetail `json:"run_details"` +} + +// CalculateStability calculates stability metrics from run details +func (sr *StabilityResult) CalculateStability() { + if len(sr.RunDetails) == 0 { + return + } + + sr.Runs = len(sr.RunDetails) + sr.Passed = 0 + sr.Failed = 0 + + var totalDuration int64 + sr.MinDurationMs = math.MaxInt64 + sr.MaxDurationMs = 0 + + for _, rd := range sr.RunDetails { + if rd.Status == StatusPassed { + sr.Passed++ + } else { + sr.Failed++ + } + + totalDuration += rd.DurationMs + if rd.DurationMs < sr.MinDurationMs { + sr.MinDurationMs = rd.DurationMs + } + if rd.DurationMs > sr.MaxDurationMs { + sr.MaxDurationMs = rd.DurationMs + } + } + + // Calculate pass rate + sr.PassRate = float64(sr.Passed) / float64(sr.Runs) * 100 + + // Calculate average duration + sr.AvgDurationMs = float64(totalDuration) / float64(sr.Runs) + + // Calculate standard deviation + var sumSquares float64 + for _, rd := range sr.RunDetails { + diff := float64(rd.DurationMs) - sr.AvgDurationMs + sumSquares += diff * diff + } + sr.StdDeviationMs = math.Sqrt(sumSquares / float64(sr.Runs)) + + // Determine stability classification + sr.StabilityClass = ClassifyStability(sr.PassRate) + sr.Stable = sr.PassRate == 100 + + // Calculate consistency (simplified: based on pass rate) + sr.Consistency = sr.PassRate / 100 +} + +// ClassifyStability returns the stability classification based on pass rate +func ClassifyStability(passRate float64) StabilityClass { + switch { + case passRate == 100: + return StabilityStable + case passRate >= 80: + return StabilityMostlyStable + case passRate >= 50: + return StabilityUnstable + default: + return StabilityHighlyUnstable + } +} + +// Summary contains aggregated statistics for the test run +type Summary struct { + // Total number of test cases + Total int `json:"total"` + + // Passed number of test cases that passed + Passed int `json:"passed"` + + // Failed number of test cases that failed + Failed int `json:"failed"` + + // Skipped number of test cases that were skipped + Skipped int `json:"skipped"` + + // Errors number of test cases with runtime errors + Errors int `json:"errors"` + + // Timeouts number of test cases that timed out + Timeouts int `json:"timeouts"` + + // DurationMs is the total execution duration in milliseconds + DurationMs int64 `json:"duration_ms"` + + // AgentID is the ID of the agent being tested + AgentID string `json:"agent_id"` + + // AgentPath is the file path of the agent (for path-based resolution) + AgentPath string `json:"agent_path,omitempty"` + + // Connector is the connector used for the test + Connector string `json:"connector"` + + // Stability metrics (when Runs > 1) + // =============================== + + // RunsPerCase is the number of runs per test case + RunsPerCase int `json:"runs_per_case,omitempty"` + + // TotalRuns is the total number of runs (Total * RunsPerCase) + TotalRuns int `json:"total_runs,omitempty"` + + // OverallPassRate is the overall pass rate percentage + OverallPassRate float64 `json:"overall_pass_rate,omitempty"` + + // StableCases is the number of cases with 100% pass rate + StableCases int `json:"stable_cases,omitempty"` + + // UnstableCases is the number of cases with < 100% pass rate + UnstableCases int `json:"unstable_cases,omitempty"` +} + +// Report represents the complete test report +type Report struct { + // Summary contains aggregated statistics + Summary *Summary `json:"summary"` + + // Environment contains the test environment configuration + Environment *Environment `json:"environment,omitempty"` + + // Results contains individual test results (for single run) + Results []*Result `json:"results,omitempty"` + + // StabilityResults contains stability analysis results (for multiple runs) + StabilityResults []*StabilityResult `json:"stability_results,omitempty"` + + // Metadata contains additional report metadata + Metadata *ReportMetadata `json:"metadata"` +} + +// ReportMetadata contains metadata about the test report +type ReportMetadata struct { + // StartedAt is when the test run started + StartedAt time.Time `json:"started_at"` + + // CompletedAt is when the test run completed + CompletedAt time.Time `json:"completed_at"` + + // Version is the Yao version + Version string `json:"version"` + + // InputFile is the path to the input file + InputFile string `json:"input_file"` + + // OutputFile is the path to the output file + OutputFile string `json:"output_file"` + + // Options contains the test options used + Options *Options `json:"options,omitempty"` +} + +// HasFailures returns true if there are any failed, error, or timeout tests +func (r *Report) HasFailures() bool { + return r.Summary.Failed > 0 || r.Summary.Errors > 0 || r.Summary.Timeouts > 0 +} + +// PassRate returns the pass rate as a percentage (0-100) +func (r *Report) PassRate() float64 { + if r.Summary.Total == 0 { + return 0 + } + return float64(r.Summary.Passed) / float64(r.Summary.Total) * 100 +} + +// IsStabilityTest returns true if this is a stability test (multiple runs) +func (r *Report) IsStabilityTest() bool { + return r.Summary.RunsPerCase > 1 +} + +// AgentInfo contains information about the agent being tested +type AgentInfo struct { + // ID is the agent identifier + ID string `json:"id"` + + // Name is the human-readable name + Name string `json:"name"` + + // Description is the agent description + Description string `json:"description,omitempty"` + + // Path is the file system path to the agent + Path string `json:"path"` + + // Connector is the default connector + Connector string `json:"connector"` + + // Type is the agent type (e.g., "worker", "assistant") + Type string `json:"type,omitempty"` +} + +// ReporterInput is the input passed to a custom reporter agent +type ReporterInput struct { + // Report is the test report to format + Report *Report `json:"report"` + + // Format is the desired output format + Format string `json:"format"` + + // Options contains additional formatting options + Options *ReporterOptions `json:"options,omitempty"` +} + +// ReporterOptions contains options for custom reporter agents +type ReporterOptions struct { + // Verbose includes detailed output in the report + Verbose bool `json:"verbose,omitempty"` + + // IncludeOutputs includes full outputs in the report + IncludeOutputs bool `json:"include_outputs,omitempty"` + + // IncludeInputs includes full inputs in the report + IncludeInputs bool `json:"include_inputs,omitempty"` + + // MaxOutputLength limits the output length in the report + MaxOutputLength int `json:"max_output_length,omitempty"` + + // Theme is the report theme (for HTML reports) + Theme string `json:"theme,omitempty"` + + // Title is the report title + Title string `json:"title,omitempty"` +} diff --git a/cmd/README.md b/cmd/README.md new file mode 100644 index 00000000..544a96c5 --- /dev/null +++ b/cmd/README.md @@ -0,0 +1,373 @@ +# Yao CLI Commands + +The Yao CLI provides a set of commands for managing, running, and testing Yao applications. + +## Installation + +```bash +# Build from source +go build -o yao . + +# Or install via go install +go install github.com/yaoapp/yao@latest +``` + +## Global Flags + +| Flag | Short | Description | +|------|-------|-------------| +| `--app` | `-a` | Application directory path | +| `--file` | `-f` | Application package file (.yaz) | +| `--key` | `-k` | Application license key | + +## Environment Variables + +| Variable | Description | +|----------|-------------| +| `YAO_ROOT` | Application root directory | +| `YAO_LANG` | Language setting (e.g., `zh-CN` for Chinese) | + +## Commands + +### `yao start` + +Start the Yao application engine. + +```bash +# Start in current directory +yao start + +# Start with specific app directory +yao start -a /path/to/app + +# Start in debug mode +yao start --debug +``` + +**Flags:** + +| Flag | Description | +|------|-------------| +| `--debug` | Enable development/debug mode | +| `--disable-watching` | Disable file watching | + +--- + +### `yao run` + +Execute a Yao process. + +```bash +# Run a process +yao run models.user.Find 1 + +# Run with JSON arguments +yao run models.user.Create '::[{"name":"John","age":30}]' + +# Run in silent mode (JSON output only) +yao run -s models.user.Find 1 +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--silent` | `-s` | Silent mode - output result as JSON only | + +**Argument Syntax:** + +- Regular arguments: `arg1 arg2` +- JSON arguments: `'::[{"key":"value"}]'` (prefix with `::`) +- Escaped `::`: `'\::literal'` + +--- + +### `yao migrate` + +Update database schema based on model definitions. + +```bash +# Migrate all models +yao migrate + +# Migrate specific model +yao migrate -n user + +# Force migrate in production mode +yao migrate --force + +# Reset (drop and recreate) tables +yao migrate --reset +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--name` | `-n` | Specific model name to migrate | +| `--force` | | Force migrate in production mode | +| `--reset` | | Drop tables before migration | + +--- + +### `yao inspect` + +Display application configuration. + +```bash +yao inspect +``` + +--- + +### `yao version` + +Show Yao version information. + +```bash +# Show version +yao version + +# Show all version details +yao version --all +``` + +**Flags:** + +| Flag | Description | +|------|-------------| +| `--all` | Print all version information (Go version, commit, build time, etc.) | + +--- + +## Agent Commands + +Commands for testing and managing AI agents. + +### `yao agent test` + +Test an agent with input cases from a JSONL file or direct message. + +```bash +# Test with direct message (development mode) +yao agent test -i "Extract keywords from: AI and machine learning" -n workers.system.keyword + +# Test with JSONL file +yao agent test -i tests/inputs.jsonl + +# Test with custom output file +yao agent test -i tests/inputs.jsonl -o report.html + +# Test with specific connector +yao agent test -i tests/inputs.jsonl -c openai.gpt4 + +# Stability testing (multiple runs) +yao agent test -i tests/inputs.jsonl --runs 5 + +# Parallel execution +yao agent test -i tests/inputs.jsonl --parallel 4 + +# Verbose output +yao agent test -i tests/inputs.jsonl -v +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--input` | `-i` | Input: JSONL file path or direct message (required) | +| `--output` | `-o` | Output file path (default: `output-{timestamp}.jsonl`) | +| `--name` | `-n` | Agent ID (default: auto-detect from path) | +| `--connector` | `-c` | Override default connector | +| `--user` | `-u` | Test user ID (default: `test-user`) | +| `--team` | `-t` | Test team ID (default: `test-team`) | +| `--reporter` | `-r` | Reporter agent ID for custom report generation | +| `--runs` | | Number of runs per test case for stability analysis (default: 1) | +| `--timeout` | | Timeout per test case (default: `5m`) | +| `--parallel` | | Number of parallel test cases (default: 1) | +| `--verbose` | `-v` | Enable verbose output | +| `--fail-fast` | | Stop on first failure | +| `--app` | `-a` | Application directory | +| `--env` | `-e` | Environment file | + +**Input Modes:** + +1. **Direct Message Mode**: For quick development/debugging + ```bash + yao agent test -i "Hello world" -n my.agent + ``` + - Outputs result directly to stdout + - No report file generated + - Ideal for iterative development + +2. **File Mode**: For comprehensive testing + ```bash + yao agent test -i tests/inputs.jsonl + ``` + - Reads test cases from JSONL file + - Generates detailed report + - Supports stability analysis + +**JSONL Input Format:** + +```jsonl +{"id": "T001", "input": "Simple text input"} +{"id": "T002", "input": {"role": "user", "content": "Message with role"}} +{"id": "T003", "input": [{"role": "system", "content": "System prompt"}, {"role": "user", "content": "User message"}]} +{"id": "T004", "input": "Test with timeout", "timeout": "30s"} +{"id": "T005", "input": "Skip this test", "skip": true} +{"id": "T006", "input": "Test with specific user", "user": "alice", "team": "engineering"} +``` + +**Output Formats:** + +| Extension | Format | Description | +|-----------|--------|-------------| +| `.jsonl` | JSONL | Streaming format (default) | +| `.json` | JSON | Complete structured report | +| `.md` | Markdown | Human-readable with tables | +| `.html` | HTML | Interactive web report | + +**Agent Resolution:** + +The agent is resolved in the following priority order: + +1. Explicit `-n` flag: `yao agent test -i msg -n my.agent` +2. `YAO_ROOT` environment variable +3. Auto-detect from input file path (traverses up to find `package.yao`) +4. Auto-detect from current working directory + +--- + +## SUI Commands + +SUI (Serverless UI) template engine commands. + +### `yao sui watch` + +Auto-build templates when files change. + +```bash +yao sui watch [data] + +# Example +yao sui watch default index '::{}' +``` + +### `yao sui build` + +Build a template. + +```bash +yao sui build [data] + +# Example +yao sui build default index '::{}' + +# Debug mode +yao sui build default index '::{}' --debug +``` + +### `yao sui trans` + +Translate template content. + +```bash +yao sui trans + +# With specific locales +yao sui trans default index -l "en-US,zh-CN,ja-JP" +``` + +**SUI Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--data` | `-d` | Session data as JSON (prefix with `::`) | +| `--debug` | `-D` | Enable debug mode | +| `--locales` | `-l` | Locales for translation (comma-separated) | + +--- + +## Examples + +### Development Workflow + +```bash +# Start development server +yao start --debug + +# Run a process +yao run scripts.test.Hello "World" + +# Test an agent interactively +yao agent test -i "What is the weather today?" -n assistant.weather + +# Watch and auto-build templates +yao sui watch default home +``` + +### Testing Workflow + +```bash +# Run comprehensive agent tests +yao agent test -i tests/inputs.jsonl -o report.html -v + +# Stability analysis (run each test 10 times) +yao agent test -i tests/inputs.jsonl --runs 10 -o stability-report.json + +# Parallel testing with timeout +yao agent test -i tests/inputs.jsonl --parallel 4 --timeout 2m + +# CI/CD integration +yao agent test -i tests/inputs.jsonl -o results.jsonl && echo "Tests passed" +``` + +### Database Migration + +```bash +# Migrate all models +yao migrate + +# Migrate specific model with reset +yao migrate -n user --reset --force +``` + +--- + +## Exit Codes + +| Code | Description | +|------|-------------| +| 0 | Success | +| 1 | Error or test failure | + +--- + +## Directory Structure + +``` +myapp/ +โ”œโ”€โ”€ app.yao # Application configuration +โ”œโ”€โ”€ .env # Environment variables +โ”œโ”€โ”€ models/ # Data models +โ”œโ”€โ”€ apis/ # API definitions +โ”œโ”€โ”€ flows/ # Business flows +โ”œโ”€โ”€ scripts/ # JavaScript/TypeScript scripts +โ”œโ”€โ”€ assistants/ # AI agents +โ”‚ โ””โ”€โ”€ my-agent/ +โ”‚ โ”œโ”€โ”€ package.yao # Agent configuration +โ”‚ โ”œโ”€โ”€ prompts.yml # Agent prompts +โ”‚ โ””โ”€โ”€ tests/ +โ”‚ โ””โ”€โ”€ inputs.jsonl # Test cases +โ””โ”€โ”€ public/ # Static files +``` + +--- + +## See Also + +- [Yao Documentation](https://yaoapps.com/docs) +- [Agent Test Design](../agent/test/DESIGN.md) +- [SUI Documentation](https://yaoapps.com/docs/sui) + diff --git a/cmd/agent/agent.go b/cmd/agent/agent.go new file mode 100644 index 00000000..d5fcffaf --- /dev/null +++ b/cmd/agent/agent.go @@ -0,0 +1,72 @@ +package agent + +import ( + "os" + "path/filepath" + + "github.com/yaoapp/kun/exception" + "github.com/yaoapp/yao/config" +) + +var appPath string +var envFile string + +var langs = map[string]string{ + "Test an agent with input cases": "ไฝฟ็”จๆต‹่ฏ•็”จไพ‹ๆต‹่ฏ•ๆ™บ่ƒฝไฝ“", + "Test an agent with input cases from JSONL file or direct message": "ไฝฟ็”จ JSONL ๆ–‡ไปถๆˆ–็›ดๆŽฅๆถˆๆฏๆต‹่ฏ•ๆ™บ่ƒฝไฝ“", + "Application directory": "ๅบ”็”จ็›ฎๅฝ•", + "Environment file": "็Žฏๅขƒๅ˜้‡ๆ–‡ไปถ", + "Input: JSONL file path or message (required)": "่พ“ๅ…ฅ: JSONL ๆ–‡ไปถ่ทฏๅพ„ๆˆ–ๆถˆๆฏ (ๅฟ…้œ€)", + "Path to output file (default: output-{timestamp}.jsonl)": "่พ“ๅ‡บๆ–‡ไปถ่ทฏๅพ„ (้ป˜่ฎค: output-{timestamp}.jsonl)", + "Explicit agent ID (default: auto-detect)": "ๆŒ‡ๅฎšๆ™บ่ƒฝไฝ“ ID (้ป˜่ฎค: ่‡ชๅŠจๆฃ€ๆต‹)", + "Override connector": "่ฆ†็›–่ฟžๆŽฅๅ™จ", + "Test user ID (default: test-user)": "ๆต‹่ฏ•็”จๆˆท ID (้ป˜่ฎค: test-user)", + "Test team ID (default: test-team)": "ๆต‹่ฏ•ๅ›ข้˜Ÿ ID (้ป˜่ฎค: test-team)", + "Reporter agent ID for custom report": "่‡ชๅฎšไน‰ๆŠฅๅ‘Š็”Ÿๆˆๅ™จๆ™บ่ƒฝไฝ“ ID", + "Number of runs for stability analysis": "็จณๅฎšๆ€งๅˆ†ๆž็š„่ฟ่กŒๆฌกๆ•ฐ", + "Default timeout per test case": "ๆฏไธชๆต‹่ฏ•็”จไพ‹็š„้ป˜่ฎค่ถ…ๆ—ถๆ—ถ้—ด", + "Number of parallel test cases": "ๅนถ่กŒๆต‹่ฏ•็”จไพ‹ๆ•ฐ", + "Verbose output": "่ฏฆ็ป†่พ“ๅ‡บ", + "Stop on first failure": "้‡ๅˆฐ็ฌฌไธ€ไธชๅคฑ่ดฅๆ—ถๅœๆญข", + "Error: input is required (-i flag)": "้”™่ฏฏ: ้œ€่ฆ่พ“ๅ…ฅ (-i ๅ‚ๆ•ฐ)", + "Error: failed to get current directory": "้”™่ฏฏ: ่Žทๅ–ๅฝ“ๅ‰็›ฎๅฝ•ๅคฑ่ดฅ", + "Error: agent (-n) is required when using direct message input and not in an agent directory": "้”™่ฏฏ: ไฝฟ็”จ็›ดๆŽฅๆถˆๆฏ่พ“ๅ…ฅไธ”ไธๅœจๆ™บ่ƒฝไฝ“็›ฎๅฝ•ๆ—ถ้œ€่ฆๆŒ‡ๅฎš -n ๅ‚ๆ•ฐ", + "Hint: Make sure you're in a Yao application directory or specify --app flag": "ๆ็คบ: ็กฎไฟๅœจ Yao ๅบ”็”จ็›ฎๅฝ•ไธญๆˆ–ไฝฟ็”จ --app ๅ‚ๆ•ฐๆŒ‡ๅฎš", + "Error: invalid timeout format": "้”™่ฏฏ: ๆ— ๆ•ˆ็š„่ถ…ๆ—ถๆ ผๅผ", +} + +// L Language switch +func L(words string) string { + var lang = os.Getenv("YAO_LANG") + if lang == "" { + return words + } + + if trans, has := langs[words]; has { + return trans + } + return words +} + +// Boot sets the configuration +func Boot() { + root := config.Conf.Root + if appPath != "" { + r, err := filepath.Abs(appPath) + if err != nil { + exception.New("Root error %s", 500, err.Error()).Throw() + } + root = r + } + if envFile != "" { + config.Conf = config.LoadFrom(envFile) + } else { + config.Conf = config.LoadFrom(filepath.Join(root, ".env")) + } + + if config.Conf.Mode == "production" { + config.Production() + } else if config.Conf.Mode == "development" { + config.Development() + } +} diff --git a/cmd/agent/test.go b/cmd/agent/test.go new file mode 100644 index 00000000..3098a5e3 --- /dev/null +++ b/cmd/agent/test.go @@ -0,0 +1,244 @@ +package agent + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/yaoapp/gou/plugin" + "github.com/yaoapp/yao/agent" + "github.com/yaoapp/yao/agent/test" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/engine" + "github.com/yaoapp/yao/kb" + "github.com/yaoapp/yao/share" +) + +// Test command flags +var ( + testInput string + testOutput string + testAgent string + testConnector string + testUser string + testTeam string + testReporter string + testRuns int + testTimeout string + testParallel int + testVerbose bool + testFailFast bool +) + +// TestCmd is the agent test command +var TestCmd = &cobra.Command{ + Use: "test", + Short: L("Test an agent with input cases"), + Long: L("Test an agent with input cases from JSONL file or direct message"), + Run: func(cmd *cobra.Command, args []string) { + defer share.SessionStop() + defer plugin.KillAll() + + // Validate input + if testInput == "" { + color.Red(L("Error: input is required (-i flag)") + "\n") + os.Exit(1) + } + + // Detect input mode + inputMode := test.DetectInputMode(testInput) + + // For message mode, agent must be specified or resolvable from cwd + if inputMode == test.InputModeMessage && testAgent == "" { + // Try to find app root from current directory + cwd, err := os.Getwd() + if err != nil { + color.Red(L("Error: failed to get current directory")+": %s\n", err.Error()) + os.Exit(1) + } + + // Try to find package.yao from cwd + resolver := test.NewResolver() + _, err = resolver.ResolveFromPath(cwd) + if err != nil { + color.Red(L("Error: agent (-n) is required when using direct message input and not in an agent directory") + "\n") + os.Exit(1) + } + } + + // Find app root directory + // Priority: -a flag > YAO_ROOT env > auto-detect from path + var err error + + if appPath == "" { + // Check YAO_ROOT environment variable + if yaoRoot := os.Getenv("YAO_ROOT"); yaoRoot != "" { + appPath = yaoRoot + } + } + + if appPath == "" { + // Auto-detect from path + if inputMode == test.InputModeFile { + // For file mode, find app root from input file path + appPath, err = findAppRoot(testInput) + } else { + // For message mode, find app root from current directory + cwd, _ := os.Getwd() + appPath, err = findAppRoot(cwd) + } + + if err != nil { + color.Red("Error: %s\n", err.Error()) + color.Yellow(L("Hint: Make sure you're in a Yao application directory or specify --app flag") + "\n") + os.Exit(1) + } + } + + // Boot the application + Boot() + + // Set Runtime Mode + config.Conf.Runtime.Mode = "standard" + cfg := config.Conf + cfg.Session.IsCLI = true + + // Load engine + _, err = engine.Load(cfg, engine.LoadOption{Action: "agent-test"}) + if err != nil { + color.Red("Engine: %s\n", err.Error()) + os.Exit(1) + } + + // Load KB (required for agent KB features) + _, err = kb.Load(cfg) + if err != nil { + color.Red("KB: %s\n", err.Error()) + os.Exit(1) + } + + // Load agent + err = agent.Load(cfg) + if err != nil { + color.Red("Agent: %s\n", err.Error()) + os.Exit(1) + } + + // Parse timeout + timeout := 5 * time.Minute + if testTimeout != "" { + d, err := time.ParseDuration(testTimeout) + if err != nil { + color.Red(L("Error: invalid timeout format")+": %s\n", testTimeout) + os.Exit(1) + } + timeout = d + } + + // Build test options + opts := &test.Options{ + Input: testInput, + InputMode: inputMode, + OutputFile: testOutput, + AgentID: testAgent, + Connector: testConnector, + UserID: testUser, + TeamID: testTeam, + ReporterID: testReporter, + Runs: testRuns, + Timeout: timeout, + Parallel: testParallel, + Verbose: testVerbose, + FailFast: testFailFast, + } + + // Merge with defaults + opts = test.MergeOptions(opts, test.DefaultOptions()) + + // Resolve output path (only for file mode, direct message mode outputs to stdout) + if inputMode == test.InputModeFile { + opts.OutputFile = test.ResolveOutputPath(opts) + } + + // Run tests + runner := test.NewRunner(opts) + report, err := runner.Run() + if err != nil { + color.Red("Error: %s\n", err.Error()) + os.Exit(1) + } + + // Exit with appropriate code + if report.HasFailures() { + os.Exit(1) + } + }, +} + +// findAppRoot finds the Yao application root directory by looking for app.yao +// It traverses up from the given path until it finds app.yao or reaches the filesystem root +func findAppRoot(startPath string) (string, error) { + // Get absolute path + absPath, err := filepath.Abs(startPath) + if err != nil { + return "", fmt.Errorf("failed to get absolute path: %w", err) + } + + // If it's a file, start from its directory + info, err := os.Stat(absPath) + if err != nil { + return "", fmt.Errorf("path not found: %s", absPath) + } + + var dir string + if info.IsDir() { + dir = absPath + } else { + dir = filepath.Dir(absPath) + } + + // Traverse up to find app.yao + for { + // Check for app.yao, app.json, or app.jsonc + for _, appFile := range []string{"app.yao", "app.json", "app.jsonc"} { + appFilePath := filepath.Join(dir, appFile) + if _, err := os.Stat(appFilePath); err == nil { + return dir, nil + } + } + + // Move to parent directory + parent := filepath.Dir(dir) + if parent == dir { + // Reached root, no app.yao found + break + } + dir = parent + } + + return "", fmt.Errorf("no app.yao found in path hierarchy of %s", startPath) +} + +func init() { + // Test command flags + TestCmd.Flags().StringVarP(&appPath, "app", "a", "", L("Application directory")) + TestCmd.Flags().StringVarP(&envFile, "env", "e", "", L("Environment file")) + TestCmd.Flags().StringVarP(&testInput, "input", "i", "", L("Input: JSONL file path or message (required)")) + TestCmd.Flags().StringVarP(&testOutput, "output", "o", "", L("Path to output file (default: output-{timestamp}.jsonl)")) + TestCmd.Flags().StringVarP(&testAgent, "name", "n", "", L("Explicit agent ID (default: auto-detect)")) + TestCmd.Flags().StringVarP(&testConnector, "connector", "c", "", L("Override connector")) + TestCmd.Flags().StringVarP(&testUser, "user", "u", "", L("Test user ID (default: test-user)")) + TestCmd.Flags().StringVarP(&testTeam, "team", "t", "", L("Test team ID (default: test-team)")) + TestCmd.Flags().StringVarP(&testReporter, "reporter", "r", "", L("Reporter agent ID for custom report")) + TestCmd.Flags().IntVar(&testRuns, "runs", 1, L("Number of runs for stability analysis")) + TestCmd.Flags().StringVar(&testTimeout, "timeout", "5m", L("Default timeout per test case")) + TestCmd.Flags().IntVar(&testParallel, "parallel", 1, L("Number of parallel test cases")) + TestCmd.Flags().BoolVarP(&testVerbose, "verbose", "v", false, L("Verbose output")) + TestCmd.Flags().BoolVar(&testFailFast, "fail-fast", false, L("Stop on first failure")) + + // Mark input as required + TestCmd.MarkFlagRequired("input") +} diff --git a/cmd/root.go b/cmd/root.go index f5a62e13..b89f4c96 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -7,6 +7,7 @@ import ( "github.com/spf13/cobra" "github.com/yaoapp/kun/exception" + "github.com/yaoapp/yao/cmd/agent" "github.com/yaoapp/yao/cmd/sui" "github.com/yaoapp/yao/config" "github.com/yaoapp/yao/pack" @@ -84,7 +85,6 @@ var rootCmd = &cobra.Command{ Use: share.BUILDNAME, Short: "Yao App Engine", Long: `Yao App Engine`, - Args: cobra.MinimumNArgs(1), CompletionOptions: cobra.CompletionOptions{ DisableDefaultCmd: true, }, @@ -93,10 +93,14 @@ var rootCmd = &cobra.Command{ switch args[0] { case "fuxi": fuxi() + return } + fmt.Fprintln(os.Stderr, L("One or more arguments are not correct"), args) + os.Exit(1) + return } - fmt.Fprintln(os.Stderr, L("One or more arguments are not correct"), args) - os.Exit(1) + // No arguments - show help + cmd.Help() }, } @@ -104,13 +108,11 @@ var studioCmd = &cobra.Command{ Use: "studio", Short: "Yao Studio CLI", Long: `Yao Studio CLI`, - Args: cobra.MinimumNArgs(1), CompletionOptions: cobra.CompletionOptions{ DisableDefaultCmd: true, }, Run: func(cmd *cobra.Command, args []string) { - fmt.Fprintln(os.Stderr, L("One or more arguments are not correct"), args) - os.Exit(1) + cmd.Help() }, } @@ -118,13 +120,23 @@ var suiCmd = &cobra.Command{ Use: "sui", Short: L("SUI Template Engine"), Long: L("SUI Template Engine"), - Args: cobra.MinimumNArgs(1), CompletionOptions: cobra.CompletionOptions{ DisableDefaultCmd: true, }, Run: func(cmd *cobra.Command, args []string) { - fmt.Fprintln(os.Stderr, L("One or more arguments are not correct"), args) - os.Exit(1) + cmd.Help() + }, +} + +var agentCmd = &cobra.Command{ + Use: "agent", + Short: L("Agent commands"), + Long: L("Agent commands for testing and management"), + CompletionOptions: cobra.CompletionOptions{ + DisableDefaultCmd: true, + }, + Run: func(cmd *cobra.Command, args []string) { + cmd.Help() }, } @@ -138,6 +150,9 @@ func init() { suiCmd.AddCommand(sui.BuildCmd) suiCmd.AddCommand(sui.TransCmd) + // Agent + agentCmd.AddCommand(agent.TestCmd) + rootCmd.AddCommand( versionCmd, migrateCmd, @@ -152,6 +167,7 @@ func init() { // packCmd, // studioCmd, suiCmd, + agentCmd, // upgradeCmd, ) // rootCmd.SetHelpCommand(helpCmd) diff --git a/cmd/run.go b/cmd/run.go index d1ce8c30..aa840ae8 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -3,6 +3,8 @@ package cmd import ( "context" "fmt" + "os" + "path/filepath" "strings" "github.com/fatih/color" @@ -40,6 +42,16 @@ var runCmd = &cobra.Command{ } }() + // Auto-detect app root if not specified + if appPath == "" { + cwd, err := os.Getwd() + if err == nil { + if root, err := findAppRootFromPath(cwd); err == nil { + appPath = root + } + } + } + Boot() // Set Runtime Mode @@ -174,3 +186,47 @@ var runCmd = &cobra.Command{ func init() { runCmd.PersistentFlags().BoolVarP(&runSilent, "silent", "s", false, L("Silent mode")) } + +// findAppRootFromPath finds the Yao application root directory by looking for app.yao +// It traverses up from the given path until it finds app.yao or reaches the filesystem root +func findAppRootFromPath(startPath string) (string, error) { + // Get absolute path + absPath, err := filepath.Abs(startPath) + if err != nil { + return "", fmt.Errorf("failed to get absolute path: %w", err) + } + + // If it's a file, start from its directory + info, err := os.Stat(absPath) + if err != nil { + return "", fmt.Errorf("path not found: %s", absPath) + } + + var dir string + if info.IsDir() { + dir = absPath + } else { + dir = filepath.Dir(absPath) + } + + // Traverse up to find app.yao + for { + // Check for app.yao, app.json, or app.jsonc + for _, appFile := range []string{"app.yao", "app.json", "app.jsonc"} { + appFilePath := filepath.Join(dir, appFile) + if _, err := os.Stat(appFilePath); err == nil { + return dir, nil + } + } + + // Move to parent directory + parent := filepath.Dir(dir) + if parent == dir { + // Reached root, no app.yao found + break + } + dir = parent + } + + return "", fmt.Errorf("no app.yao found in path hierarchy of %s", startPath) +} diff --git a/openapi/config.go b/openapi/config.go index 0deb31e3..9925ef37 100644 --- a/openapi/config.go +++ b/openapi/config.go @@ -2,7 +2,7 @@ package openapi import ( "errors" - "fmt" + // "fmt" "path/filepath" "strings" "time" @@ -135,10 +135,10 @@ func (config *Config) UnmarshalJSON(data []byte) error { Features: tempConfig.OAuth.Features, } - fmt.Println("----debug----") - fmt.Println("tempConfig.OAuth.IssuerURL", tempConfig.OAuth.IssuerURL) - fmt.Println("config.OAuth.IssuerURL", config.OAuth.IssuerURL) - fmt.Println("----debug----") + // fmt.Println("----debug----") + // fmt.Println("tempConfig.OAuth.IssuerURL", tempConfig.OAuth.IssuerURL) + // fmt.Println("config.OAuth.IssuerURL", config.OAuth.IssuerURL) + // fmt.Println("----debug----") // Convert signing config with duration parsing config.OAuth.Signing = types.SigningConfig{